{"item_id":"senseval2.d000.s000.t000","gold_sense_keys":["art%1:09:00::"],"candidates":[{"index":1,"sense_key":"art%1:06:00::","synset_id":"art.n.01"},{"index":2,"sense_key":"art%1:04:00::","synset_id":"art.n.02"},{"index":3,"sense_key":"art%1:09:00::","synset_id":"art.n.03"},{"index":4,"sense_key":"art%1:10:00::","synset_id":"artwork.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"art%1:09:00::","call_ids":["senseval2.d000.s000.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"art%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":573,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001266,"input_uncached_usd":0.0011459999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.474544959026389}
{"item_id":"senseval2.d000.s000.t002","gold_sense_keys":["peculiar%5:00:00:characteristic:00","peculiar%5:00:00:specific:00"],"candidates":[{"index":1,"sense_key":"peculiar%5:00:00:strange:00","synset_id":"curious.s.01"},{"index":2,"sense_key":"peculiar%5:00:00:specific:00","synset_id":"particular.s.01"},{"index":3,"sense_key":"peculiar%5:00:00:unusual:00","synset_id":"peculiar.s.03"},{"index":4,"sense_key":"peculiar%5:00:00:characteristic:00","synset_id":"peculiar.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"peculiar%5:00:00:characteristic:00","call_ids":["senseval2.d000.s000.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"peculiar%5:00:00:characteristic:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":636,"cached_input_tokens":0,"output_tokens":221,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003482,"input_uncached_usd":0.001272,"input_cached_usd":0.0,"output_usd":0.00221,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.9049250000389293}
{"item_id":"senseval2.d000.s000.t003","gold_sense_keys":["english%1:18:00::"],"candidates":[{"index":1,"sense_key":"english%1:10:00::","synset_id":"english.n.01"},{"index":2,"sense_key":"english%1:18:00::","synset_id":"english.n.02"},{"index":3,"sense_key":"english%1:09:00::","synset_id":"english.n.03"},{"index":4,"sense_key":"english%1:11:00::","synset_id":"english.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"english%1:18:00::","call_ids":["senseval2.d000.s000.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"english%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":489,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001098,"input_uncached_usd":0.000978,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8124938749242574}
{"item_id":"senseval2.d000.s000.t004","gold_sense_keys":["most%3:00:02::"],"candidates":[{"index":1,"sense_key":"most%3:00:02::","synset_id":"most.a.01"},{"index":2,"sense_key":"most%3:00:01::","synset_id":"most.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"most%3:00:02::","call_ids":["senseval2.d000.s000.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"most%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":459,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0016480000000000002,"input_uncached_usd":0.000918,"input_cached_usd":0.0,"output_usd":0.0007300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2742488749790937}
{"item_id":"senseval2.d000.s000.t005","gold_sense_keys":["english%3:01:00::"],"candidates":[{"index":1,"sense_key":"english%3:01:00::","synset_id":"english.a.01"},{"index":2,"sense_key":"english%3:01:01::","synset_id":"english.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"english%3:01:00::","call_ids":["senseval2.d000.s000.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"english%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":376,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0008719999999999999,"input_uncached_usd":0.000752,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.803010666044429}
{"item_id":"senseval2.d000.s000.t006","gold_sense_keys":["peculiarity%1:07:02::","peculiarity%1:09:00::"],"candidates":[{"index":1,"sense_key":"peculiarity%1:09:00::","synset_id":"peculiarity.n.01"},{"index":2,"sense_key":"peculiarity%1:07:02::","synset_id":"peculiarity.n.02"},{"index":3,"sense_key":"peculiarity%1:06:00::","synset_id":"curio.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"peculiarity%1:09:00::","call_ids":["senseval2.d000.s000.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"peculiarity%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":482,"cached_input_tokens":0,"output_tokens":231,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003274,"input_uncached_usd":0.0009639999999999999,"input_cached_usd":0.0,"output_usd":0.00231,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.9883900829590857}
{"item_id":"senseval2.d000.s000.t007","gold_sense_keys":["unintelligible%5:00:00:incomprehensible:00"],"candidates":[{"index":1,"sense_key":"unintelligible%3:00:00::","synset_id":"unintelligible.a.01"},{"index":2,"sense_key":"unintelligible%5:00:00:incomprehensible:00","synset_id":"opaque.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"unintelligible%5:00:00:incomprehensible:00","call_ids":["senseval2.d000.s000.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"unintelligible%5:00:00:incomprehensible:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":407,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001474,"input_uncached_usd":0.0008139999999999999,"input_cached_usd":0.0,"output_usd":0.0006600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.446021917043254}
{"item_id":"senseval2.d000.s000.t008","gold_sense_keys":["rest%1:24:00::"],"candidates":[{"index":1,"sense_key":"rest%1:24:00::","synset_id":"remainder.n.01"},{"index":2,"sense_key":"rest%1:04:00::","synset_id":"rest.n.02"},{"index":3,"sense_key":"rest%1:28:00::","synset_id":"respite.n.04"},{"index":4,"sense_key":"rest%1:26:00::","synset_id":"rest.n.04"},{"index":5,"sense_key":"rest%1:26:01::","synset_id":"rest.n.05"},{"index":6,"sense_key":"rest%1:06:00::","synset_id":"rest.n.06"},{"index":7,"sense_key":"rest%1:10:00::","synset_id":"rest.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"rest%1:24:00::","call_ids":["senseval2.d000.s000.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"rest%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":740,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0016,"input_uncached_usd":0.00148,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.82636391709093}
{"item_id":"senseval2.d000.s000.t009","gold_sense_keys":["world%1:14:02::"],"candidates":[{"index":1,"sense_key":"world%1:17:01::","synset_id":"universe.n.01"},{"index":2,"sense_key":"world%1:14:01::","synset_id":"world.n.02"},{"index":3,"sense_key":"world%1:09:00::","synset_id":"world.n.03"},{"index":4,"sense_key":"world%1:17:00::","synset_id":"earth.n.01"},{"index":5,"sense_key":"world%1:14:02::","synset_id":"populace.n.01"},{"index":6,"sense_key":"world%1:17:02::","synset_id":"world.n.06"},{"index":7,"sense_key":"world%1:09:01::","synset_id":"worldly_concern.n.01"},{"index":8,"sense_key":"world%1:05:00::","synset_id":"world.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"world%1:05:00::","call_ids":["senseval2.d000.s000.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"world%1:05:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":846,"cached_input_tokens":0,"output_tokens":116,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002852,"input_uncached_usd":0.001692,"input_cached_usd":0.0,"output_usd":0.00116,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.349225000012666}
{"item_id":"senseval2.d000.s003.t000","gold_sense_keys":["scene%1:09:00::"],"candidates":[{"index":1,"sense_key":"scene%1:15:00::","synset_id":"scene.n.01"},{"index":2,"sense_key":"scene%1:11:00::","synset_id":"scene.n.02"},{"index":3,"sense_key":"scene%1:09:00::","synset_id":"view.n.02"},{"index":4,"sense_key":"scene%1:10:01::","synset_id":"scene.n.04"},{"index":5,"sense_key":"scene%1:26:00::","synset_id":"picture.n.04"},{"index":6,"sense_key":"scene%1:10:00::","synset_id":"scene.n.06"},{"index":7,"sense_key":"scene%1:26:01::","synset_id":"fit.n.01"},{"index":8,"sense_key":"scene%1:06:01::","synset_id":"scene.n.08"},{"index":9,"sense_key":"scene%1:15:01::","synset_id":"setting.n.01"},{"index":10,"sense_key":"scene%1:06:00::","synset_id":"scenery.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"scene%1:09:00::","call_ids":["senseval2.d000.s003.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"scene%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1009,"cached_input_tokens":0,"output_tokens":118,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003198,"input_uncached_usd":0.002018,"input_cached_usd":0.0,"output_usd":0.00118,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.586235166992992}
{"item_id":"senseval2.d000.s003.t001","gold_sense_keys":["evoke%2:32:01::"],"candidates":[{"index":1,"sense_key":"evoke%2:37:00::","synset_id":"arouse.v.01"},{"index":2,"sense_key":"evoke%2:36:00::","synset_id":"provoke.v.02"},{"index":3,"sense_key":"evoke%2:36:01::","synset_id":"educe.v.01"},{"index":4,"sense_key":"evoke%2:36:02::","synset_id":"raise.v.07"},{"index":5,"sense_key":"evoke%2:32:01::","synset_id":"suggest.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"evoke%2:32:01::","call_ids":["senseval2.d000.s003.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"evoke%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":787,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002164,"input_uncached_usd":0.0015739999999999999,"input_cached_usd":0.0,"output_usd":0.00059,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.5631159170297906}
{"item_id":"senseval2.d000.s003.t004","gold_sense_keys":["lovely%5:00:00:beautiful:00"],"candidates":[{"index":1,"sense_key":"lovely%5:00:00:beautiful:00","synset_id":"lovely.s.01"},{"index":2,"sense_key":"lovely%5:00:00:lovable:00","synset_id":"adorable.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"lovely%5:00:00:beautiful:00","call_ids":["senseval2.d000.s003.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"lovely%5:00:00:beautiful:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":501,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001122,"input_uncached_usd":0.0010019999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6868559170980006}
{"item_id":"senseval2.d000.s003.t005","gold_sense_keys":["ancient%5:00:00:old:02"],"candidates":[{"index":1,"sense_key":"ancient%5:00:00:past:00","synset_id":"ancient.s.01"},{"index":2,"sense_key":"ancient%5:00:00:old:02","synset_id":"ancient.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"ancient%5:00:00:old:02","call_ids":["senseval2.d000.s003.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"ancient%5:00:00:old:02","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":527,"cached_input_tokens":0,"output_tokens":69,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001744,"input_uncached_usd":0.001054,"input_cached_usd":0.0,"output_usd":0.0006900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.3333771670004353}
{"item_id":"senseval2.d000.s003.t006","gold_sense_keys":["stone%1:06:00::"],"candidates":[{"index":1,"sense_key":"stone%1:17:00::","synset_id":"rock.n.01"},{"index":2,"sense_key":"stone%1:06:00::","synset_id":"stone.n.02"},{"index":3,"sense_key":"stone%1:27:00::","synset_id":"rock.n.02"},{"index":4,"sense_key":"stone%1:27:01::","synset_id":"gem.n.02"},{"index":5,"sense_key":"stone%1:23:00::","synset_id":"stone.n.05"},{"index":6,"sense_key":"stone%1:20:00::","synset_id":"stone.n.06"},{"index":7,"sense_key":"stone%1:18:05::","synset_id":"stone.n.07"},{"index":8,"sense_key":"stone%1:18:04::","synset_id":"stone.n.08"},{"index":9,"sense_key":"stone%1:18:03::","synset_id":"stone.n.09"},{"index":10,"sense_key":"stone%1:18:02::","synset_id":"stone.n.10"},{"index":11,"sense_key":"stone%1:18:01::","synset_id":"stone.n.11"},{"index":12,"sense_key":"stone%1:18:00::","synset_id":"stone.n.12"},{"index":13,"sense_key":"stone%1:07:00::","synset_id":"stone.n.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"stone%1:27:00::","call_ids":["senseval2.d000.s003.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"stone%1:27:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1322,"cached_input_tokens":0,"output_tokens":122,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0038640000000000002,"input_uncached_usd":0.002644,"input_cached_usd":0.0,"output_usd":0.0012200000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.169972207979299}
{"item_id":"senseval2.d000.s003.t007","gold_sense_keys":["church%1:06:00::"],"candidates":[{"index":1,"sense_key":"church%1:14:00::","synset_id":"church.n.01"},{"index":2,"sense_key":"church%1:06:00::","synset_id":"church.n.02"},{"index":3,"sense_key":"church%1:04:00::","synset_id":"church_service.n.01"},{"index":4,"sense_key":"church%1:14:01::","synset_id":"church.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"church%1:06:00::","call_ids":["senseval2.d000.s003.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"church%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":606,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001332,"input_uncached_usd":0.001212,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9233820419758558}
{"item_id":"senseval2.d000.s003.t008","gold_sense_keys":["stand%2:42:03::"],"candidates":[{"index":1,"sense_key":"stand%2:35:00::","synset_id":"stand.v.01"},{"index":2,"sense_key":"stand%2:42:04::","synset_id":"stand.v.02"},{"index":3,"sense_key":"stand%2:42:03::","synset_id":"stand.v.03"},{"index":4,"sense_key":"stand%2:42:00::","synset_id":"stand.v.04"},{"index":5,"sense_key":"stand%2:31:00::","synset_id":"digest.v.03"},{"index":6,"sense_key":"stand%2:31:01::","synset_id":"stand.v.06"},{"index":7,"sense_key":"stand%2:42:06::","synset_id":"stand.v.07"},{"index":8,"sense_key":"stand%2:42:02::","synset_id":"stand.v.08"},{"index":9,"sense_key":"stand%2:42:01::","synset_id":"stand.v.09"},{"index":10,"sense_key":"stand%2:35:01::","synset_id":"stand.v.10"},{"index":11,"sense_key":"stand%2:33:00::","synset_id":"resist.v.04"},{"index":12,"sense_key":"stand%2:35:10::","synset_id":"stand.v.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"stand%2:42:03::","call_ids":["senseval2.d000.s003.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"stand%2:42:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1053,"cached_input_tokens":0,"output_tokens":74,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002846,"input_uncached_usd":0.002106,"input_cached_usd":0.0,"output_usd":0.0007400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.7065390419447795}
{"item_id":"senseval2.d000.s003.t009","gold_sense_keys":["field%1:15:00::"],"candidates":[{"index":1,"sense_key":"field%1:15:00::","synset_id":"field.n.01"},{"index":2,"sense_key":"field%1:15:04::","synset_id":"battlefield.n.01"},{"index":3,"sense_key":"field%1:15:01::","synset_id":"field.n.03"},{"index":4,"sense_key":"field%1:09:00::","synset_id":"discipline.n.01"},{"index":5,"sense_key":"field%1:19:00::","synset_id":"field.n.05"},{"index":6,"sense_key":"field%1:04:00::","synset_id":"field.n.06"},{"index":7,"sense_key":"field%1:26:00::","synset_id":"sphere.n.01"},{"index":8,"sense_key":"field%1:15:02::","synset_id":"playing_field.n.02"},{"index":9,"sense_key":"field%1:17:00::","synset_id":"plain.n.01"},{"index":10,"sense_key":"field%1:14:00::","synset_id":"field.n.10"},{"index":11,"sense_key":"field%1:15:03::","synset_id":"field.n.11"},{"index":12,"sense_key":"field%1:14:01::","synset_id":"field.n.12"},{"index":13,"sense_key":"field%1:14:02::","synset_id":"field.n.13"},{"index":14,"sense_key":"field%1:15:05::","synset_id":"field.n.14"},{"index":15,"sense_key":"field%1:14:03::","synset_id":"field.n.15"},{"index":16,"sense_key":"field%1:09:01::","synset_id":"field.n.16"},{"index":17,"sense_key":"field%1:06:00::","synset_id":"airfield.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"field%1:15:00::","call_ids":["senseval2.d000.s003.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"field%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1478,"cached_input_tokens":0,"output_tokens":135,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004306,"input_uncached_usd":0.002956,"input_cached_usd":0.0,"output_usd":0.00135,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.7533967919880524}
{"item_id":"senseval2.d000.s003.t010","gold_sense_keys":["sound%1:07:00::"],"candidates":[{"index":1,"sense_key":"sound%1:07:00::","synset_id":"sound.n.01"},{"index":2,"sense_key":"sound%1:09:00::","synset_id":"sound.n.02"},{"index":3,"sense_key":"sound%1:19:00::","synset_id":"sound.n.03"},{"index":4,"sense_key":"sound%1:11:00::","synset_id":"sound.n.04"},{"index":5,"sense_key":"sound%1:10:00::","synset_id":"audio.n.01"},{"index":6,"sense_key":"sound%1:10:01::","synset_id":"phone.n.02"},{"index":7,"sense_key":"sound%1:17:01::","synset_id":"strait.n.01"},{"index":8,"sense_key":"sound%1:17:00::","synset_id":"sound.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"sound%1:07:00::","call_ids":["senseval2.d000.s003.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"sound%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":868,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002376,"input_uncached_usd":0.001736,"input_cached_usd":0.0,"output_usd":0.00064,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.6290201670490205}
{"item_id":"senseval2.d000.s003.t011","gold_sense_keys":["bell%1:06:00::"],"candidates":[{"index":1,"sense_key":"bell%1:06:00::","synset_id":"bell.n.01"},{"index":2,"sense_key":"bell%1:06:02::","synset_id":"doorbell.n.01"},{"index":3,"sense_key":"bell%1:11:00::","synset_id":"bell.n.03"},{"index":4,"sense_key":"bell%1:28:00::","synset_id":"bell.n.04"},{"index":5,"sense_key":"bell%1:25:00::","synset_id":"bell.n.05"},{"index":6,"sense_key":"bell%1:18:02::","synset_id":"bell.n.06"},{"index":7,"sense_key":"bell%1:18:01::","synset_id":"bell.n.07"},{"index":8,"sense_key":"bell%1:18:00::","synset_id":"bell.n.08"},{"index":9,"sense_key":"bell%1:06:04::","synset_id":"chime.n.01"},{"index":10,"sense_key":"bell%1:06:03::","synset_id":"bell.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bell%1:06:00::","call_ids":["senseval2.d000.s003.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bell%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1046,"cached_input_tokens":0,"output_tokens":122,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0033120000000000003,"input_uncached_usd":0.002092,"input_cached_usd":0.0,"output_usd":0.0012200000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.6695090829161927}
{"item_id":"senseval2.d000.s003.t012","gold_sense_keys":["cascade%2:38:00::"],"candidates":[{"index":1,"sense_key":"cascade%2:38:00::","synset_id":"cascade.v.01"},{"index":2,"sense_key":"cascade%2:35:00::","synset_id":"cascade.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cascade%2:38:00::","call_ids":["senseval2.d000.s003.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cascade%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":503,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001126,"input_uncached_usd":0.001006,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.868446166976355}
{"item_id":"senseval2.d000.s003.t013","gold_sense_keys":["tower%1:06:00::"],"candidates":[{"index":1,"sense_key":"tower%1:06:00::","synset_id":"tower.n.01"},{"index":2,"sense_key":"tower%1:25:00::","synset_id":"column.n.04"},{"index":3,"sense_key":"tower%1:06:01::","synset_id":"tugboat.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"tower%1:06:00::","call_ids":["senseval2.d000.s003.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"tower%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":581,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0012820000000000002,"input_uncached_usd":0.001162,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9306671669473872}
{"item_id":"senseval2.d000.s003.t014","gold_sense_keys":["call%2:32:05::"],"candidates":[{"index":1,"sense_key":"call%2:32:02::","synset_id":"name.v.01"},{"index":2,"sense_key":"call%2:32:00::","synset_id":"call.v.02"},{"index":3,"sense_key":"call%2:32:01::","synset_id":"call.v.03"},{"index":4,"sense_key":"call%2:32:09::","synset_id":"shout.v.02"},{"index":5,"sense_key":"call%2:32:05::","synset_id":"call.v.05"},{"index":6,"sense_key":"call%2:41:00::","synset_id":"visit.v.03"},{"index":7,"sense_key":"call%2:41:04::","synset_id":"call.v.07"},{"index":8,"sense_key":"call%2:32:13::","synset_id":"call.v.08"},{"index":9,"sense_key":"call%2:32:04::","synset_id":"call.v.09"},{"index":10,"sense_key":"call%2:32:03::","synset_id":"call.v.10"},{"index":11,"sense_key":"call%2:42:09::","synset_id":"call.v.11"},{"index":12,"sense_key":"call%2:41:14::","synset_id":"address.v.06"},{"index":13,"sense_key":"call%2:41:12::","synset_id":"call.v.13"},{"index":14,"sense_key":"call%2:40:09::","synset_id":"call.v.14"},{"index":15,"sense_key":"call%2:40:05::","synset_id":"bid.v.04"},{"index":16,"sense_key":"call%2:38:00::","synset_id":"call.v.16"},{"index":17,"sense_key":"call%2:32:06::","synset_id":"call.v.17"},{"index":18,"sense_key":"call%2:32:10::","synset_id":"predict.v.01"},{"index":19,"sense_key":"call%2:32:12::","synset_id":"call.v.19"},{"index":20,"sense_key":"call%2:32:07::","synset_id":"call.v.20"},{"index":21,"sense_key":"call%2:32:08::","synset_id":"call.v.21"},{"index":22,"sense_key":"call%2:32:11::","synset_id":"call.v.22"},{"index":23,"sense_key":"call%2:32:15::","synset_id":"call.v.23"},{"index":24,"sense_key":"call%2:32:14::","synset_id":"call.v.24"},{"index":25,"sense_key":"call%2:31:13::","synset_id":"call.v.25"},{"index":26,"sense_key":"call%2:31:00::","synset_id":"call.v.26"},{"index":27,"sense_key":"call%2:31:05::","synset_id":"call.v.27"},{"index":28,"sense_key":"call%2:29:05::","synset_id":"call.v.28"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"call%2:32:05::","call_ids":["senseval2.d000.s003.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"call%2:32:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2022,"cached_input_tokens":0,"output_tokens":96,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.005004,"input_uncached_usd":0.004044,"input_cached_usd":0.0,"output_usd":0.0009600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.352641333010979}
{"item_id":"senseval2.d000.s003.t015","gold_sense_keys":["faithful%1:14:00::"],"candidates":[{"index":1,"sense_key":"faithful%1:14:01::","synset_id":"faithful.n.01"},{"index":2,"sense_key":"faithful%1:14:00::","synset_id":"congregation.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"faithful%1:14:00::","call_ids":["senseval2.d000.s003.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"faithful%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":492,"cached_input_tokens":0,"output_tokens":71,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0016939999999999998,"input_uncached_usd":0.0009839999999999998,"input_cached_usd":0.0,"output_usd":0.00071,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.7079796249745414}
{"item_id":"senseval2.d000.s003.t016","gold_sense_keys":["evensong%1:10:00::"],"candidates":[{"index":1,"sense_key":"evensong%1:28:00::","synset_id":"vespers.n.01"},{"index":2,"sense_key":"evensong%1:10:00::","synset_id":"evening_prayer.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"evensong%1:10:00::","call_ids":["senseval2.d000.s003.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"evensong%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":537,"cached_input_tokens":0,"output_tokens":81,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0018839999999999998,"input_uncached_usd":0.0010739999999999999,"input_cached_usd":0.0,"output_usd":0.0008100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2781367079587653}
{"item_id":"senseval2.d000.s004.t001","gold_sense_keys":["stop%2:38:00::"],"candidates":[{"index":1,"sense_key":"stop%2:38:00::","synset_id":"stop.v.01"},{"index":2,"sense_key":"stop%2:42:00::","synset_id":"discontinue.v.01"},{"index":3,"sense_key":"stop%2:41:00::","synset_id":"stop.v.03"},{"index":4,"sense_key":"stop%2:38:02::","synset_id":"stop.v.04"},{"index":5,"sense_key":"stop%2:38:01::","synset_id":"stop.v.05"},{"index":6,"sense_key":"stop%2:30:05::","synset_id":"break.v.10"},{"index":7,"sense_key":"stop%2:33:00::","synset_id":"check.v.18"},{"index":8,"sense_key":"stop%2:35:00::","synset_id":"intercept.v.01"},{"index":9,"sense_key":"stop%2:42:13::","synset_id":"end.v.01"},{"index":10,"sense_key":"stop%2:35:12::","synset_id":"barricade.v.01"},{"index":11,"sense_key":"stop%2:30:12::","synset_id":"hold_on.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"stop%2:38:00::","call_ids":["senseval2.d000.s004.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"stop%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1200,"cached_input_tokens":0,"output_tokens":115,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00355,"input_uncached_usd":0.0024,"input_cached_usd":0.0,"output_usd":0.0011500000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.735737916897051}
{"item_id":"senseval2.d000.s004.t003","gold_sense_keys":["church%1:06:00::"],"candidates":[{"index":1,"sense_key":"church%1:14:00::","synset_id":"church.n.01"},{"index":2,"sense_key":"church%1:06:00::","synset_id":"church.n.02"},{"index":3,"sense_key":"church%1:04:00::","synset_id":"church_service.n.01"},{"index":4,"sense_key":"church%1:14:01::","synset_id":"church.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"church%1:06:00::","call_ids":["senseval2.d000.s004.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"church%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":645,"cached_input_tokens":0,"output_tokens":41,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0017,"input_uncached_usd":0.00129,"input_cached_usd":0.0,"output_usd":0.00041000000000000005,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.7033065839204937}
{"item_id":"senseval2.d000.s004.t004","gold_sense_keys":["door%1:06:00::","door%1:06:01::"],"candidates":[{"index":1,"sense_key":"door%1:06:00::","synset_id":"door.n.01"},{"index":2,"sense_key":"door%1:06:01::","synset_id":"doorway.n.01"},{"index":3,"sense_key":"door%1:07:00::","synset_id":"door.n.03"},{"index":4,"sense_key":"door%1:06:04::","synset_id":"door.n.04"},{"index":5,"sense_key":"door%1:06:03::","synset_id":"door.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"door%1:06:01::","call_ids":["senseval2.d000.s004.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"door%1:06:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":757,"cached_input_tokens":0,"output_tokens":113,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002644,"input_uncached_usd":0.001514,"input_cached_usd":0.0,"output_usd":0.0011300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.7310200829524547}
{"item_id":"senseval2.d000.s004.t005","gold_sense_keys":["member%1:18:00::"],"candidates":[{"index":1,"sense_key":"member%1:18:00::","synset_id":"member.n.01"},{"index":2,"sense_key":"member%1:24:00::","synset_id":"member.n.02"},{"index":3,"sense_key":"member%1:08:00::","synset_id":"extremity.n.01"},{"index":4,"sense_key":"member%1:14:00::","synset_id":"member.n.04"},{"index":5,"sense_key":"member%1:08:01::","synset_id":"penis.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"member%1:18:00::","call_ids":["senseval2.d000.s004.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"member%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":767,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001654,"input_uncached_usd":0.001534,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8794637919636443}
{"item_id":"senseval2.d000.s004.t006","gold_sense_keys":["here%4:02:00::"],"candidates":[{"index":1,"sense_key":"here%4:02:00::","synset_id":"here.r.01"},{"index":2,"sense_key":"here%4:02:02::","synset_id":"here.r.02"},{"index":3,"sense_key":"here%4:02:01::","synset_id":"here.r.03"},{"index":4,"sense_key":"here%4:02:04::","synset_id":"here.r.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"here%4:02:00::","call_ids":["senseval2.d000.s004.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"here%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":645,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00141,"input_uncached_usd":0.00129,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8332069589523599}
{"item_id":"senseval2.d000.s004.t007","gold_sense_keys":["always%4:02:00::"],"candidates":[{"index":1,"sense_key":"always%4:02:00::","synset_id":"always.r.01"},{"index":2,"sense_key":"always%4:02:04::","synset_id":"constantly.r.01"},{"index":3,"sense_key":"always%4:02:03::","synset_id":"constantly.r.02"},{"index":4,"sense_key":"always%4:02:02::","synset_id":"always.r.04"},{"index":5,"sense_key":"always%4:02:01::","synset_id":"always.r.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"always%4:02:00::","call_ids":["senseval2.d000.s004.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"always%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":724,"cached_input_tokens":0,"output_tokens":129,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002738,"input_uncached_usd":0.0014479999999999999,"input_cached_usd":0.0,"output_usd":0.0012900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.8421136249089614}
{"item_id":"senseval2.d000.s005.t000","gold_sense_keys":["tower%1:06:00::"],"candidates":[{"index":1,"sense_key":"tower%1:06:00::","synset_id":"tower.n.01"},{"index":2,"sense_key":"tower%1:25:00::","synset_id":"column.n.04"},{"index":3,"sense_key":"tower%1:06:01::","synset_id":"tugboat.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"tower%1:06:00::","call_ids":["senseval2.d000.s005.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"tower%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":671,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001462,"input_uncached_usd":0.0013419999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8925338329281658}
{"item_id":"senseval2.d000.s005.t001","gold_sense_keys":["man%1:18:00::"],"candidates":[{"index":1,"sense_key":"man%1:18:00::","synset_id":"man.n.01"},{"index":2,"sense_key":"man%1:18:03::","synset_id":"serviceman.n.01"},{"index":3,"sense_key":"man%1:18:04::","synset_id":"man.n.03"},{"index":4,"sense_key":"man%1:05:01::","synset_id":"homo.n.02"},{"index":5,"sense_key":"man%1:18:08::","synset_id":"man.n.05"},{"index":6,"sense_key":"man%1:18:06::","synset_id":"man.n.06"},{"index":7,"sense_key":"man%1:18:05::","synset_id":"valet.n.01"},{"index":8,"sense_key":"man%1:18:07::","synset_id":"man.n.08"},{"index":9,"sense_key":"man%1:15:00::","synset_id":"man.n.09"},{"index":10,"sense_key":"man%1:06:00::","synset_id":"man.n.10"},{"index":11,"sense_key":"man%1:05:00::","synset_id":"world.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"man%1:18:00::","call_ids":["senseval2.d000.s005.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"man%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1229,"cached_input_tokens":0,"output_tokens":38,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002838,"input_uncached_usd":0.0024579999999999997,"input_cached_usd":0.0,"output_usd":0.00038,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2155202500289306}
{"item_id":"senseval2.d000.s005.t002","gold_sense_keys":["woman%1:18:00::"],"candidates":[{"index":1,"sense_key":"woman%1:18:00::","synset_id":"woman.n.01"},{"index":2,"sense_key":"woman%1:18:01::","synset_id":"woman.n.02"},{"index":3,"sense_key":"woman%1:18:02::","synset_id":"charwoman.n.01"},{"index":4,"sense_key":"woman%1:14:00::","synset_id":"womanhood.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"woman%1:18:00::","call_ids":["senseval2.d000.s005.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"woman%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":771,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001662,"input_uncached_usd":0.001542,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.304130124975927}
{"item_id":"senseval2.d000.s005.t003","gold_sense_keys":["pull%2:35:04::"],"candidates":[{"index":1,"sense_key":"pull%2:35:00::","synset_id":"pull.v.01"},{"index":2,"sense_key":"pull%2:35:02::","synset_id":"attract.v.01"},{"index":3,"sense_key":"pull%2:38:01::","synset_id":"pull.v.03"},{"index":4,"sense_key":"pull%2:35:04::","synset_id":"pull.v.04"},{"index":5,"sense_key":"pull%2:41:00::","synset_id":"perpetrate.v.01"},{"index":6,"sense_key":"pull%2:38:03::","synset_id":"draw.v.05"},{"index":7,"sense_key":"pull%2:38:00::","synset_id":"pull.v.07"},{"index":8,"sense_key":"pull%2:29:00::","synset_id":"pull.v.08"},{"index":9,"sense_key":"pull%2:38:13::","synset_id":"pull.v.09"},{"index":10,"sense_key":"pull%2:38:02::","synset_id":"pull.v.10"},{"index":11,"sense_key":"pull%2:38:04::","synset_id":"pull.v.11"},{"index":12,"sense_key":"pull%2:35:01::","synset_id":"rend.v.01"},{"index":13,"sense_key":"pull%2:35:03::","synset_id":"pull.v.13"},{"index":14,"sense_key":"pull%2:35:05::","synset_id":"pluck.v.05"},{"index":15,"sense_key":"pull%2:35:10::","synset_id":"extract.v.01"},{"index":16,"sense_key":"pull%2:33:13::","synset_id":"pull.v.16"},{"index":17,"sense_key":"pull%2:30:11::","synset_id":"pull.v.17"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"pull%2:35:04::","call_ids":["senseval2.d000.s005.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"pull%2:35:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1578,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003786,"input_uncached_usd":0.003156,"input_cached_usd":0.0,"output_usd":0.00063,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.3478983329841867}
{"item_id":"senseval2.d000.s005.t005","gold_sense_keys":["rope%1:06:00::"],"candidates":[{"index":1,"sense_key":"rope%1:06:00::","synset_id":"rope.n.01"},{"index":2,"sense_key":"rope%1:06:01::","synset_id":"r-2.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"rope%1:06:00::","call_ids":["senseval2.d000.s005.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"rope%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":582,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001284,"input_uncached_usd":0.0011639999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8787046250654384}
{"item_id":"senseval2.d000.s005.t007","gold_sense_keys":["same%3:00:02::"],"candidates":[{"index":1,"sense_key":"same%3:00:02::","synset_id":"same.a.01"},{"index":2,"sense_key":"same%3:00:00::","synset_id":"same.a.02"},{"index":3,"sense_key":"same%3:00:04::","synset_id":"like.a.02"},{"index":4,"sense_key":"same%5:00:00:unchanged:00","synset_id":"same.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"same%3:00:02::","call_ids":["senseval2.d000.s005.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"same%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":710,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00201,"input_uncached_usd":0.00142,"input_cached_usd":0.0,"output_usd":0.00059,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.779991791001521}
{"item_id":"senseval2.d000.s005.t008","gold_sense_keys":["bell%1:06:00::"],"candidates":[{"index":1,"sense_key":"bell%1:06:00::","synset_id":"bell.n.01"},{"index":2,"sense_key":"bell%1:06:02::","synset_id":"doorbell.n.01"},{"index":3,"sense_key":"bell%1:11:00::","synset_id":"bell.n.03"},{"index":4,"sense_key":"bell%1:28:00::","synset_id":"bell.n.04"},{"index":5,"sense_key":"bell%1:25:00::","synset_id":"bell.n.05"},{"index":6,"sense_key":"bell%1:18:02::","synset_id":"bell.n.06"},{"index":7,"sense_key":"bell%1:18:01::","synset_id":"bell.n.07"},{"index":8,"sense_key":"bell%1:18:00::","synset_id":"bell.n.08"},{"index":9,"sense_key":"bell%1:06:04::","synset_id":"chime.n.01"},{"index":10,"sense_key":"bell%1:06:03::","synset_id":"bell.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bell%1:06:00::","call_ids":["senseval2.d000.s005.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bell%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1136,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0023919999999999996,"input_uncached_usd":0.0022719999999999997,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.5546804999466985}
{"item_id":"senseval2.d000.s005.t009","gold_sense_keys":["first%4:02:02::"],"candidates":[{"index":1,"sense_key":"first%4:02:00::","synset_id":"first.r.01"},{"index":2,"sense_key":"first%4:02:02::","synset_id":"first.r.02"},{"index":3,"sense_key":"first%4:02:03::","synset_id":"first.r.03"},{"index":4,"sense_key":"first%4:02:01::","synset_id":"foremost.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"first%4:02:02::","call_ids":["senseval2.d000.s005.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"first%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":692,"cached_input_tokens":0,"output_tokens":45,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0018340000000000001,"input_uncached_usd":0.001384,"input_cached_usd":0.0,"output_usd":0.00045000000000000004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2504622919950634}
{"item_id":"senseval2.d000.s005.t010","gold_sense_keys":["sound%2:39:00::","sound%2:39:03::"],"candidates":[{"index":1,"sense_key":"sound%2:39:06::","synset_id":"sound.v.01"},{"index":2,"sense_key":"sound%2:39:00::","synset_id":"sound.v.02"},{"index":3,"sense_key":"sound%2:39:03::","synset_id":"sound.v.03"},{"index":4,"sense_key":"sound%2:39:02::","synset_id":"sound.v.04"},{"index":5,"sense_key":"sound%2:32:00::","synset_id":"voice.v.02"},{"index":6,"sense_key":"sound%2:39:01::","synset_id":"sound.v.06"},{"index":7,"sense_key":"sound%2:30:00::","synset_id":"fathom.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"sound%2:39:00::","call_ids":["senseval2.d000.s005.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"sound%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":822,"cached_input_tokens":0,"output_tokens":214,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003784,"input_uncached_usd":0.0016439999999999998,"input_cached_usd":0.0,"output_usd":0.00214,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.372270749998279}
{"item_id":"senseval2.d000.s005.t011","gold_sense_keys":["here%4:02:00::"],"candidates":[{"index":1,"sense_key":"here%4:02:00::","synset_id":"here.r.01"},{"index":2,"sense_key":"here%4:02:02::","synset_id":"here.r.02"},{"index":3,"sense_key":"here%4:02:01::","synset_id":"here.r.03"},{"index":4,"sense_key":"here%4:02:04::","synset_id":"here.r.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"here%4:02:00::","call_ids":["senseval2.d000.s005.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"here%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":696,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001512,"input_uncached_usd":0.001392,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8218746669590473}
{"item_id":"senseval2.d000.s006.t001","gold_sense_keys":["discordant%3:00:00::"],"candidates":[{"index":1,"sense_key":"discordant%3:00:00::","synset_id":"discordant.a.01"},{"index":2,"sense_key":"discordant%5:00:00:inharmonious:00","synset_id":"discordant.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"discordant%3:00:00::","call_ids":["senseval2.d000.s006.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"discordant%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":614,"cached_input_tokens":0,"output_tokens":205,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003278,"input_uncached_usd":0.001228,"input_cached_usd":0.0,"output_usd":0.00205,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.7458845840301365}
{"item_id":"senseval2.d000.s006.t003","gold_sense_keys":["note%1:07:00::"],"candidates":[{"index":1,"sense_key":"note%1:10:00::","synset_id":"note.n.01"},{"index":2,"sense_key":"note%1:10:03::","synset_id":"note.n.02"},{"index":3,"sense_key":"note%1:10:04::","synset_id":"note.n.03"},{"index":4,"sense_key":"note%1:10:05::","synset_id":"note.n.04"},{"index":5,"sense_key":"note%1:07:00::","synset_id":"note.n.05"},{"index":6,"sense_key":"note%1:21:01::","synset_id":"bill.n.03"},{"index":7,"sense_key":"note%1:10:02::","synset_id":"note.n.07"},{"index":8,"sense_key":"note%1:26:00::","synset_id":"eminence.n.01"},{"index":9,"sense_key":"note%1:21:00::","synset_id":"note.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"note%1:07:00::","call_ids":["senseval2.d000.s006.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"note%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1083,"cached_input_tokens":0,"output_tokens":114,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003306,"input_uncached_usd":0.002166,"input_cached_usd":0.0,"output_usd":0.0011400000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.006028207950294}
{"item_id":"senseval2.d000.s006.t005","gold_sense_keys":["enjoy%2:34:00::","enjoy%2:37:00::","enjoy%2:37:01::"],"candidates":[{"index":1,"sense_key":"enjoy%2:37:00::","synset_id":"enjoy.v.01"},{"index":2,"sense_key":"enjoy%2:34:01::","synset_id":"enjoy.v.02"},{"index":3,"sense_key":"enjoy%2:37:01::","synset_id":"love.v.02"},{"index":4,"sense_key":"enjoy%2:39:00::","synset_id":"enjoy.v.04"},{"index":5,"sense_key":"enjoy%2:34:00::","synset_id":"delight.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"enjoy%2:37:00::","call_ids":["senseval2.d000.s006.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"enjoy%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":778,"cached_input_tokens":0,"output_tokens":185,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003406,"input_uncached_usd":0.0015559999999999999,"input_cached_usd":0.0,"output_usd":0.00185,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.0815263339318335}
{"item_id":"senseval2.d000.s006.t007","gold_sense_keys":["bell%1:06:00::"],"candidates":[{"index":1,"sense_key":"bell%1:06:00::","synset_id":"bell.n.01"},{"index":2,"sense_key":"bell%1:06:02::","synset_id":"doorbell.n.01"},{"index":3,"sense_key":"bell%1:11:00::","synset_id":"bell.n.03"},{"index":4,"sense_key":"bell%1:28:00::","synset_id":"bell.n.04"},{"index":5,"sense_key":"bell%1:25:00::","synset_id":"bell.n.05"},{"index":6,"sense_key":"bell%1:18:02::","synset_id":"bell.n.06"},{"index":7,"sense_key":"bell%1:18:01::","synset_id":"bell.n.07"},{"index":8,"sense_key":"bell%1:18:00::","synset_id":"bell.n.08"},{"index":9,"sense_key":"bell%1:06:04::","synset_id":"chime.n.01"},{"index":10,"sense_key":"bell%1:06:03::","synset_id":"bell.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bell%1:06:00::","call_ids":["senseval2.d000.s006.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bell%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1152,"cached_input_tokens":0,"output_tokens":169,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003994,"input_uncached_usd":0.002304,"input_cached_usd":0.0,"output_usd":0.00169,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.328663625055924}
{"item_id":"senseval2.d000.s006.t008","gold_sense_keys":["cool%3:00:01::"],"candidates":[{"index":1,"sense_key":"cool%3:00:01::","synset_id":"cool.a.01"},{"index":2,"sense_key":"cool%5:00:00:composed:00","synset_id":"cool.s.01"},{"index":3,"sense_key":"cool%3:00:03::","synset_id":"cool.a.03"},{"index":4,"sense_key":"cool%3:00:02::","synset_id":"cool.a.04"},{"index":5,"sense_key":"cool%5:00:00:unqualified:02","synset_id":"cool.s.02"},{"index":6,"sense_key":"cool%5:00:00:fashionable:00","synset_id":"cool.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cool%3:00:01::","call_ids":["senseval2.d000.s006.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cool%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":896,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001912,"input_uncached_usd":0.001792,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.857553125009872}
{"item_id":"senseval2.d000.s006.t010","gold_sense_keys":["evening%1:28:00::"],"candidates":[{"index":1,"sense_key":"evening%1:28:00::","synset_id":"evening.n.01"},{"index":2,"sense_key":"evening%1:28:01::","synset_id":"evening.n.02"},{"index":3,"sense_key":"evening%1:28:02::","synset_id":"evening.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"evening%1:28:00::","call_ids":["senseval2.d000.s006.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"evening%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":691,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002102,"input_uncached_usd":0.001382,"input_cached_usd":0.0,"output_usd":0.00072,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.7436676250072196}
{"item_id":"senseval2.d000.s007.t000","gold_sense_keys":["most%3:00:02::"],"candidates":[{"index":1,"sense_key":"most%3:00:02::","synset_id":"most.a.01"},{"index":2,"sense_key":"most%3:00:01::","synset_id":"most.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"most%3:00:02::","call_ids":["senseval2.d000.s007.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"most%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":675,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002,"input_uncached_usd":0.0013499999999999999,"input_cached_usd":0.0,"output_usd":0.0006500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.3584567080251873}
{"item_id":"senseval2.d000.s007.t001","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":2,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":3,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"other%3:00:00::","call_ids":["senseval2.d000.s007.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":743,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001606,"input_uncached_usd":0.001486,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8867984160315245}
{"item_id":"senseval2.d000.s007.t002","gold_sense_keys":["church%1:06:00::"],"candidates":[{"index":1,"sense_key":"church%1:14:00::","synset_id":"church.n.01"},{"index":2,"sense_key":"church%1:06:00::","synset_id":"church.n.02"},{"index":3,"sense_key":"church%1:04:00::","synset_id":"church_service.n.01"},{"index":4,"sense_key":"church%1:14:01::","synset_id":"church.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"church%1:06:00::","call_ids":["senseval2.d000.s007.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"church%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":717,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0020340000000000002,"input_uncached_usd":0.001434,"input_cached_usd":0.0,"output_usd":0.0006000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2787866659928113}
{"item_id":"senseval2.d000.s007.t004","gold_sense_keys":["set%1:14:00::"],"candidates":[{"index":1,"sense_key":"set%1:14:00::","synset_id":"set.n.01"},{"index":2,"sense_key":"set%1:14:02::","synset_id":"set.n.02"},{"index":3,"sense_key":"set%1:04:00::","synset_id":"set.n.03"},{"index":4,"sense_key":"set%1:06:00::","synset_id":"stage_set.n.01"},{"index":5,"sense_key":"set%1:14:01::","synset_id":"set.n.05"},{"index":6,"sense_key":"set%1:09:01::","synset_id":"bent.n.01"},{"index":7,"sense_key":"set%1:04:01::","synset_id":"set.n.07"},{"index":8,"sense_key":"set%1:28:00::","synset_id":"set.n.08"},{"index":9,"sense_key":"set%1:22:00::","synset_id":"hardening.n.02"},{"index":10,"sense_key":"set%1:18:00::","synset_id":"set.n.10"},{"index":11,"sense_key":"set%1:11:00::","synset_id":"set.n.11"},{"index":12,"sense_key":"set%1:09:00::","synset_id":"set.n.12"},{"index":13,"sense_key":"set%1:06:01::","synset_id":"set.n.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"set%1:14:00::","call_ids":["senseval2.d000.s007.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"set%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1327,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0027739999999999996,"input_uncached_usd":0.0026539999999999997,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6903563749510795}
{"item_id":"senseval2.d000.s007.t005","gold_sense_keys":["bell%1:06:00::"],"candidates":[{"index":1,"sense_key":"bell%1:06:00::","synset_id":"bell.n.01"},{"index":2,"sense_key":"bell%1:06:02::","synset_id":"doorbell.n.01"},{"index":3,"sense_key":"bell%1:11:00::","synset_id":"bell.n.03"},{"index":4,"sense_key":"bell%1:28:00::","synset_id":"bell.n.04"},{"index":5,"sense_key":"bell%1:25:00::","synset_id":"bell.n.05"},{"index":6,"sense_key":"bell%1:18:02::","synset_id":"bell.n.06"},{"index":7,"sense_key":"bell%1:18:01::","synset_id":"bell.n.07"},{"index":8,"sense_key":"bell%1:18:00::","synset_id":"bell.n.08"},{"index":9,"sense_key":"bell%1:06:04::","synset_id":"chime.n.01"},{"index":10,"sense_key":"bell%1:06:03::","synset_id":"bell.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bell%1:06:00::","call_ids":["senseval2.d000.s007.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bell%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1156,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002842,"input_uncached_usd":0.002312,"input_cached_usd":0.0,"output_usd":0.0005300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2038627499714494}
{"item_id":"senseval2.d000.s007.t006","gold_sense_keys":["once%4:02:01::"],"candidates":[{"index":1,"sense_key":"once%4:02:00::","synset_id":"once.r.01"},{"index":2,"sense_key":"once%4:02:02::","synset_id":"once.r.02"},{"index":3,"sense_key":"once%4:02:01::","synset_id":"once.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"once%4:02:01::","call_ids":["senseval2.d000.s007.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"once%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":662,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001444,"input_uncached_usd":0.0013239999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.951766125042923}
{"item_id":"senseval2.d000.s007.t009","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"},{"index":2,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"},{"index":3,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":4,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s007.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":712,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001544,"input_uncached_usd":0.001424,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.323880208074115}
{"item_id":"senseval2.d000.s007.t011","gold_sense_keys":["sunday%1:28:00::"],"candidates":[{"index":1,"sense_key":"sunday%1:28:00::","synset_id":"sunday.n.01"},{"index":2,"sense_key":"sunday%1:18:00::","synset_id":"sunday.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"sunday%1:28:00::","call_ids":["senseval2.d000.s007.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"sunday%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":639,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001398,"input_uncached_usd":0.0012779999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.82866387499962}
{"item_id":"senseval2.d000.s007.t012","gold_sense_keys":["morning%1:28:00::"],"candidates":[{"index":1,"sense_key":"morning%1:28:00::","synset_id":"morning.n.01"},{"index":2,"sense_key":"morning%1:10:00::","synset_id":"good_morning.n.01"},{"index":3,"sense_key":"morning%1:28:01::","synset_id":"dawn.n.01"},{"index":4,"sense_key":"morning%1:11:00::","synset_id":"dawn.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"morning%1:28:00::","call_ids":["senseval2.d000.s007.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"morning%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":746,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001612,"input_uncached_usd":0.0014919999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8538426660234109}
{"item_id":"senseval2.d000.s007.t013","gold_sense_keys":["evening%1:28:00::"],"candidates":[{"index":1,"sense_key":"evening%1:28:00::","synset_id":"evening.n.01"},{"index":2,"sense_key":"evening%1:28:01::","synset_id":"evening.n.02"},{"index":3,"sense_key":"evening%1:28:02::","synset_id":"evening.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"evening%1:28:00::","call_ids":["senseval2.d000.s007.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"evening%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":695,"cached_input_tokens":0,"output_tokens":108,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00247,"input_uncached_usd":0.00139,"input_cached_usd":0.0,"output_usd":0.00108,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.015072208014317}
{"item_id":"senseval2.d000.s007.t014","gold_sense_keys":["service%1:04:01::"],"candidates":[{"index":1,"sense_key":"service%1:04:08::","synset_id":"service.n.01"},{"index":2,"sense_key":"service%1:04:00::","synset_id":"service.n.02"},{"index":3,"sense_key":"service%1:04:01::","synset_id":"service.n.03"},{"index":4,"sense_key":"service%1:14:05::","synset_id":"service.n.04"},{"index":5,"sense_key":"service%1:04:07::","synset_id":"service.n.05"},{"index":6,"sense_key":"service%1:14:00::","synset_id":"military_service.n.01"},{"index":7,"sense_key":"service%1:18:00::","synset_id":"service.n.07"},{"index":8,"sense_key":"service%1:07:00::","synset_id":"avail.n.01"},{"index":9,"sense_key":"service%1:06:00::","synset_id":"service.n.09"},{"index":10,"sense_key":"service%1:04:03::","synset_id":"servicing.n.01"},{"index":11,"sense_key":"service%1:04:09::","synset_id":"service.n.11"},{"index":12,"sense_key":"service%1:04:02::","synset_id":"serve.n.01"},{"index":13,"sense_key":"service%1:04:04::","synset_id":"service.n.13"},{"index":14,"sense_key":"service%1:04:05::","synset_id":"overhaul.n.01"},{"index":15,"sense_key":"service%1:04:06::","synset_id":"service.n.15"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"service%1:04:01::","call_ids":["senseval2.d000.s007.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"service%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1307,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002734,"input_uncached_usd":0.002614,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.29230687499512}
{"item_id":"senseval2.d000.s008.t000","gold_sense_keys":["now%4:02:04::"],"candidates":[{"index":1,"sense_key":"now%4:02:05::","synset_id":"now.r.01"},{"index":2,"sense_key":"now%4:02:00::","synset_id":"nowadays.r.01"},{"index":3,"sense_key":"now%4:02:02::","synset_id":"now.r.03"},{"index":4,"sense_key":"now%4:02:04::","synset_id":"now.r.04"},{"index":5,"sense_key":"now%4:02:01::","synset_id":"immediately.r.01"},{"index":6,"sense_key":"now%4:02:07::","synset_id":"now.r.06"},{"index":7,"sense_key":"now%4:02:06::","synset_id":"now.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"now%4:02:00::","call_ids":["senseval2.d000.s008.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"now%4:02:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":950,"cached_input_tokens":0,"output_tokens":152,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0034200000000000003,"input_uncached_usd":0.0019,"input_cached_usd":0.0,"output_usd":0.00152,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.309577582986094}
{"item_id":"senseval2.d000.s008.t001","gold_sense_keys":["only%4:02:02::"],"candidates":[{"index":1,"sense_key":"only%4:02:02::","synset_id":"merely.r.01"},{"index":2,"sense_key":"only%4:02:01::","synset_id":"entirely.r.02"},{"index":3,"sense_key":"only%4:02:00::","synset_id":"only.r.03"},{"index":4,"sense_key":"only%4:02:04::","synset_id":"only.r.04"},{"index":5,"sense_key":"only%4:02:06::","synset_id":"only.r.05"},{"index":6,"sense_key":"only%4:02:03::","synset_id":"only.r.06"},{"index":7,"sense_key":"only%4:02:05::","synset_id":"only.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"only%4:02:02::","call_ids":["senseval2.d000.s008.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"only%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":876,"cached_input_tokens":0,"output_tokens":111,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002862,"input_uncached_usd":0.0017519999999999999,"input_cached_usd":0.0,"output_usd":0.00111,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.6803248330252245}
{"item_id":"senseval2.d000.s008.t002","gold_sense_keys":["local%3:01:01::"],"candidates":[{"index":1,"sense_key":"local%3:00:01::","synset_id":"local.a.01"},{"index":2,"sense_key":"local%3:01:01::","synset_id":"local.a.02"},{"index":3,"sense_key":"local%3:00:02::","synset_id":"local.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"local%3:01:01::","call_ids":["senseval2.d000.s008.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"local%3:01:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":673,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001466,"input_uncached_usd":0.001346,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.412601333926432}
{"item_id":"senseval2.d000.s008.t003","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"},{"index":2,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"},{"index":3,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":4,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s008.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":713,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001546,"input_uncached_usd":0.001426,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8772225419525057}
{"item_id":"senseval2.d000.s008.t004","gold_sense_keys":["remain%2:42:05::"],"candidates":[{"index":1,"sense_key":"remain%2:30:00::","synset_id":"stay.v.01"},{"index":2,"sense_key":"remain%2:42:03::","synset_id":"stay.v.04"},{"index":3,"sense_key":"remain%2:42:05::","synset_id":"remain.v.03"},{"index":4,"sense_key":"remain%2:42:01::","synset_id":"persist.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"remain%2:42:05::","call_ids":["senseval2.d000.s008.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"remain%2:42:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":820,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00221,"input_uncached_usd":0.00164,"input_cached_usd":0.0,"output_usd":0.0005700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1391137500759214}
{"item_id":"senseval2.d000.s009.t000","gold_sense_keys":["here%4:02:00::"],"candidates":[{"index":1,"sense_key":"here%4:02:00::","synset_id":"here.r.01"},{"index":2,"sense_key":"here%4:02:02::","synset_id":"here.r.02"},{"index":3,"sense_key":"here%4:02:01::","synset_id":"here.r.03"},{"index":4,"sense_key":"here%4:02:04::","synset_id":"here.r.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"here%4:02:00::","call_ids":["senseval2.d000.s009.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"here%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":732,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0015840000000000001,"input_uncached_usd":0.001464,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.328314375015907}
{"item_id":"senseval2.d000.s009.t001","gold_sense_keys":["today%1:28:01::"],"candidates":[{"index":1,"sense_key":"today%1:28:00::","synset_id":"today.n.01"},{"index":2,"sense_key":"today%1:28:01::","synset_id":"today.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"today%1:28:00::","call_ids":["senseval2.d000.s009.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"today%1:28:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":633,"cached_input_tokens":0,"output_tokens":70,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001966,"input_uncached_usd":0.001266,"input_cached_usd":0.0,"output_usd":0.0007000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2236630000406876}
{"item_id":"senseval2.d000.s009.t002","gold_sense_keys":["live%2:42:08::"],"candidates":[{"index":1,"sense_key":"live%2:42:08::","synset_id":"populate.v.01"},{"index":2,"sense_key":"live%2:42:06::","synset_id":"live.v.02"},{"index":3,"sense_key":"live%2:42:04::","synset_id":"survive.v.01"},{"index":4,"sense_key":"live%2:42:01::","synset_id":"exist.v.02"},{"index":5,"sense_key":"live%2:42:00::","synset_id":"be.v.11"},{"index":6,"sense_key":"live%2:31:00::","synset_id":"know.v.05"},{"index":7,"sense_key":"live%2:42:07::","synset_id":"live.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"live%2:42:08::","call_ids":["senseval2.d000.s009.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"live%2:42:08::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1008,"cached_input_tokens":0,"output_tokens":83,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002846,"input_uncached_usd":0.002016,"input_cached_usd":0.0,"output_usd":0.0008300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.072367792017758}
{"item_id":"senseval2.d000.s010.t001","gold_sense_keys":["group%1:03:00::"],"candidates":[{"index":1,"sense_key":"group%1:03:00::","synset_id":"group.n.01"},{"index":2,"sense_key":"group%1:27:00::","synset_id":"group.n.02"},{"index":3,"sense_key":"group%1:09:00::","synset_id":"group.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"group%1:03:00::","call_ids":["senseval2.d000.s010.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"group%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":696,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001512,"input_uncached_usd":0.001392,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8470067920861766}
{"item_id":"senseval2.d000.s010.t002","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"},{"index":2,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"},{"index":3,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":4,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s010.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":738,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001596,"input_uncached_usd":0.0014759999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1232338750269264}
{"item_id":"senseval2.d000.s010.t003","gold_sense_keys":["include%2:42:00::"],"candidates":[{"index":1,"sense_key":"include%2:42:00::","synset_id":"include.v.01"},{"index":2,"sense_key":"include%2:31:00::","synset_id":"include.v.02"},{"index":3,"sense_key":"include%2:30:00::","synset_id":"include.v.03"},{"index":4,"sense_key":"include%2:41:03::","synset_id":"admit.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"include%2:42:00::","call_ids":["senseval2.d000.s010.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"include%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":770,"cached_input_tokens":0,"output_tokens":87,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00241,"input_uncached_usd":0.00154,"input_cached_usd":0.0,"output_usd":0.0008700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1420266670174897}
{"item_id":"senseval2.d000.s010.t006","gold_sense_keys":["training%1:04:00::"],"candidates":[{"index":1,"sense_key":"training%1:04:00::","synset_id":"training.n.01"},{"index":2,"sense_key":"training%1:07:00::","synset_id":"education.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"training%1:04:00::","call_ids":["senseval2.d000.s010.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"training%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":649,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001418,"input_uncached_usd":0.001298,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.159791374928318}
{"item_id":"senseval2.d000.s010.t007","gold_sense_keys":["drive%2:38:00::"],"candidates":[{"index":1,"sense_key":"drive%2:38:01::","synset_id":"drive.v.01"},{"index":2,"sense_key":"drive%2:38:00::","synset_id":"drive.v.02"},{"index":3,"sense_key":"drive%2:38:02::","synset_id":"drive.v.03"},{"index":4,"sense_key":"drive%2:35:00::","synset_id":"force.v.06"},{"index":5,"sense_key":"drive%2:41:02::","synset_id":"drive.v.05"},{"index":6,"sense_key":"drive%2:35:11::","synset_id":"repel.v.01"},{"index":7,"sense_key":"drive%2:36:00::","synset_id":"drive.v.07"},{"index":8,"sense_key":"drive%2:35:07::","synset_id":"drive.v.08"},{"index":9,"sense_key":"drive%2:35:01::","synset_id":"drive.v.09"},{"index":10,"sense_key":"drive%2:41:00::","synset_id":"tug.v.02"},{"index":11,"sense_key":"drive%2:32:00::","synset_id":"drive.v.11"},{"index":12,"sense_key":"drive%2:42:00::","synset_id":"drive.v.12"},{"index":13,"sense_key":"drive%2:41:13::","synset_id":"drive.v.13"},{"index":14,"sense_key":"drive%2:38:09::","synset_id":"drive.v.14"},{"index":15,"sense_key":"drive%2:38:10::","synset_id":"drive.v.15"},{"index":16,"sense_key":"drive%2:38:11::","synset_id":"drive.v.16"},{"index":17,"sense_key":"drive%2:35:03::","synset_id":"drive.v.17"},{"index":18,"sense_key":"drive%2:35:02::","synset_id":"drive.v.18"},{"index":19,"sense_key":"drive%2:35:06::","synset_id":"drive.v.19"},{"index":20,"sense_key":"drive%2:34:07::","synset_id":"drive.v.20"},{"index":21,"sense_key":"drive%2:33:01::","synset_id":"drive.v.21"},{"index":22,"sense_key":"drive%2:33:00::","synset_id":"drive.v.22"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"drive%2:38:00::","call_ids":["senseval2.d000.s010.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"drive%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1736,"cached_input_tokens":0,"output_tokens":123,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004702,"input_uncached_usd":0.003472,"input_cached_usd":0.0,"output_usd":0.0012300000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6554707089671865}
{"item_id":"senseval2.d000.s010.t008","gold_sense_keys":["sunday%1:28:00::"],"candidates":[{"index":1,"sense_key":"sunday%1:28:00::","synset_id":"sunday.n.01"},{"index":2,"sense_key":"sunday%1:18:00::","synset_id":"sunday.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"sunday%1:28:00::","call_ids":["senseval2.d000.s010.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"sunday%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":665,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0014500000000000001,"input_uncached_usd":0.00133,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.970838624984026}
{"item_id":"senseval2.d000.s010.t009","gold_sense_keys":["church%1:06:00::"],"candidates":[{"index":1,"sense_key":"church%1:14:00::","synset_id":"church.n.01"},{"index":2,"sense_key":"church%1:06:00::","synset_id":"church.n.02"},{"index":3,"sense_key":"church%1:04:00::","synset_id":"church_service.n.01"},{"index":4,"sense_key":"church%1:14:01::","synset_id":"church.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"church%1:06:00::","call_ids":["senseval2.d000.s010.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"church%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":742,"cached_input_tokens":0,"output_tokens":50,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0019839999999999997,"input_uncached_usd":0.0014839999999999999,"input_cached_usd":0.0,"output_usd":0.0005,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8457309999503195}
{"item_id":"senseval2.d000.s010.t010","gold_sense_keys":["church%1:06:00::"],"candidates":[{"index":1,"sense_key":"church%1:14:00::","synset_id":"church.n.01"},{"index":2,"sense_key":"church%1:06:00::","synset_id":"church.n.02"},{"index":3,"sense_key":"church%1:04:00::","synset_id":"church_service.n.01"},{"index":4,"sense_key":"church%1:14:01::","synset_id":"church.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"church%1:06:00::","call_ids":["senseval2.d000.s010.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"church%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":742,"cached_input_tokens":0,"output_tokens":46,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001944,"input_uncached_usd":0.0014839999999999999,"input_cached_usd":0.0,"output_usd":0.00046,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3927596249850467}
{"item_id":"senseval2.d000.s010.t011","gold_sense_keys":["effort%1:04:00::"],"candidates":[{"index":1,"sense_key":"effort%1:04:00::","synset_id":"attempt.n.01"},{"index":2,"sense_key":"effort%1:04:01::","synset_id":"effort.n.02"},{"index":3,"sense_key":"effort%1:04:02::","synset_id":"feat.n.01"},{"index":4,"sense_key":"effort%1:04:03::","synset_id":"campaign.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"effort%1:04:00::","call_ids":["senseval2.d000.s010.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"effort%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":854,"cached_input_tokens":0,"output_tokens":183,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0035380000000000003,"input_uncached_usd":0.0017079999999999999,"input_cached_usd":0.0,"output_usd":0.0018300000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.6528127079363912}
{"item_id":"senseval2.d000.s010.t012","gold_sense_keys":["keep%2:42:00::"],"candidates":[{"index":1,"sense_key":"keep%2:42:00::","synset_id":"keep.v.01"},{"index":2,"sense_key":"keep%2:42:07::","synset_id":"continue.v.01"},{"index":3,"sense_key":"keep%2:40:00::","synset_id":"keep.v.03"},{"index":4,"sense_key":"keep%2:41:01::","synset_id":"prevent.v.02"},{"index":5,"sense_key":"keep%2:41:03::","synset_id":"observe.v.09"},{"index":6,"sense_key":"keep%2:31:00::","synset_id":"observe.v.08"},{"index":7,"sense_key":"keep%2:40:01::","synset_id":"keep.v.07"},{"index":8,"sense_key":"keep%2:32:00::","synset_id":"keep.v.08"},{"index":9,"sense_key":"keep%2:42:01::","synset_id":"keep.v.09"},{"index":10,"sense_key":"keep%2:41:02::","synset_id":"retain.v.02"},{"index":11,"sense_key":"keep%2:34:00::","synset_id":"sustain.v.04"},{"index":12,"sense_key":"keep%2:42:03::","synset_id":"keep.v.12"},{"index":13,"sense_key":"keep%2:41:05::","synset_id":"observe.v.06"},{"index":14,"sense_key":"keep%2:41:00::","synset_id":"restrain.v.01"},{"index":15,"sense_key":"keep%2:42:02::","synset_id":"keep.v.15"},{"index":16,"sense_key":"keep%2:42:12::","synset_id":"keep.v.16"},{"index":17,"sense_key":"keep%2:40:09::","synset_id":"keep_open.v.01"},{"index":18,"sense_key":"keep%2:40:13::","synset_id":"keep.v.18"},{"index":19,"sense_key":"keep%2:40:12::","synset_id":"keep.v.19"},{"index":20,"sense_key":"keep%2:40:10::","synset_id":"keep.v.20"},{"index":21,"sense_key":"keep%2:35:10::","synset_id":"keep.v.21"},{"index":22,"sense_key":"keep%2:30:10::","synset_id":"preserve.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"keep%2:42:00::","call_ids":["senseval2.d000.s010.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"keep%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1881,"cached_input_tokens":0,"output_tokens":128,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.005042,"input_uncached_usd":0.0037619999999999997,"input_cached_usd":0.0,"output_usd":0.00128,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6988475839607418}
{"item_id":"senseval2.d000.s010.t013","gold_sense_keys":["bell%1:06:00::"],"candidates":[{"index":1,"sense_key":"bell%1:06:00::","synset_id":"bell.n.01"},{"index":2,"sense_key":"bell%1:06:02::","synset_id":"doorbell.n.01"},{"index":3,"sense_key":"bell%1:11:00::","synset_id":"bell.n.03"},{"index":4,"sense_key":"bell%1:28:00::","synset_id":"bell.n.04"},{"index":5,"sense_key":"bell%1:25:00::","synset_id":"bell.n.05"},{"index":6,"sense_key":"bell%1:18:02::","synset_id":"bell.n.06"},{"index":7,"sense_key":"bell%1:18:01::","synset_id":"bell.n.07"},{"index":8,"sense_key":"bell%1:18:00::","synset_id":"bell.n.08"},{"index":9,"sense_key":"bell%1:06:04::","synset_id":"chime.n.01"},{"index":10,"sense_key":"bell%1:06:03::","synset_id":"bell.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bell%1:06:00::","call_ids":["senseval2.d000.s010.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bell%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1182,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0030039999999999997,"input_uncached_usd":0.0023639999999999998,"input_cached_usd":0.0,"output_usd":0.00064,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3877612919313833}
{"item_id":"senseval2.d000.s010.t014","gold_sense_keys":["sound%2:39:00::","sound%2:39:03::"],"candidates":[{"index":1,"sense_key":"sound%2:39:06::","synset_id":"sound.v.01"},{"index":2,"sense_key":"sound%2:39:00::","synset_id":"sound.v.02"},{"index":3,"sense_key":"sound%2:39:03::","synset_id":"sound.v.03"},{"index":4,"sense_key":"sound%2:39:02::","synset_id":"sound.v.04"},{"index":5,"sense_key":"sound%2:32:00::","synset_id":"voice.v.02"},{"index":6,"sense_key":"sound%2:39:01::","synset_id":"sound.v.06"},{"index":7,"sense_key":"sound%2:30:00::","synset_id":"fathom.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"sound%2:39:01::","call_ids":["senseval2.d000.s010.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"sound%2:39:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":868,"cached_input_tokens":0,"output_tokens":107,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002806,"input_uncached_usd":0.001736,"input_cached_usd":0.0,"output_usd":0.00107,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8575218330370262}
{"item_id":"senseval2.d000.s011.t000","gold_sense_keys":["even%4:02:00::"],"candidates":[{"index":1,"sense_key":"even%4:02:00::","synset_id":"even.r.01"},{"index":2,"sense_key":"even%4:02:03::","synset_id":"even.r.02"},{"index":3,"sense_key":"even%4:02:01::","synset_id":"even.r.03"},{"index":4,"sense_key":"even%4:02:04::","synset_id":"even.r.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"even%4:02:00::","call_ids":["senseval2.d000.s011.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"even%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":754,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001628,"input_uncached_usd":0.001508,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9814081249060109}
{"item_id":"senseval2.d000.s011.t001","gold_sense_keys":["service%1:04:01::"],"candidates":[{"index":1,"sense_key":"service%1:04:08::","synset_id":"service.n.01"},{"index":2,"sense_key":"service%1:04:00::","synset_id":"service.n.02"},{"index":3,"sense_key":"service%1:04:01::","synset_id":"service.n.03"},{"index":4,"sense_key":"service%1:14:05::","synset_id":"service.n.04"},{"index":5,"sense_key":"service%1:04:07::","synset_id":"service.n.05"},{"index":6,"sense_key":"service%1:14:00::","synset_id":"military_service.n.01"},{"index":7,"sense_key":"service%1:18:00::","synset_id":"service.n.07"},{"index":8,"sense_key":"service%1:07:00::","synset_id":"avail.n.01"},{"index":9,"sense_key":"service%1:06:00::","synset_id":"service.n.09"},{"index":10,"sense_key":"service%1:04:03::","synset_id":"servicing.n.01"},{"index":11,"sense_key":"service%1:04:09::","synset_id":"service.n.11"},{"index":12,"sense_key":"service%1:04:02::","synset_id":"serve.n.01"},{"index":13,"sense_key":"service%1:04:04::","synset_id":"service.n.13"},{"index":14,"sense_key":"service%1:04:05::","synset_id":"overhaul.n.01"},{"index":15,"sense_key":"service%1:04:06::","synset_id":"service.n.15"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"service%1:04:01::","call_ids":["senseval2.d000.s011.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"service%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1325,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00277,"input_uncached_usd":0.00265,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.054217082913965}
{"item_id":"senseval2.d000.s011.t002","gold_sense_keys":["tower%1:06:00::"],"candidates":[{"index":1,"sense_key":"tower%1:06:00::","synset_id":"tower.n.01"},{"index":2,"sense_key":"tower%1:25:00::","synset_id":"column.n.04"},{"index":3,"sense_key":"tower%1:06:01::","synset_id":"tugboat.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"tower%1:06:00::","call_ids":["senseval2.d000.s011.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"tower%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":709,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001538,"input_uncached_usd":0.001418,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2030663749901578}
{"item_id":"senseval2.d000.s011.t003","gold_sense_keys":["scrape%2:35:03::"],"candidates":[{"index":1,"sense_key":"scrape%2:35:00::","synset_id":"scrape.v.01"},{"index":2,"sense_key":"scrape%2:36:00::","synset_id":"scrape.v.02"},{"index":3,"sense_key":"scrape%2:35:02::","synset_id":"scratch.v.02"},{"index":4,"sense_key":"scrape%2:38:00::","synset_id":"scrape.v.04"},{"index":5,"sense_key":"scrape%2:35:03::","synset_id":"scrape.v.05"},{"index":6,"sense_key":"scrape%2:35:01::","synset_id":"skin.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"scrape%2:35:03::","call_ids":["senseval2.d000.s011.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"scrape%2:35:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":872,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002474,"input_uncached_usd":0.0017439999999999999,"input_cached_usd":0.0,"output_usd":0.0007300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5544910829048604}
{"item_id":"senseval2.d000.s011.t004","gold_sense_keys":["say%2:32:00::","say%2:32:15::"],"candidates":[{"index":1,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":6,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":9,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":10,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":11,"sense_key":"say%2:32:09::","synset_id":"say.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"say%2:32:00::","call_ids":["senseval2.d000.s011.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"say%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1185,"cached_input_tokens":0,"output_tokens":108,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00345,"input_uncached_usd":0.0023699999999999997,"input_cached_usd":0.0,"output_usd":0.00108,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7831650000298396}
{"item_id":"senseval2.d000.s011.t006","gold_sense_keys":["worker%1:18:00::"],"candidates":[{"index":1,"sense_key":"worker%1:18:00::","synset_id":"worker.n.01"},{"index":2,"sense_key":"worker%1:18:01::","synset_id":"proletarian.n.01"},{"index":3,"sense_key":"worker%1:05:01::","synset_id":"worker.n.03"},{"index":4,"sense_key":"worker%1:18:02::","synset_id":"actor.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"worker%1:18:00::","call_ids":["senseval2.d000.s011.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"worker%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":749,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001618,"input_uncached_usd":0.001498,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6588049998972565}
{"item_id":"senseval2.d000.s012.t000","gold_sense_keys":["try%2:41:00::"],"candidates":[{"index":1,"sense_key":"try%2:41:00::","synset_id":"try.v.01"},{"index":2,"sense_key":"try%2:41:01::","synset_id":"test.v.01"},{"index":3,"sense_key":"try%2:41:06::","synset_id":"judge.v.05"},{"index":4,"sense_key":"try%2:34:00::","synset_id":"sample.v.01"},{"index":5,"sense_key":"try%2:41:05::","synset_id":"hear.v.03"},{"index":6,"sense_key":"try%2:37:01::","synset_id":"try.v.06"},{"index":7,"sense_key":"try%2:37:00::","synset_id":"try.v.07"},{"index":8,"sense_key":"try%2:30:00::","synset_id":"try.v.08"},{"index":9,"sense_key":"try%2:29:00::","synset_id":"try_on.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"try%2:41:00::","call_ids":["senseval2.d000.s012.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"try%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1130,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0023799999999999997,"input_uncached_usd":0.00226,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.458864958025515}
{"item_id":"senseval2.d000.s012.t001","gold_sense_keys":["train%2:31:01::","train%2:32:00::","train%2:41:00::"],"candidates":[{"index":1,"sense_key":"train%2:31:01::","synset_id":"train.v.01"},{"index":2,"sense_key":"train%2:31:00::","synset_id":"train.v.02"},{"index":3,"sense_key":"train%2:41:02::","synset_id":"discipline.v.01"},{"index":4,"sense_key":"train%2:41:00::","synset_id":"prepare.v.05"},{"index":5,"sense_key":"train%2:41:01::","synset_id":"educate.v.03"},{"index":6,"sense_key":"train%2:33:00::","synset_id":"aim.v.01"},{"index":7,"sense_key":"train%2:32:00::","synset_id":"coach.v.01"},{"index":8,"sense_key":"train%2:29:00::","synset_id":"train.v.08"},{"index":9,"sense_key":"train%2:41:03::","synset_id":"train.v.09"},{"index":10,"sense_key":"train%2:38:00::","synset_id":"train.v.10"},{"index":11,"sense_key":"train%2:35:04::","synset_id":"trail.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"train%2:31:01::","call_ids":["senseval2.d000.s012.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"train%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1314,"cached_input_tokens":0,"output_tokens":189,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0045179999999999994,"input_uncached_usd":0.0026279999999999997,"input_cached_usd":0.0,"output_usd":0.0018900000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.593867582967505}
{"item_id":"senseval2.d000.s012.t003","gold_sense_keys":["disco%1:06:00::"],"candidates":[{"index":1,"sense_key":"disco%1:10:00::","synset_id":"disco.n.01"},{"index":2,"sense_key":"disco%1:06:00::","synset_id":"disco.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"disco%1:06:00::","call_ids":["senseval2.d000.s012.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"disco%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":653,"cached_input_tokens":0,"output_tokens":69,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001996,"input_uncached_usd":0.0013059999999999999,"input_cached_usd":0.0,"output_usd":0.0006900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3083279998973012}
{"item_id":"senseval2.d000.s012.t004","gold_sense_keys":["dance%1:11:00::","dance%1:14:00::"],"candidates":[{"index":1,"sense_key":"dance%1:10:00::","synset_id":"dance.n.01"},{"index":2,"sense_key":"dance%1:14:00::","synset_id":"dance.n.02"},{"index":3,"sense_key":"dance%1:04:00::","synset_id":"dancing.n.01"},{"index":4,"sense_key":"dance%1:11:00::","synset_id":"dance.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"dance%1:11:00::","call_ids":["senseval2.d000.s012.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"dance%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":690,"cached_input_tokens":0,"output_tokens":102,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0024000000000000002,"input_uncached_usd":0.00138,"input_cached_usd":0.0,"output_usd":0.00102,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6184954170603305}
{"item_id":"senseval2.d000.s012.t005","gold_sense_keys":["just%4:02:03::"],"candidates":[{"index":1,"sense_key":"just%4:02:00::","synset_id":"merely.r.01"},{"index":2,"sense_key":"just%4:02:05::","synset_id":"precisely.r.01"},{"index":3,"sense_key":"just%4:02:01::","synset_id":"just.r.03"},{"index":4,"sense_key":"just%4:02:03::","synset_id":"just.r.04"},{"index":5,"sense_key":"just%4:02:04::","synset_id":"barely.r.01"},{"index":6,"sense_key":"just%4:02:06::","synset_id":"just.r.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"just%4:02:03::","call_ids":["senseval2.d000.s012.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"just%4:02:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":926,"cached_input_tokens":0,"output_tokens":168,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003532,"input_uncached_usd":0.001852,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.265006124973297}
{"item_id":"senseval2.d000.s012.t006","gold_sense_keys":["drift%2:38:04::"],"candidates":[{"index":1,"sense_key":"drift%2:38:02::","synset_id":"float.v.01"},{"index":2,"sense_key":"drift%2:38:04::","synset_id":"stray.v.02"},{"index":3,"sense_key":"drift%2:38:01::","synset_id":"roll.v.12"},{"index":4,"sense_key":"drift%2:42:03::","synset_id":"drift.v.04"},{"index":5,"sense_key":"drift%2:42:00::","synset_id":"freewheel.v.01"},{"index":6,"sense_key":"drift%2:38:05::","synset_id":"drift.v.06"},{"index":7,"sense_key":"drift%2:38:06::","synset_id":"drift.v.07"},{"index":8,"sense_key":"drift%2:35:03::","synset_id":"drift.v.08"},{"index":9,"sense_key":"drift%2:30:10::","synset_id":"drift.v.09"},{"index":10,"sense_key":"drift%2:30:00::","synset_id":"drift.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"drift%2:38:05::","call_ids":["senseval2.d000.s012.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"drift%2:38:05::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1138,"cached_input_tokens":0,"output_tokens":238,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004656,"input_uncached_usd":0.002276,"input_cached_usd":0.0,"output_usd":0.00238,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.859108250006102}
{"item_id":"senseval2.d000.s012.t007","gold_sense_keys":["away%4:02:00::"],"candidates":[{"index":1,"sense_key":"away%4:02:00::","synset_id":"away.r.01"},{"index":2,"sense_key":"away%4:02:01::","synset_id":"away.r.02"},{"index":3,"sense_key":"away%4:02:03::","synset_id":"aside.r.02"},{"index":4,"sense_key":"away%4:02:04::","synset_id":"away.r.04"},{"index":5,"sense_key":"away%4:02:08::","synset_id":"off.r.02"},{"index":6,"sense_key":"away%4:02:05::","synset_id":"away.r.06"},{"index":7,"sense_key":"away%4:02:09::","synset_id":"away.r.07"},{"index":8,"sense_key":"away%4:02:10::","synset_id":"away.r.08"},{"index":9,"sense_key":"away%4:02:07::","synset_id":"away.r.09"},{"index":10,"sense_key":"away%4:02:06::","synset_id":"away.r.10"},{"index":11,"sense_key":"away%4:02:02::","synset_id":"aside.r.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"away%4:02:00::","call_ids":["senseval2.d000.s012.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"away%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1187,"cached_input_tokens":0,"output_tokens":148,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003854,"input_uncached_usd":0.002374,"input_cached_usd":0.0,"output_usd":0.0014800000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9757735000457615}
{"item_id":"senseval2.d000.s013.t000","gold_sense_keys":["worry%2:37:00::"],"candidates":[{"index":1,"sense_key":"worry%2:37:00::","synset_id":"worry.v.01"},{"index":2,"sense_key":"worry%2:37:12::","synset_id":"worry.v.02"},{"index":3,"sense_key":"worry%2:37:01::","synset_id":"worry.v.03"},{"index":4,"sense_key":"worry%2:42:00::","synset_id":"concern.v.02"},{"index":5,"sense_key":"worry%2:35:00::","synset_id":"worry.v.05"},{"index":6,"sense_key":"worry%2:35:01::","synset_id":"worry.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"worry%2:37:00::","call_ids":["senseval2.d000.s013.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"worry%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":804,"cached_input_tokens":0,"output_tokens":131,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002918,"input_uncached_usd":0.0016079999999999998,"input_cached_usd":0.0,"output_usd":0.0013100000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.8747906249482185}
{"item_id":"senseval2.d000.s013.t003","gold_sense_keys":["youth%1:07:00::","youth%1:14:00::","youth%1:26:00::","youth%1:28:00::"],"candidates":[{"index":1,"sense_key":"youth%1:18:00::","synset_id":"young_person.n.01"},{"index":2,"sense_key":"youth%1:14:00::","synset_id":"young.n.09"},{"index":3,"sense_key":"youth%1:28:00::","synset_id":"youth.n.03"},{"index":4,"sense_key":"youth%1:26:00::","synset_id":"youth.n.04"},{"index":5,"sense_key":"youth%1:28:01::","synset_id":"youth.n.05"},{"index":6,"sense_key":"youth%1:07:00::","synset_id":"youth.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"youth%1:14:00::","call_ids":["senseval2.d000.s013.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"youth%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":772,"cached_input_tokens":0,"output_tokens":109,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002634,"input_uncached_usd":0.001544,"input_cached_usd":0.0,"output_usd":0.00109,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6675967500777915}
{"item_id":"senseval2.d000.s013.t005","gold_sense_keys":["rank%1:14:02::"],"candidates":[{"index":1,"sense_key":"rank%1:14:00::","synset_id":"rank.n.01"},{"index":2,"sense_key":"rank%1:26:00::","synset_id":"rank.n.02"},{"index":3,"sense_key":"rank%1:14:01::","synset_id":"rank_and_file.n.01"},{"index":4,"sense_key":"rank%1:26:01::","synset_id":"social_station.n.01"},{"index":5,"sense_key":"rank%1:14:02::","synset_id":"membership.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"rank%1:14:02::","call_ids":["senseval2.d000.s013.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"rank%1:14:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":811,"cached_input_tokens":0,"output_tokens":106,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0026820000000000004,"input_uncached_usd":0.001622,"input_cached_usd":0.0,"output_usd":0.0010600000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0841160831041634}
{"item_id":"senseval2.d000.s013.t006","gold_sense_keys":["group%1:03:00::"],"candidates":[{"index":1,"sense_key":"group%1:03:00::","synset_id":"group.n.01"},{"index":2,"sense_key":"group%1:27:00::","synset_id":"group.n.02"},{"index":3,"sense_key":"group%1:09:00::","synset_id":"group.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"group%1:03:00::","call_ids":["senseval2.d000.s013.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"group%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":645,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00141,"input_uncached_usd":0.00129,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0465084159513935}
{"item_id":"senseval2.d000.s013.t007","gold_sense_keys":["keep%2:42:00::"],"candidates":[{"index":1,"sense_key":"keep%2:42:00::","synset_id":"keep.v.01"},{"index":2,"sense_key":"keep%2:42:07::","synset_id":"continue.v.01"},{"index":3,"sense_key":"keep%2:40:00::","synset_id":"keep.v.03"},{"index":4,"sense_key":"keep%2:41:01::","synset_id":"prevent.v.02"},{"index":5,"sense_key":"keep%2:41:03::","synset_id":"observe.v.09"},{"index":6,"sense_key":"keep%2:31:00::","synset_id":"observe.v.08"},{"index":7,"sense_key":"keep%2:40:01::","synset_id":"keep.v.07"},{"index":8,"sense_key":"keep%2:32:00::","synset_id":"keep.v.08"},{"index":9,"sense_key":"keep%2:42:01::","synset_id":"keep.v.09"},{"index":10,"sense_key":"keep%2:41:02::","synset_id":"retain.v.02"},{"index":11,"sense_key":"keep%2:34:00::","synset_id":"sustain.v.04"},{"index":12,"sense_key":"keep%2:42:03::","synset_id":"keep.v.12"},{"index":13,"sense_key":"keep%2:41:05::","synset_id":"observe.v.06"},{"index":14,"sense_key":"keep%2:41:00::","synset_id":"restrain.v.01"},{"index":15,"sense_key":"keep%2:42:02::","synset_id":"keep.v.15"},{"index":16,"sense_key":"keep%2:42:12::","synset_id":"keep.v.16"},{"index":17,"sense_key":"keep%2:40:09::","synset_id":"keep_open.v.01"},{"index":18,"sense_key":"keep%2:40:13::","synset_id":"keep.v.18"},{"index":19,"sense_key":"keep%2:40:12::","synset_id":"keep.v.19"},{"index":20,"sense_key":"keep%2:40:10::","synset_id":"keep.v.20"},{"index":21,"sense_key":"keep%2:35:10::","synset_id":"keep.v.21"},{"index":22,"sense_key":"keep%2:30:10::","synset_id":"preserve.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"keep%2:42:00::","call_ids":["senseval2.d000.s013.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"keep%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1831,"cached_input_tokens":0,"output_tokens":187,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0055320000000000005,"input_uncached_usd":0.003662,"input_cached_usd":0.0,"output_usd":0.0018700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.8131290409946814}
{"item_id":"senseval2.d000.s013.t008","gold_sense_keys":["bell%1:06:00::"],"candidates":[{"index":1,"sense_key":"bell%1:06:00::","synset_id":"bell.n.01"},{"index":2,"sense_key":"bell%1:06:02::","synset_id":"doorbell.n.01"},{"index":3,"sense_key":"bell%1:11:00::","synset_id":"bell.n.03"},{"index":4,"sense_key":"bell%1:28:00::","synset_id":"bell.n.04"},{"index":5,"sense_key":"bell%1:25:00::","synset_id":"bell.n.05"},{"index":6,"sense_key":"bell%1:18:02::","synset_id":"bell.n.06"},{"index":7,"sense_key":"bell%1:18:01::","synset_id":"bell.n.07"},{"index":8,"sense_key":"bell%1:18:00::","synset_id":"bell.n.08"},{"index":9,"sense_key":"bell%1:06:04::","synset_id":"chime.n.01"},{"index":10,"sense_key":"bell%1:06:03::","synset_id":"bell.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bell%1:06:00::","call_ids":["senseval2.d000.s013.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bell%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1131,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0029219999999999997,"input_uncached_usd":0.0022619999999999997,"input_cached_usd":0.0,"output_usd":0.0006600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5759100000141189}
{"item_id":"senseval2.d000.s014.t000","gold_sense_keys":["history%1:28:00::"],"candidates":[{"index":1,"sense_key":"history%1:28:00::","synset_id":"history.n.01"},{"index":2,"sense_key":"history%1:10:00::","synset_id":"history.n.02"},{"index":3,"sense_key":"history%1:09:00::","synset_id":"history.n.03"},{"index":4,"sense_key":"history%1:28:02::","synset_id":"history.n.04"},{"index":5,"sense_key":"history%1:09:01::","synset_id":"history.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"history%1:28:02::","call_ids":["senseval2.d000.s014.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"history%1:28:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":798,"cached_input_tokens":0,"output_tokens":162,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003216,"input_uncached_usd":0.001596,"input_cached_usd":0.0,"output_usd":0.0016200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0468398340744898}
{"item_id":"senseval2.d000.s014.t001","gold_sense_keys":["side%1:10:00::","side%1:14:00::"],"candidates":[{"index":1,"sense_key":"side%1:15:02::","synset_id":"side.n.01"},{"index":2,"sense_key":"side%1:14:00::","synset_id":"side.n.02"},{"index":3,"sense_key":"side%1:08:00::","synset_id":"side.n.03"},{"index":4,"sense_key":"side%1:15:00::","synset_id":"side.n.04"},{"index":5,"sense_key":"side%1:06:00::","synset_id":"side.n.05"},{"index":6,"sense_key":"side%1:09:00::","synset_id":"side.n.06"},{"index":7,"sense_key":"side%1:15:01::","synset_id":"side.n.07"},{"index":8,"sense_key":"side%1:14:01::","synset_id":"side.n.08"},{"index":9,"sense_key":"side%1:13:00::","synset_id":"side.n.09"},{"index":10,"sense_key":"side%1:10:00::","synset_id":"side.n.10"},{"index":11,"sense_key":"side%1:17:01::","synset_id":"slope.n.01"},{"index":12,"sense_key":"side%1:11:00::","synset_id":"english.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"side%1:14:00::","call_ids":["senseval2.d000.s014.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"side%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1193,"cached_input_tokens":0,"output_tokens":119,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003576,"input_uncached_usd":0.002386,"input_cached_usd":0.0,"output_usd":0.00119,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1135017500491813}
{"item_id":"senseval2.d000.s015.t001","gold_sense_keys":["survey%1:04:02::"],"candidates":[{"index":1,"sense_key":"survey%1:04:02::","synset_id":"survey.n.01"},{"index":2,"sense_key":"survey%1:10:00::","synset_id":"sketch.n.03"},{"index":3,"sense_key":"survey%1:04:01::","synset_id":"view.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"survey%1:04:02::","call_ids":["senseval2.d000.s015.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"survey%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":669,"cached_input_tokens":0,"output_tokens":118,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0025180000000000003,"input_uncached_usd":0.001338,"input_cached_usd":0.0,"output_usd":0.00118,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7101178340381011}
{"item_id":"senseval2.d000.s015.t002","gold_sense_keys":["take%2:41:04::"],"candidates":[{"index":1,"sense_key":"take%2:41:04::","synset_id":"take.v.01"},{"index":2,"sense_key":"take%2:40:06::","synset_id":"take.v.02"},{"index":3,"sense_key":"take%2:38:09::","synset_id":"lead.v.01"},{"index":4,"sense_key":"take%2:35:00::","synset_id":"take.v.04"},{"index":5,"sense_key":"take%2:30:01::","synset_id":"assume.v.03"},{"index":6,"sense_key":"take%2:31:07::","synset_id":"take.v.06"},{"index":7,"sense_key":"take%2:38:10::","synset_id":"bring.v.01"},{"index":8,"sense_key":"take%2:40:00::","synset_id":"take.v.08"},{"index":9,"sense_key":"take%2:38:02::","synset_id":"take.v.09"},{"index":10,"sense_key":"take%2:31:01::","synset_id":"choose.v.01"},{"index":11,"sense_key":"take%2:40:05::","synset_id":"accept.v.02"},{"index":12,"sense_key":"take%2:41:00::","synset_id":"fill.v.04"},{"index":13,"sense_key":"take%2:31:03::","synset_id":"consider.v.03"},{"index":14,"sense_key":"take%2:42:00::","synset_id":"necessitate.v.01"},{"index":15,"sense_key":"take%2:39:00::","synset_id":"take.v.15"},{"index":16,"sense_key":"take%2:32:02::","synset_id":"film.v.01"},{"index":17,"sense_key":"take%2:30:00::","synset_id":"remove.v.01"},{"index":18,"sense_key":"take%2:34:00::","synset_id":"consume.v.02"},{"index":19,"sense_key":"take%2:31:09::","synset_id":"take.v.19"},{"index":20,"sense_key":"take%2:40:15::","synset_id":"take.v.20"},{"index":21,"sense_key":"take%2:40:01::","synset_id":"take.v.21"},{"index":22,"sense_key":"take%2:38:00::","synset_id":"assume.v.05"},{"index":23,"sense_key":"take%2:40:07::","synset_id":"accept.v.05"},{"index":24,"sense_key":"take%2:30:08::","synset_id":"take.v.24"},{"index":25,"sense_key":"take%2:31:02::","synset_id":"learn.v.04"},{"index":26,"sense_key":"take%2:32:00::","synset_id":"claim.v.05"},{"index":27,"sense_key":"take%2:38:05::","synset_id":"take.v.27"},{"index":28,"sense_key":"take%2:33:09::","synset_id":"aim.v.01"},{"index":29,"sense_key":"take%2:30:09::","synset_id":"take.v.29"},{"index":30,"sense_key":"take%2:42:10::","synset_id":"carry.v.02"},{"index":31,"sense_key":"take%2:40:03::","synset_id":"lease.v.04"},{"index":32,"sense_key":"take%2:40:04::","synset_id":"subscribe.v.05"},{"index":33,"sense_key":"take%2:40:02::","synset_id":"take.v.33"},{"index":34,"sense_key":"take%2:40:08::","synset_id":"take.v.34"},{"index":35,"sense_key":"take%2:35:14::","synset_id":"take.v.35"},{"index":36,"sense_key":"take%2:32:09::","synset_id":"claim.v.04"},{"index":37,"sense_key":"take%2:42:03::","synset_id":"accept.v.08"},{"index":38,"sense_key":"take%2:42:15::","synset_id":"contain.v.05"},{"index":39,"sense_key":"take%2:41:13::","synset_id":"take.v.39"},{"index":40,"sense_key":"take%2:38:11::","synset_id":"drive.v.16"},{"index":41,"sense_key":"take%2:33:08::","synset_id":"take.v.41"},{"index":42,"sense_key":"take%2:29:08::","synset_id":"contract.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":24,"chosen_sense_key":"take%2:30:08::","call_ids":["senseval2.d000.s015.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":24,"predicted_sense_key":"take%2:30:08::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":3116,"cached_input_tokens":0,"output_tokens":193,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.008161999999999999,"input_uncached_usd":0.006232,"input_cached_usd":0.0,"output_usd":0.00193,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1651020830031484}
{"item_id":"senseval2.d000.s015.t003","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["senseval2.d000.s015.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":773,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001666,"input_uncached_usd":0.0015459999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.140207124990411}
{"item_id":"senseval2.d000.s015.t007","gold_sense_keys":["church%1:06:00::"],"candidates":[{"index":1,"sense_key":"church%1:14:00::","synset_id":"church.n.01"},{"index":2,"sense_key":"church%1:06:00::","synset_id":"church.n.02"},{"index":3,"sense_key":"church%1:04:00::","synset_id":"church_service.n.01"},{"index":4,"sense_key":"church%1:14:01::","synset_id":"church.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"church%1:06:00::","call_ids":["senseval2.d000.s015.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"church%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":721,"cached_input_tokens":0,"output_tokens":46,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001902,"input_uncached_usd":0.001442,"input_cached_usd":0.0,"output_usd":0.00046,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3510387500282377}
{"item_id":"senseval2.d000.s015.t008","gold_sense_keys":["bell%1:06:00::"],"candidates":[{"index":1,"sense_key":"bell%1:06:00::","synset_id":"bell.n.01"},{"index":2,"sense_key":"bell%1:06:02::","synset_id":"doorbell.n.01"},{"index":3,"sense_key":"bell%1:11:00::","synset_id":"bell.n.03"},{"index":4,"sense_key":"bell%1:28:00::","synset_id":"bell.n.04"},{"index":5,"sense_key":"bell%1:25:00::","synset_id":"bell.n.05"},{"index":6,"sense_key":"bell%1:18:02::","synset_id":"bell.n.06"},{"index":7,"sense_key":"bell%1:18:01::","synset_id":"bell.n.07"},{"index":8,"sense_key":"bell%1:18:00::","synset_id":"bell.n.08"},{"index":9,"sense_key":"bell%1:06:04::","synset_id":"chime.n.01"},{"index":10,"sense_key":"bell%1:06:03::","synset_id":"bell.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bell%1:06:00::","call_ids":["senseval2.d000.s015.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bell%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1161,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0024419999999999997,"input_uncached_usd":0.002322,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9760013750055805}
{"item_id":"senseval2.d000.s015.t009","gold_sense_keys":["no%4:02:01::"],"candidates":[{"index":1,"sense_key":"no%4:02:02::","synset_id":"no.r.01"},{"index":2,"sense_key":"no%4:02:01::","synset_id":"no.r.02"},{"index":3,"sense_key":"no%4:02:00::","synset_id":"no.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"no%4:02:01::","call_ids":["senseval2.d000.s015.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"no%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":673,"cached_input_tokens":0,"output_tokens":293,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004276,"input_uncached_usd":0.001346,"input_cached_usd":0.0,"output_usd":0.0029300000000000003,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.0380453329999}
{"item_id":"senseval2.d000.s015.t011","gold_sense_keys":["ring%2:39:01::"],"candidates":[{"index":1,"sense_key":"ring%2:39:00::","synset_id":"ring.v.01"},{"index":2,"sense_key":"ring%2:39:04::","synset_id":"resound.v.01"},{"index":3,"sense_key":"ring%2:39:01::","synset_id":"ring.v.03"},{"index":4,"sense_key":"ring%2:32:00::","synset_id":"call.v.03"},{"index":5,"sense_key":"ring%2:35:03::","synset_id":"surround.v.01"},{"index":6,"sense_key":"ring%2:35:00::","synset_id":"ring.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"ring%2:39:01::","call_ids":["senseval2.d000.s015.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"ring%2:39:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":918,"cached_input_tokens":0,"output_tokens":102,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002856,"input_uncached_usd":0.001836,"input_cached_usd":0.0,"output_usd":0.00102,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7640544169116765}
{"item_id":"senseval2.d000.s015.t012","gold_sense_keys":["sunday%1:28:00::"],"candidates":[{"index":1,"sense_key":"sunday%1:28:00::","synset_id":"sunday.n.01"},{"index":2,"sense_key":"sunday%1:18:00::","synset_id":"sunday.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"sunday%1:28:00::","call_ids":["senseval2.d000.s015.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"sunday%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":644,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001408,"input_uncached_usd":0.0012879999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9967057090252638}
{"item_id":"senseval2.d000.s015.t013","gold_sense_keys":["ring%2:39:01::"],"candidates":[{"index":1,"sense_key":"ring%2:39:00::","synset_id":"ring.v.01"},{"index":2,"sense_key":"ring%2:39:04::","synset_id":"resound.v.01"},{"index":3,"sense_key":"ring%2:39:01::","synset_id":"ring.v.03"},{"index":4,"sense_key":"ring%2:32:00::","synset_id":"call.v.03"},{"index":5,"sense_key":"ring%2:35:03::","synset_id":"surround.v.01"},{"index":6,"sense_key":"ring%2:35:00::","synset_id":"ring.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"ring%2:39:01::","call_ids":["senseval2.d000.s015.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"ring%2:39:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":918,"cached_input_tokens":0,"output_tokens":82,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002656,"input_uncached_usd":0.001836,"input_cached_usd":0.0,"output_usd":0.0008200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4561845840653405}
{"item_id":"senseval2.d000.s016.t000","gold_sense_keys":["easy%3:00:01::"],"candidates":[{"index":1,"sense_key":"easy%3:00:01::","synset_id":"easy.a.01"},{"index":2,"sense_key":"easy%5:00:00:unhurried:00","synset_id":"easy.s.01"},{"index":3,"sense_key":"easy%3:00:02::","synset_id":"easy.a.03"},{"index":4,"sense_key":"easy%5:00:00:pleasing:00","synset_id":"easy.s.02"},{"index":5,"sense_key":"easy%5:00:00:light:08","synset_id":"easy.s.03"},{"index":6,"sense_key":"easy%5:00:00:impressionable:00","synset_id":"easy.s.04"},{"index":7,"sense_key":"easy%5:00:00:rich:00","synset_id":"comfortable.s.03"},{"index":8,"sense_key":"easy%5:00:00:gradual:02","synset_id":"easy.s.06"},{"index":9,"sense_key":"easy%5:00:00:comfortable:00","synset_id":"easy.s.07"},{"index":10,"sense_key":"easy%5:00:00:unchaste:00","synset_id":"easy.s.08"},{"index":11,"sense_key":"easy%5:00:00:available:00","synset_id":"easy.s.09"},{"index":12,"sense_key":"easy%5:00:00:abundant:00","synset_id":"easy.s.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"easy%3:00:01::","call_ids":["senseval2.d000.s016.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"easy%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1245,"cached_input_tokens":0,"output_tokens":47,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00296,"input_uncached_usd":0.00249,"input_cached_usd":0.0,"output_usd":0.00047000000000000004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4028364999685436}
{"item_id":"senseval2.d000.s016.t001","gold_sense_keys":["see%2:31:01::"],"candidates":[{"index":1,"sense_key":"see%2:39:00::","synset_id":"see.v.01"},{"index":2,"sense_key":"see%2:31:01::","synset_id":"understand.v.02"},{"index":3,"sense_key":"see%2:39:02::","synset_id":"witness.v.02"},{"index":4,"sense_key":"see%2:36:00::","synset_id":"visualize.v.01"},{"index":5,"sense_key":"see%2:31:00::","synset_id":"see.v.05"},{"index":6,"sense_key":"see%2:31:02::","synset_id":"learn.v.02"},{"index":7,"sense_key":"see%2:39:03::","synset_id":"watch.v.03"},{"index":8,"sense_key":"see%2:38:00::","synset_id":"meet.v.01"},{"index":9,"sense_key":"see%2:32:00::","synset_id":"determine.v.08"},{"index":10,"sense_key":"see%2:31:03::","synset_id":"see.v.10"},{"index":11,"sense_key":"see%2:41:11::","synset_id":"see.v.11"},{"index":12,"sense_key":"see%2:41:12::","synset_id":"see.v.12"},{"index":13,"sense_key":"see%2:41:10::","synset_id":"visit.v.01"},{"index":14,"sense_key":"see%2:41:14::","synset_id":"attend.v.02"},{"index":15,"sense_key":"see%2:35:09::","synset_id":"see.v.15"},{"index":16,"sense_key":"see%2:41:00::","synset_id":"go_steady.v.01"},{"index":17,"sense_key":"see%2:39:05::","synset_id":"see.v.17"},{"index":18,"sense_key":"see%2:31:08::","synset_id":"see.v.18"},{"index":19,"sense_key":"see%2:39:11::","synset_id":"see.v.19"},{"index":20,"sense_key":"see%2:39:13::","synset_id":"examine.v.02"},{"index":21,"sense_key":"see%2:39:12::","synset_id":"experience.v.01"},{"index":22,"sense_key":"see%2:38:03::","synset_id":"see.v.22"},{"index":23,"sense_key":"see%2:33:00::","synset_id":"see.v.23"},{"index":24,"sense_key":"see%2:31:13::","synset_id":"interpret.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"see%2:31:01::","call_ids":["senseval2.d000.s016.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"see%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2038,"cached_input_tokens":0,"output_tokens":36,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004436,"input_uncached_usd":0.004076,"input_cached_usd":0.0,"output_usd":0.00036,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9739911248907447}
{"item_id":"senseval2.d000.s016.t002","gold_sense_keys":["ancient%5:00:00:old:02"],"candidates":[{"index":1,"sense_key":"ancient%5:00:00:past:00","synset_id":"ancient.s.01"},{"index":2,"sense_key":"ancient%5:00:00:old:02","synset_id":"ancient.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"ancient%5:00:00:old:02","call_ids":["senseval2.d000.s016.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"ancient%5:00:00:old:02","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":612,"cached_input_tokens":0,"output_tokens":112,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002344,"input_uncached_usd":0.001224,"input_cached_usd":0.0,"output_usd":0.0011200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8766835000133142}
{"item_id":"senseval2.d000.s016.t003","gold_sense_keys":["art%1:09:00::"],"candidates":[{"index":1,"sense_key":"art%1:06:00::","synset_id":"art.n.01"},{"index":2,"sense_key":"art%1:04:00::","synset_id":"art.n.02"},{"index":3,"sense_key":"art%1:09:00::","synset_id":"art.n.03"},{"index":4,"sense_key":"art%1:10:00::","synset_id":"artwork.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"art%1:09:00::","call_ids":["senseval2.d000.s016.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"art%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":764,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0021279999999999997,"input_uncached_usd":0.0015279999999999998,"input_cached_usd":0.0,"output_usd":0.0006000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6075548330554739}
{"item_id":"senseval2.d000.s017.t000","gold_sense_keys":["less%4:02:00::"],"candidates":[{"index":1,"sense_key":"less%4:02:00::","synset_id":"less.r.01"},{"index":2,"sense_key":"less%4:02:01::","synset_id":"less.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"less%4:02:00::","call_ids":["senseval2.d000.s017.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"less%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":611,"cached_input_tokens":0,"output_tokens":45,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001672,"input_uncached_usd":0.001222,"input_cached_usd":0.0,"output_usd":0.00045000000000000004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5662304579745978}
{"item_id":"senseval2.d000.s017.t002","gold_sense_keys":["version%1:09:01::"],"candidates":[{"index":1,"sense_key":"version%1:10:01::","synset_id":"version.n.01"},{"index":2,"sense_key":"version%1:09:01::","synset_id":"version.n.02"},{"index":3,"sense_key":"version%1:10:02::","synset_id":"adaptation.n.01"},{"index":4,"sense_key":"version%1:10:00::","synset_id":"translation.n.01"},{"index":5,"sense_key":"version%1:09:00::","synset_id":"interpretation.n.01"},{"index":6,"sense_key":"version%1:04:00::","synset_id":"version.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"version%1:09:01::","call_ids":["senseval2.d000.s017.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"version%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":849,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002368,"input_uncached_usd":0.0016979999999999999,"input_cached_usd":0.0,"output_usd":0.00067,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2603659169981256}
{"item_id":"senseval2.d000.s017.t003","gold_sense_keys":["play%2:36:00::","play%2:36:01::","play%2:36:12::"],"candidates":[{"index":1,"sense_key":"play%2:33:00::","synset_id":"play.v.01"},{"index":2,"sense_key":"play%2:41:12::","synset_id":"play.v.02"},{"index":3,"sense_key":"play%2:36:12::","synset_id":"play.v.03"},{"index":4,"sense_key":"play%2:36:02::","synset_id":"act.v.03"},{"index":5,"sense_key":"play%2:41:03::","synset_id":"play.v.05"},{"index":6,"sense_key":"play%2:36:00::","synset_id":"play.v.06"},{"index":7,"sense_key":"play%2:36:01::","synset_id":"play.v.07"},{"index":8,"sense_key":"play%2:29:01::","synset_id":"act.v.05"},{"index":9,"sense_key":"play%2:38:00::","synset_id":"play.v.09"},{"index":10,"sense_key":"play%2:33:01::","synset_id":"play.v.10"},{"index":11,"sense_key":"play%2:41:00::","synset_id":"play.v.11"},{"index":12,"sense_key":"play%2:36:06::","synset_id":"play.v.12"},{"index":13,"sense_key":"play%2:36:11::","synset_id":"play.v.13"},{"index":14,"sense_key":"play%2:36:03::","synset_id":"play.v.14"},{"index":15,"sense_key":"play%2:33:03::","synset_id":"play.v.15"},{"index":16,"sense_key":"play%2:29:00::","synset_id":"play.v.16"},{"index":17,"sense_key":"play%2:41:02::","synset_id":"play.v.17"},{"index":18,"sense_key":"play%2:36:05::","synset_id":"play.v.18"},{"index":19,"sense_key":"play%2:35:00::","synset_id":"toy.v.02"},{"index":20,"sense_key":"play%2:34:13::","synset_id":"play.v.20"},{"index":21,"sense_key":"play%2:31:01::","synset_id":"dally.v.04"},{"index":22,"sense_key":"play%2:42:00::","synset_id":"play.v.22"},{"index":23,"sense_key":"play%2:41:13::","synset_id":"dally.v.01"},{"index":24,"sense_key":"play%2:38:03::","synset_id":"play.v.24"},{"index":25,"sense_key":"play%2:36:10::","synset_id":"act.v.10"},{"index":26,"sense_key":"play%2:36:04::","synset_id":"play.v.26"},{"index":27,"sense_key":"play%2:36:13::","synset_id":"bring.v.03"},{"index":28,"sense_key":"play%2:35:07::","synset_id":"play.v.28"},{"index":29,"sense_key":"play%2:33:07::","synset_id":"play.v.29"},{"index":30,"sense_key":"play%2:33:08::","synset_id":"bet.v.02"},{"index":31,"sense_key":"play%2:33:09::","synset_id":"play.v.31"},{"index":32,"sense_key":"play%2:33:13::","synset_id":"play.v.32"},{"index":33,"sense_key":"play%2:33:12::","synset_id":"play.v.33"},{"index":34,"sense_key":"play%2:33:02::","synset_id":"meet.v.10"},{"index":35,"sense_key":"play%2:29:04::","synset_id":"play.v.35"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"play%2:36:12::","call_ids":["senseval2.d000.s017.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"play%2:36:12::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2632,"cached_input_tokens":0,"output_tokens":310,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.008364,"input_uncached_usd":0.005264,"input_cached_usd":0.0,"output_usd":0.0031000000000000003,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.5384882919024676}
{"item_id":"senseval2.d000.s017.t004","gold_sense_keys":["tune%1:10:00::"],"candidates":[{"index":1,"sense_key":"tune%1:10:00::","synset_id":"tune.n.01"},{"index":2,"sense_key":"tune%1:07:00::","synset_id":"tune.n.02"},{"index":3,"sense_key":"tune%1:04:00::","synset_id":"tune.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"tune%1:10:00::","call_ids":["senseval2.d000.s017.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"tune%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":694,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001508,"input_uncached_usd":0.001388,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.104496082989499}
{"item_id":"senseval2.d000.s017.t005","gold_sense_keys":["bell%1:06:00::"],"candidates":[{"index":1,"sense_key":"bell%1:06:00::","synset_id":"bell.n.01"},{"index":2,"sense_key":"bell%1:06:02::","synset_id":"doorbell.n.01"},{"index":3,"sense_key":"bell%1:11:00::","synset_id":"bell.n.03"},{"index":4,"sense_key":"bell%1:28:00::","synset_id":"bell.n.04"},{"index":5,"sense_key":"bell%1:25:00::","synset_id":"bell.n.05"},{"index":6,"sense_key":"bell%1:18:02::","synset_id":"bell.n.06"},{"index":7,"sense_key":"bell%1:18:01::","synset_id":"bell.n.07"},{"index":8,"sense_key":"bell%1:18:00::","synset_id":"bell.n.08"},{"index":9,"sense_key":"bell%1:06:04::","synset_id":"chime.n.01"},{"index":10,"sense_key":"bell%1:06:03::","synset_id":"bell.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bell%1:06:00::","call_ids":["senseval2.d000.s017.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bell%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1151,"cached_input_tokens":0,"output_tokens":242,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0047220000000000005,"input_uncached_usd":0.002302,"input_cached_usd":0.0,"output_usd":0.0024200000000000003,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.4081680830568075}
{"item_id":"senseval2.d000.s017.t008","gold_sense_keys":["europe%1:14:01::","europe%1:17:00::"],"candidates":[{"index":1,"sense_key":"europe%1:17:00::","synset_id":"europe.n.01"},{"index":2,"sense_key":"europe%1:14:00::","synset_id":"european_union.n.01"},{"index":3,"sense_key":"europe%1:14:01::","synset_id":"europe.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"europe%1:17:00::","call_ids":["senseval2.d000.s017.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"europe%1:17:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":738,"cached_input_tokens":0,"output_tokens":95,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002426,"input_uncached_usd":0.0014759999999999999,"input_cached_usd":0.0,"output_usd":0.0009500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0163260409608483}
{"item_id":"senseval2.d000.s017.t009","gold_sense_keys":["consider%2:31:00::"],"candidates":[{"index":1,"sense_key":"consider%2:31:00::","synset_id":"see.v.05"},{"index":2,"sense_key":"consider%2:39:00::","synset_id":"study.v.03"},{"index":3,"sense_key":"consider%2:31:01::","synset_id":"consider.v.03"},{"index":4,"sense_key":"consider%2:32:00::","synset_id":"consider.v.04"},{"index":5,"sense_key":"consider%2:32:02::","synset_id":"consider.v.05"},{"index":6,"sense_key":"consider%2:31:08::","synset_id":"think.v.01"},{"index":7,"sense_key":"consider%2:39:04::","synset_id":"regard.v.02"},{"index":8,"sense_key":"consider%2:39:08::","synset_id":"view.v.02"},{"index":9,"sense_key":"consider%2:31:05::","synset_id":"consider.v.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"consider%2:31:00::","call_ids":["senseval2.d000.s017.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"consider%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1036,"cached_input_tokens":0,"output_tokens":97,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003042,"input_uncached_usd":0.002072,"input_cached_usd":0.0,"output_usd":0.00097,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8216174999251962}
{"item_id":"senseval2.d000.s017.t010","gold_sense_keys":["english%1:18:00::"],"candidates":[{"index":1,"sense_key":"english%1:10:00::","synset_id":"english.n.01"},{"index":2,"sense_key":"english%1:18:00::","synset_id":"english.n.02"},{"index":3,"sense_key":"english%1:09:00::","synset_id":"english.n.03"},{"index":4,"sense_key":"english%1:11:00::","synset_id":"english.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"english%1:18:00::","call_ids":["senseval2.d000.s017.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"english%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":700,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00152,"input_uncached_usd":0.0014,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.012728540925309}
{"item_id":"senseval2.d000.s017.t012","gold_sense_keys":["fit%3:00:02::"],"candidates":[{"index":1,"sense_key":"fit%3:00:02::","synset_id":"fit.a.01"},{"index":2,"sense_key":"fit%5:00:00:ready:00","synset_id":"fit.s.01"},{"index":3,"sense_key":"fit%3:00:01::","synset_id":"fit.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"fit%3:00:02::","call_ids":["senseval2.d000.s017.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"fit%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":708,"cached_input_tokens":0,"output_tokens":116,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002576,"input_uncached_usd":0.001416,"input_cached_usd":0.0,"output_usd":0.00116,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6731462499592453}
{"item_id":"senseval2.d000.s017.t013","gold_sense_keys":["only%4:02:01::"],"candidates":[{"index":1,"sense_key":"only%4:02:02::","synset_id":"merely.r.01"},{"index":2,"sense_key":"only%4:02:01::","synset_id":"entirely.r.02"},{"index":3,"sense_key":"only%4:02:00::","synset_id":"only.r.03"},{"index":4,"sense_key":"only%4:02:04::","synset_id":"only.r.04"},{"index":5,"sense_key":"only%4:02:06::","synset_id":"only.r.05"},{"index":6,"sense_key":"only%4:02:03::","synset_id":"only.r.06"},{"index":7,"sense_key":"only%4:02:05::","synset_id":"only.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"only%4:02:01::","call_ids":["senseval2.d000.s017.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"only%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":870,"cached_input_tokens":0,"output_tokens":89,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00263,"input_uncached_usd":0.00174,"input_cached_usd":0.0,"output_usd":0.0008900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8746924160514027}
{"item_id":"senseval2.d000.s017.t014","gold_sense_keys":["foreigner%1:18:00::"],"candidates":[{"index":1,"sense_key":"foreigner%1:18:00::","synset_id":"foreigner.n.01"},{"index":2,"sense_key":"foreigner%1:18:01::","synset_id":"foreigner.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"foreigner%1:18:00::","call_ids":["senseval2.d000.s017.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"foreigner%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":626,"cached_input_tokens":0,"output_tokens":71,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0019619999999999998,"input_uncached_usd":0.0012519999999999999,"input_cached_usd":0.0,"output_usd":0.00071,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3147403749171644}
{"item_id":"senseval2.d000.s018.t002","gold_sense_keys":["english%1:18:00::"],"candidates":[{"index":1,"sense_key":"english%1:10:00::","synset_id":"english.n.01"},{"index":2,"sense_key":"english%1:18:00::","synset_id":"english.n.02"},{"index":3,"sense_key":"english%1:09:00::","synset_id":"english.n.03"},{"index":4,"sense_key":"english%1:11:00::","synset_id":"english.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"english%1:18:00::","call_ids":["senseval2.d000.s018.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"english%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":726,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001572,"input_uncached_usd":0.001452,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9806085840100423}
{"item_id":"senseval2.d000.s018.t003","gold_sense_keys":["invent%2:36:00::"],"candidates":[{"index":1,"sense_key":"invent%2:36:00::","synset_id":"invent.v.01"},{"index":2,"sense_key":"invent%2:36:01::","synset_id":"fabricate.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"invent%2:36:00::","call_ids":["senseval2.d000.s018.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"invent%2:36:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":677,"cached_input_tokens":0,"output_tokens":86,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002214,"input_uncached_usd":0.001354,"input_cached_usd":0.0,"output_usd":0.0008600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7139304580632597}
{"item_id":"senseval2.d000.s018.t004","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["senseval2.d000.s018.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":789,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001698,"input_uncached_usd":0.001578,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1406769580207765}
{"item_id":"senseval2.d000.s018.t006","gold_sense_keys":["require%2:34:00::","require%2:42:00::"],"candidates":[{"index":1,"sense_key":"require%2:42:00::","synset_id":"necessitate.v.01"},{"index":2,"sense_key":"require%2:32:01::","synset_id":"ask.v.04"},{"index":3,"sense_key":"require%2:32:00::","synset_id":"command.v.02"},{"index":4,"sense_key":"require%2:34:00::","synset_id":"want.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"require%2:42:00::","call_ids":["senseval2.d000.s018.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"require%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":765,"cached_input_tokens":0,"output_tokens":76,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00229,"input_uncached_usd":0.00153,"input_cached_usd":0.0,"output_usd":0.00076,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3464111669454724}
{"item_id":"senseval2.d000.s018.t007","gold_sense_keys":["physical%3:00:00::"],"candidates":[{"index":1,"sense_key":"physical%3:00:00::","synset_id":"physical.a.01"},{"index":2,"sense_key":"physical%3:01:00::","synset_id":"physical.a.02"},{"index":3,"sense_key":"physical%5:00:00:material:01","synset_id":"physical.s.01"},{"index":4,"sense_key":"physical%5:00:00:natural:03","synset_id":"physical.s.02"},{"index":5,"sense_key":"physical%5:00:00:energetic:00","synset_id":"physical.s.03"},{"index":6,"sense_key":"physical%5:00:00:forceful:00","synset_id":"forcible.s.01"},{"index":7,"sense_key":"physical%5:00:02:material:01","synset_id":"physical.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"physical%3:00:00::","call_ids":["senseval2.d000.s018.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"physical%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":944,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002008,"input_uncached_usd":0.001888,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.441012250026688}
{"item_id":"senseval2.d000.s018.t009","gold_sense_keys":["bell%1:06:00::"],"candidates":[{"index":1,"sense_key":"bell%1:06:00::","synset_id":"bell.n.01"},{"index":2,"sense_key":"bell%1:06:02::","synset_id":"doorbell.n.01"},{"index":3,"sense_key":"bell%1:11:00::","synset_id":"bell.n.03"},{"index":4,"sense_key":"bell%1:28:00::","synset_id":"bell.n.04"},{"index":5,"sense_key":"bell%1:25:00::","synset_id":"bell.n.05"},{"index":6,"sense_key":"bell%1:18:02::","synset_id":"bell.n.06"},{"index":7,"sense_key":"bell%1:18:01::","synset_id":"bell.n.07"},{"index":8,"sense_key":"bell%1:18:00::","synset_id":"bell.n.08"},{"index":9,"sense_key":"bell%1:06:04::","synset_id":"chime.n.01"},{"index":10,"sense_key":"bell%1:06:03::","synset_id":"bell.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bell%1:06:00::","call_ids":["senseval2.d000.s018.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bell%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1177,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0024739999999999996,"input_uncached_usd":0.0023539999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9117901249555871}
{"item_id":"senseval2.d000.s018.t010","gold_sense_keys":["weigh%2:42:01::"],"candidates":[{"index":1,"sense_key":"weigh%2:42:01::","synset_id":"weigh.v.01"},{"index":2,"sense_key":"weigh%2:32:00::","synset_id":"consider.v.04"},{"index":3,"sense_key":"weigh%2:42:00::","synset_id":"weigh.v.03"},{"index":4,"sense_key":"weigh%2:42:03::","synset_id":"count.v.02"},{"index":5,"sense_key":"weigh%2:42:04::","synset_id":"weigh.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"weigh%2:42:01::","call_ids":["senseval2.d000.s018.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"weigh%2:42:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":804,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001728,"input_uncached_usd":0.0016079999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1364400420570746}
{"item_id":"senseval2.d000.s018.t014","gold_sense_keys":["intense%3:00:00::"],"candidates":[{"index":1,"sense_key":"intense%3:00:00::","synset_id":"intense.a.01"},{"index":2,"sense_key":"intense%5:00:00:sharp:04","synset_id":"acute.s.01"},{"index":3,"sense_key":"intense%5:00:00:saturated:03","synset_id":"intense.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"intense%3:00:00::","call_ids":["senseval2.d000.s018.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"intense%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":710,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0015400000000000001,"input_uncached_usd":0.00142,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.116719999932684}
{"item_id":"senseval2.d000.s018.t015","gold_sense_keys":["mental%3:00:00::","mental%3:01:00::"],"candidates":[{"index":1,"sense_key":"mental%3:00:00::","synset_id":"mental.a.01"},{"index":2,"sense_key":"mental%3:01:00::","synset_id":"mental.a.02"},{"index":3,"sense_key":"mental%3:01:03::","synset_id":"mental.a.03"},{"index":4,"sense_key":"mental%3:01:04::","synset_id":"genial.a.02"},{"index":5,"sense_key":"mental%5:00:00:unhealthy:00","synset_id":"mental.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"mental%3:00:00::","call_ids":["senseval2.d000.s018.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"mental%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":773,"cached_input_tokens":0,"output_tokens":93,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002476,"input_uncached_usd":0.0015459999999999998,"input_cached_usd":0.0,"output_usd":0.00093,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0304262910503894}
{"item_id":"senseval2.d000.s018.t016","gold_sense_keys":["concentration%1:07:01::","concentration%1:09:00::"],"candidates":[{"index":1,"sense_key":"concentration%1:07:02::","synset_id":"concentration.n.01"},{"index":2,"sense_key":"concentration%1:07:03::","synset_id":"concentration.n.02"},{"index":3,"sense_key":"concentration%1:04:01::","synset_id":"concentration.n.03"},{"index":4,"sense_key":"concentration%1:11:00::","synset_id":"concentration.n.04"},{"index":5,"sense_key":"concentration%1:09:00::","synset_id":"concentration.n.05"},{"index":6,"sense_key":"concentration%1:04:02::","synset_id":"concentration.n.06"},{"index":7,"sense_key":"concentration%1:07:01::","synset_id":"assiduity.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"concentration%1:09:00::","call_ids":["senseval2.d000.s018.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"concentration%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":839,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0017980000000000001,"input_uncached_usd":0.001678,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1837412919849157}
{"item_id":"senseval2.d000.s019.t000","gold_sense_keys":["english%3:01:00::"],"candidates":[{"index":1,"sense_key":"english%3:01:00::","synset_id":"english.a.01"},{"index":2,"sense_key":"english%3:01:01::","synset_id":"english.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"english%3:01:00::","call_ids":["senseval2.d000.s019.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"english%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":599,"cached_input_tokens":0,"output_tokens":52,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001718,"input_uncached_usd":0.0011979999999999998,"input_cached_usd":0.0,"output_usd":0.0005200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3696930829901248}
{"item_id":"senseval2.d000.s019.t001","gold_sense_keys":["bell%1:06:00::"],"candidates":[{"index":1,"sense_key":"bell%1:06:00::","synset_id":"bell.n.01"},{"index":2,"sense_key":"bell%1:06:02::","synset_id":"doorbell.n.01"},{"index":3,"sense_key":"bell%1:11:00::","synset_id":"bell.n.03"},{"index":4,"sense_key":"bell%1:28:00::","synset_id":"bell.n.04"},{"index":5,"sense_key":"bell%1:25:00::","synset_id":"bell.n.05"},{"index":6,"sense_key":"bell%1:18:02::","synset_id":"bell.n.06"},{"index":7,"sense_key":"bell%1:18:01::","synset_id":"bell.n.07"},{"index":8,"sense_key":"bell%1:18:00::","synset_id":"bell.n.08"},{"index":9,"sense_key":"bell%1:06:04::","synset_id":"chime.n.01"},{"index":10,"sense_key":"bell%1:06:03::","synset_id":"bell.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bell%1:06:00::","call_ids":["senseval2.d000.s019.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bell%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1163,"cached_input_tokens":0,"output_tokens":51,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002836,"input_uncached_usd":0.002326,"input_cached_usd":0.0,"output_usd":0.00051,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2729149999795482}
{"item_id":"senseval2.d000.s019.t003","gold_sense_keys":["round%1:28:01::"],"candidates":[{"index":1,"sense_key":"round%1:06:01::","synset_id":"round.n.01"},{"index":2,"sense_key":"round%1:28:01::","synset_id":"cycle.n.01"},{"index":3,"sense_key":"round%1:15:00::","synset_id":"beat.n.01"},{"index":4,"sense_key":"round%1:04:00::","synset_id":"round.n.04"},{"index":5,"sense_key":"round%1:04:03::","synset_id":"round_of_golf.n.01"},{"index":6,"sense_key":"round%1:04:01::","synset_id":"round.n.06"},{"index":7,"sense_key":"round%1:28:00::","synset_id":"turn.n.09"},{"index":8,"sense_key":"round%1:17:00::","synset_id":"round.n.08"},{"index":9,"sense_key":"round%1:13:01::","synset_id":"round.n.09"},{"index":10,"sense_key":"round%1:13:00::","synset_id":"round.n.10"},{"index":11,"sense_key":"round%1:10:00::","synset_id":"round.n.11"},{"index":12,"sense_key":"round%1:10:01::","synset_id":"round.n.12"},{"index":13,"sense_key":"round%1:06:03::","synset_id":"rung.n.01"},{"index":14,"sense_key":"round%1:06:00::","synset_id":"circle.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"round%1:10:00::","call_ids":["senseval2.d000.s019.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"round%1:10:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1300,"cached_input_tokens":0,"output_tokens":147,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00407,"input_uncached_usd":0.0026,"input_cached_usd":0.0,"output_usd":0.0014700000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0510230840882286}
{"item_id":"senseval2.d000.s019.t004","gold_sense_keys":["high-pitched%3:00:00::"],"candidates":[{"index":1,"sense_key":"high-pitched%3:00:00::","synset_id":"high.a.04"},{"index":2,"sense_key":"high-pitched%5:00:00:inclined:01","synset_id":"high-pitched.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"high-pitched%3:00:00::","call_ids":["senseval2.d000.s019.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"high-pitched%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":629,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001378,"input_uncached_usd":0.001258,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1889891249593347}
{"item_id":"senseval2.d000.s019.t005","gold_sense_keys":["bell%1:06:00::"],"candidates":[{"index":1,"sense_key":"bell%1:06:00::","synset_id":"bell.n.01"},{"index":2,"sense_key":"bell%1:06:02::","synset_id":"doorbell.n.01"},{"index":3,"sense_key":"bell%1:11:00::","synset_id":"bell.n.03"},{"index":4,"sense_key":"bell%1:28:00::","synset_id":"bell.n.04"},{"index":5,"sense_key":"bell%1:25:00::","synset_id":"bell.n.05"},{"index":6,"sense_key":"bell%1:18:02::","synset_id":"bell.n.06"},{"index":7,"sense_key":"bell%1:18:01::","synset_id":"bell.n.07"},{"index":8,"sense_key":"bell%1:18:00::","synset_id":"bell.n.08"},{"index":9,"sense_key":"bell%1:06:04::","synset_id":"chime.n.01"},{"index":10,"sense_key":"bell%1:06:03::","synset_id":"bell.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bell%1:06:00::","call_ids":["senseval2.d000.s019.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bell%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1163,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002446,"input_uncached_usd":0.002326,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0616854580584913}
{"item_id":"senseval2.d000.s019.t006","gold_sense_keys":["low%3:00:03::"],"candidates":[{"index":1,"sense_key":"low%3:00:02::","synset_id":"low.a.01"},{"index":2,"sense_key":"low%3:00:01::","synset_id":"low.a.02"},{"index":3,"sense_key":"low%5:00:00:soft:04","synset_id":"low.s.01"},{"index":4,"sense_key":"low%5:00:00:unrefined:01","synset_id":"low.s.02"},{"index":5,"sense_key":"low%3:00:03::","synset_id":"low.a.05"},{"index":6,"sense_key":"low%5:00:00:contemptible:00","synset_id":"abject.s.01"},{"index":7,"sense_key":"low%5:00:00:inferior:01","synset_id":"humble.s.01"},{"index":8,"sense_key":"low%5:00:00:insufficient:00","synset_id":"depleted.s.01"},{"index":9,"sense_key":"low%5:00:00:humble:00","synset_id":"broken.s.01"},{"index":10,"sense_key":"low%5:00:00:dejected:00","synset_id":"gloomy.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"low%3:00:03::","call_ids":["senseval2.d000.s019.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"low%3:00:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1150,"cached_input_tokens":0,"output_tokens":83,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00313,"input_uncached_usd":0.0023,"input_cached_usd":0.0,"output_usd":0.0008300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4215689999982715}
{"item_id":"senseval2.d000.s019.t007","gold_sense_keys":["simple%3:00:02::","simple%5:00:00:easy:01"],"candidates":[{"index":1,"sense_key":"simple%3:00:02::","synset_id":"simple.a.01"},{"index":2,"sense_key":"simple%5:00:00:easy:01","synset_id":"elementary.s.01"},{"index":3,"sense_key":"simple%5:00:02:plain:01","synset_id":"bare.s.04"},{"index":4,"sense_key":"simple%5:00:00:naive:00","synset_id":"childlike.s.02"},{"index":5,"sense_key":"simple%5:00:00:retarded:00","synset_id":"dim-witted.s.01"},{"index":6,"sense_key":"simple%3:00:01::","synset_id":"simple.a.06"},{"index":7,"sense_key":"simple%5:00:00:plain:01","synset_id":"simple.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"simple%3:00:02::","call_ids":["senseval2.d000.s019.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"simple%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":991,"cached_input_tokens":0,"output_tokens":101,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002992,"input_uncached_usd":0.001982,"input_cached_usd":0.0,"output_usd":0.00101,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5122463330626488}
{"item_id":"senseval2.d000.s019.t009","gold_sense_keys":["scale%1:10:00::"],"candidates":[{"index":1,"sense_key":"scale%1:24:03::","synset_id":"scale.n.01"},{"index":2,"sense_key":"scale%1:24:00::","synset_id":"scale.n.02"},{"index":3,"sense_key":"scale%1:24:01::","synset_id":"scale.n.03"},{"index":4,"sense_key":"scale%1:20:00::","synset_id":"scale.n.04"},{"index":5,"sense_key":"scale%1:17:00::","synset_id":"scale.n.05"},{"index":6,"sense_key":"scale%1:10:00::","synset_id":"scale.n.06"},{"index":7,"sense_key":"scale%1:06:02::","synset_id":"scale.n.07"},{"index":8,"sense_key":"scale%1:06:00::","synset_id":"scale.n.08"},{"index":9,"sense_key":"scale%1:06:03::","synset_id":"plate.n.14"},{"index":10,"sense_key":"scale%1:05:00::","synset_id":"scale.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"scale%1:10:00::","call_ids":["senseval2.d000.s019.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"scale%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1058,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0022359999999999997,"input_uncached_usd":0.002116,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1355548329884186}
{"item_id":"senseval2.d000.s019.t010","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":2,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":3,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":4,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":5,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":6,"sense_key":"use%2:41:03::","synset_id":"use.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%2:34:01::","call_ids":["senseval2.d000.s019.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":902,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002404,"input_uncached_usd":0.0018039999999999998,"input_cached_usd":0.0,"output_usd":0.0006000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4322783750249073}
{"item_id":"senseval2.d000.s019.t012","gold_sense_keys":["church%1:06:00::"],"candidates":[{"index":1,"sense_key":"church%1:14:00::","synset_id":"church.n.01"},{"index":2,"sense_key":"church%1:06:00::","synset_id":"church.n.02"},{"index":3,"sense_key":"church%1:04:00::","synset_id":"church_service.n.01"},{"index":4,"sense_key":"church%1:14:01::","synset_id":"church.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"church%1:06:00::","call_ids":["senseval2.d000.s019.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"church%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":724,"cached_input_tokens":0,"output_tokens":32,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0017679999999999998,"input_uncached_usd":0.0014479999999999999,"input_cached_usd":0.0,"output_usd":0.00032,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1961772920330986}
{"item_id":"senseval2.d000.s019.t014","gold_sense_keys":["bell%1:06:00::"],"candidates":[{"index":1,"sense_key":"bell%1:06:00::","synset_id":"bell.n.01"},{"index":2,"sense_key":"bell%1:06:02::","synset_id":"doorbell.n.01"},{"index":3,"sense_key":"bell%1:11:00::","synset_id":"bell.n.03"},{"index":4,"sense_key":"bell%1:28:00::","synset_id":"bell.n.04"},{"index":5,"sense_key":"bell%1:25:00::","synset_id":"bell.n.05"},{"index":6,"sense_key":"bell%1:18:02::","synset_id":"bell.n.06"},{"index":7,"sense_key":"bell%1:18:01::","synset_id":"bell.n.07"},{"index":8,"sense_key":"bell%1:18:00::","synset_id":"bell.n.08"},{"index":9,"sense_key":"bell%1:06:04::","synset_id":"chime.n.01"},{"index":10,"sense_key":"bell%1:06:03::","synset_id":"bell.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bell%1:06:00::","call_ids":["senseval2.d000.s019.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bell%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1163,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002916,"input_uncached_usd":0.002326,"input_cached_usd":0.0,"output_usd":0.00059,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.226136166951619}
{"item_id":"senseval2.d000.s020.t000","gold_sense_keys":["then%4:02:00::"],"candidates":[{"index":1,"sense_key":"then%4:02:00::","synset_id":"then.r.01"},{"index":2,"sense_key":"then%4:02:02::","synset_id":"then.r.02"},{"index":3,"sense_key":"then%4:02:01::","synset_id":"then.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"then%4:02:00::","call_ids":["senseval2.d000.s020.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"then%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":688,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001496,"input_uncached_usd":0.0013759999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5635879590408877}
{"item_id":"senseval2.d000.s020.t001","gold_sense_keys":["signal%1:10:00::","signal%1:16:00::"],"candidates":[{"index":1,"sense_key":"signal%1:10:00::","synset_id":"signal.n.01"},{"index":2,"sense_key":"signal%1:16:00::","synset_id":"signal.n.02"},{"index":3,"sense_key":"signal%1:19:00::","synset_id":"signal.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"signal%1:16:00::","call_ids":["senseval2.d000.s020.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"signal%1:16:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":705,"cached_input_tokens":0,"output_tokens":145,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00286,"input_uncached_usd":0.00141,"input_cached_usd":0.0,"output_usd":0.0014500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.115118290996179}
{"item_id":"senseval2.d000.s020.t002","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"},{"index":2,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"},{"index":3,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":4,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s020.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":727,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001574,"input_uncached_usd":0.001454,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8281296249479055}
{"item_id":"senseval2.d000.s020.t003","gold_sense_keys":["begin%2:30:00::"],"candidates":[{"index":1,"sense_key":"begin%2:30:00::","synset_id":"get_down.v.07"},{"index":2,"sense_key":"begin%2:42:00::","synset_id":"begin.v.02"},{"index":3,"sense_key":"begin%2:30:01::","synset_id":"begin.v.03"},{"index":4,"sense_key":"begin%2:32:03::","synset_id":"begin.v.04"},{"index":5,"sense_key":"begin%2:42:02::","synset_id":"begin.v.05"},{"index":6,"sense_key":"begin%2:42:04::","synset_id":"begin.v.06"},{"index":7,"sense_key":"begin%2:42:03::","synset_id":"begin.v.07"},{"index":8,"sense_key":"begin%2:41:04::","synset_id":"begin.v.08"},{"index":9,"sense_key":"begin%2:41:00::","synset_id":"begin.v.09"},{"index":10,"sense_key":"begin%2:32:04::","synset_id":"begin.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"begin%2:30:00::","call_ids":["senseval2.d000.s020.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"begin%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1238,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003086,"input_uncached_usd":0.002476,"input_cached_usd":0.0,"output_usd":0.0006100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2145739999832585}
{"item_id":"senseval2.d000.s020.t004","gold_sense_keys":["vary%2:30:04::"],"candidates":[{"index":1,"sense_key":"vary%2:30:01::","synset_id":"change.v.03"},{"index":2,"sense_key":"vary%2:42:00::","synset_id":"deviate.v.02"},{"index":3,"sense_key":"vary%2:42:01::","synset_id":"vary.v.03"},{"index":4,"sense_key":"vary%2:30:04::","synset_id":"vary.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"vary%2:30:04::","call_ids":["senseval2.d000.s020.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"vary%2:30:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":778,"cached_input_tokens":0,"output_tokens":99,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0025459999999999997,"input_uncached_usd":0.0015559999999999999,"input_cached_usd":0.0,"output_usd":0.00099,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6128325840691105}
{"item_id":"senseval2.d000.s020.t005","gold_sense_keys":["order%1:14:00::"],"candidates":[{"index":1,"sense_key":"order%1:10:03::","synset_id":"order.n.01"},{"index":2,"sense_key":"order%1:07:01::","synset_id":"order.n.02"},{"index":3,"sense_key":"order%1:26:00::","synset_id":"order.n.03"},{"index":4,"sense_key":"order%1:14:00::","synset_id":"ordering.n.01"},{"index":5,"sense_key":"order%1:26:02::","synset_id":"orderliness.n.02"},{"index":6,"sense_key":"order%1:10:00::","synset_id":"decree.n.01"},{"index":7,"sense_key":"order%1:10:01::","synset_id":"order.n.07"},{"index":8,"sense_key":"order%1:14:02::","synset_id":"club.n.02"},{"index":9,"sense_key":"order%1:10:02::","synset_id":"order.n.09"},{"index":10,"sense_key":"order%1:26:01::","synset_id":"holy_order.n.01"},{"index":11,"sense_key":"order%1:14:03::","synset_id":"order.n.11"},{"index":12,"sense_key":"order%1:14:01::","synset_id":"order.n.12"},{"index":13,"sense_key":"order%1:10:04::","synset_id":"order.n.13"},{"index":14,"sense_key":"order%1:07:00::","synset_id":"order.n.14"},{"index":15,"sense_key":"order%1:04:00::","synset_id":"order.n.15"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"order%1:14:00::","call_ids":["senseval2.d000.s020.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"order%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1556,"cached_input_tokens":0,"output_tokens":130,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004412,"input_uncached_usd":0.0031119999999999997,"input_cached_usd":0.0,"output_usd":0.0013000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.959951207973063}
{"item_id":"senseval2.d000.s020.t006","gold_sense_keys":["bell%1:06:00::"],"candidates":[{"index":1,"sense_key":"bell%1:06:00::","synset_id":"bell.n.01"},{"index":2,"sense_key":"bell%1:06:02::","synset_id":"doorbell.n.01"},{"index":3,"sense_key":"bell%1:11:00::","synset_id":"bell.n.03"},{"index":4,"sense_key":"bell%1:28:00::","synset_id":"bell.n.04"},{"index":5,"sense_key":"bell%1:25:00::","synset_id":"bell.n.05"},{"index":6,"sense_key":"bell%1:18:02::","synset_id":"bell.n.06"},{"index":7,"sense_key":"bell%1:18:01::","synset_id":"bell.n.07"},{"index":8,"sense_key":"bell%1:18:00::","synset_id":"bell.n.08"},{"index":9,"sense_key":"bell%1:06:04::","synset_id":"chime.n.01"},{"index":10,"sense_key":"bell%1:06:03::","synset_id":"bell.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bell%1:06:00::","call_ids":["senseval2.d000.s020.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bell%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1171,"cached_input_tokens":0,"output_tokens":98,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0033220000000000003,"input_uncached_usd":0.002342,"input_cached_usd":0.0,"output_usd":0.0009800000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1556468750350177}
{"item_id":"senseval2.d000.s020.t007","gold_sense_keys":["sound%2:39:00::","sound%2:39:03::"],"candidates":[{"index":1,"sense_key":"sound%2:39:06::","synset_id":"sound.v.01"},{"index":2,"sense_key":"sound%2:39:00::","synset_id":"sound.v.02"},{"index":3,"sense_key":"sound%2:39:03::","synset_id":"sound.v.03"},{"index":4,"sense_key":"sound%2:39:02::","synset_id":"sound.v.04"},{"index":5,"sense_key":"sound%2:32:00::","synset_id":"voice.v.02"},{"index":6,"sense_key":"sound%2:39:01::","synset_id":"sound.v.06"},{"index":7,"sense_key":"sound%2:30:00::","synset_id":"fathom.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"sound%2:39:03::","call_ids":["senseval2.d000.s020.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"sound%2:39:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":856,"cached_input_tokens":0,"output_tokens":250,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0042120000000000005,"input_uncached_usd":0.001712,"input_cached_usd":0.0,"output_usd":0.0025,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.161081332946196}
{"item_id":"senseval2.d000.s020.t008","gold_sense_keys":["alter%2:30:01::"],"candidates":[{"index":1,"sense_key":"alter%2:30:01::","synset_id":"change.v.01"},{"index":2,"sense_key":"alter%2:30:00::","synset_id":"change.v.03"},{"index":3,"sense_key":"alter%2:36:00::","synset_id":"alter.v.03"},{"index":4,"sense_key":"alter%2:30:04::","synset_id":"interpolate.v.02"},{"index":5,"sense_key":"alter%2:29:00::","synset_id":"alter.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"alter%2:30:01::","call_ids":["senseval2.d000.s020.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"alter%2:30:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":859,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002288,"input_uncached_usd":0.001718,"input_cached_usd":0.0,"output_usd":0.0005700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4055998330004513}
{"item_id":"senseval2.d000.s020.t009","gold_sense_keys":["steady%3:00:00::"],"candidates":[{"index":1,"sense_key":"steady%3:00:00::","synset_id":"steady.a.01"},{"index":2,"sense_key":"steady%5:00:00:stable:00","synset_id":"firm.s.06"},{"index":3,"sense_key":"steady%5:00:00:secure:03","synset_id":"steady.s.02"},{"index":4,"sense_key":"steady%5:00:00:resolute:00","synset_id":"firm.s.01"},{"index":5,"sense_key":"steady%5:00:00:frequent:00","synset_id":"regular.s.06"},{"index":6,"sense_key":"steady%5:00:00:unexcitable:00","synset_id":"steady.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"steady%3:00:00::","call_ids":["senseval2.d000.s020.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"steady%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":888,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0018960000000000001,"input_uncached_usd":0.001776,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0214830419281498}
{"item_id":"senseval2.d000.s020.t010","gold_sense_keys":["rhythm%1:10:01::"],"candidates":[{"index":1,"sense_key":"rhythm%1:10:01::","synset_id":"rhythm.n.01"},{"index":2,"sense_key":"rhythm%1:07:01::","synset_id":"rhythm.n.02"},{"index":3,"sense_key":"rhythm%1:28:00::","synset_id":"cycle.n.01"},{"index":4,"sense_key":"rhythm%1:10:00::","synset_id":"rhythm.n.04"},{"index":5,"sense_key":"rhythm%1:04:00::","synset_id":"rhythm_method_of_birth_control.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"rhythm%1:07:01::","call_ids":["senseval2.d000.s020.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"rhythm%1:07:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":916,"cached_input_tokens":0,"output_tokens":138,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003212,"input_uncached_usd":0.0018319999999999999,"input_cached_usd":0.0,"output_usd":0.0013800000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8450615829788148}
{"item_id":"senseval2.d000.s020.t011","gold_sense_keys":["striking%1:04:00::"],"candidates":[{"index":1,"sense_key":"striking%1:11:00::","synset_id":"contact.n.04"},{"index":2,"sense_key":"striking%1:04:00::","synset_id":"hit.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"striking%1:04:00::","call_ids":["senseval2.d000.s020.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"striking%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":663,"cached_input_tokens":0,"output_tokens":88,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002206,"input_uncached_usd":0.001326,"input_cached_usd":0.0,"output_usd":0.00088,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3432749169878662}
{"item_id":"senseval2.d000.s021.t000","gold_sense_keys":["variation%1:10:00::"],"candidates":[{"index":1,"sense_key":"variation%1:11:01::","synset_id":"variation.n.01"},{"index":2,"sense_key":"variation%1:04:00::","synset_id":"variation.n.02"},{"index":3,"sense_key":"variation%1:10:00::","synset_id":"variation.n.03"},{"index":4,"sense_key":"variation%1:09:01::","synset_id":"version.n.02"},{"index":5,"sense_key":"variation%1:06:00::","synset_id":"variation.n.05"},{"index":6,"sense_key":"variation%1:25:00::","synset_id":"magnetic_declination.n.01"},{"index":7,"sense_key":"variation%1:22:00::","synset_id":"variation.n.07"},{"index":8,"sense_key":"variation%1:19:00::","synset_id":"variation.n.08"},{"index":9,"sense_key":"variation%1:18:00::","synset_id":"mutant.n.01"},{"index":10,"sense_key":"variation%1:04:02::","synset_id":"pas_seul.n.01"},{"index":11,"sense_key":"variation%1:04:01::","synset_id":"variation.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"variation%1:11:01::","call_ids":["senseval2.d000.s021.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"variation%1:11:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1132,"cached_input_tokens":0,"output_tokens":142,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0036839999999999998,"input_uncached_usd":0.002264,"input_cached_usd":0.0,"output_usd":0.00142,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1353208330692723}
{"item_id":"senseval2.d000.s021.t002","gold_sense_keys":["occur%2:30:00::"],"candidates":[{"index":1,"sense_key":"occur%2:30:00::","synset_id":"happen.v.01"},{"index":2,"sense_key":"occur%2:31:00::","synset_id":"occur.v.02"},{"index":3,"sense_key":"occur%2:42:03::","synset_id":"occur.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"occur%2:30:00::","call_ids":["senseval2.d000.s021.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"occur%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":731,"cached_input_tokens":0,"output_tokens":102,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002482,"input_uncached_usd":0.001462,"input_cached_usd":0.0,"output_usd":0.00102,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6275944169610739}
{"item_id":"senseval2.d000.s021.t003","gold_sense_keys":["only%4:02:02::"],"candidates":[{"index":1,"sense_key":"only%4:02:02::","synset_id":"merely.r.01"},{"index":2,"sense_key":"only%4:02:01::","synset_id":"entirely.r.02"},{"index":3,"sense_key":"only%4:02:00::","synset_id":"only.r.03"},{"index":4,"sense_key":"only%4:02:04::","synset_id":"only.r.04"},{"index":5,"sense_key":"only%4:02:06::","synset_id":"only.r.05"},{"index":6,"sense_key":"only%4:02:03::","synset_id":"only.r.06"},{"index":7,"sense_key":"only%4:02:05::","synset_id":"only.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"only%4:02:02::","call_ids":["senseval2.d000.s021.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"only%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":894,"cached_input_tokens":0,"output_tokens":95,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002738,"input_uncached_usd":0.0017879999999999999,"input_cached_usd":0.0,"output_usd":0.0009500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6587581670610234}
{"item_id":"senseval2.d000.s021.t004","gold_sense_keys":["once%4:02:00::"],"candidates":[{"index":1,"sense_key":"once%4:02:00::","synset_id":"once.r.01"},{"index":2,"sense_key":"once%4:02:02::","synset_id":"once.r.02"},{"index":3,"sense_key":"once%4:02:01::","synset_id":"once.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"once%4:02:00::","call_ids":["senseval2.d000.s021.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"once%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":681,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001942,"input_uncached_usd":0.001362,"input_cached_usd":0.0,"output_usd":0.00058,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1247123329667374}
{"item_id":"senseval2.d000.s021.t005","gold_sense_keys":["rule%1:10:00::"],"candidates":[{"index":1,"sense_key":"rule%1:09:00::","synset_id":"rule.n.01"},{"index":2,"sense_key":"rule%1:09:01::","synset_id":"convention.n.02"},{"index":3,"sense_key":"rule%1:10:00::","synset_id":"rule.n.03"},{"index":4,"sense_key":"rule%1:10:01::","synset_id":"rule.n.04"},{"index":5,"sense_key":"rule%1:09:03::","synset_id":"principle.n.01"},{"index":6,"sense_key":"rule%1:28:00::","synset_id":"rule.n.06"},{"index":7,"sense_key":"rule%1:26:00::","synset_id":"dominion.n.01"},{"index":8,"sense_key":"rule%1:10:02::","synset_id":"rule.n.08"},{"index":9,"sense_key":"rule%1:10:03::","synset_id":"rule.n.09"},{"index":10,"sense_key":"rule%1:09:02::","synset_id":"principle.n.04"},{"index":11,"sense_key":"rule%1:09:04::","synset_id":"rule.n.11"},{"index":12,"sense_key":"rule%1:06:00::","synset_id":"rule.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"rule%1:10:00::","call_ids":["senseval2.d000.s021.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"rule%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1344,"cached_input_tokens":0,"output_tokens":179,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004478,"input_uncached_usd":0.002688,"input_cached_usd":0.0,"output_usd":0.0017900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0226470000343397}
{"item_id":"senseval2.d000.s021.t006","gold_sense_keys":["state%2:32:00::"],"candidates":[{"index":1,"sense_key":"state%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"state%2:32:01::","synset_id":"submit.v.02"},{"index":3,"sense_key":"state%2:32:02::","synset_id":"express.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"state%2:32:00::","call_ids":["senseval2.d000.s021.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"state%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":701,"cached_input_tokens":0,"output_tokens":128,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0026820000000000004,"input_uncached_usd":0.001402,"input_cached_usd":0.0,"output_usd":0.00128,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.37704854097683}
{"item_id":"senseval2.d000.s022.t000","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"},{"index":2,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"},{"index":3,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":4,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s022.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":739,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001598,"input_uncached_usd":0.001478,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0120707920286804}
{"item_id":"senseval2.d000.s022.t005","gold_sense_keys":["method%1:09:00::"],"candidates":[{"index":1,"sense_key":"method%1:09:00::","synset_id":"method.n.01"},{"index":2,"sense_key":"method%1:04:00::","synset_id":"method_acting.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"method%1:09:00::","call_ids":["senseval2.d000.s022.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"method%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":672,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001464,"input_uncached_usd":0.001344,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.458554792101495}
{"item_id":"senseval2.d000.s022.t006","gold_sense_keys":["name%1:10:00::"],"candidates":[{"index":1,"sense_key":"name%1:10:00::","synset_id":"name.n.01"},{"index":2,"sense_key":"name%1:26:00::","synset_id":"name.n.02"},{"index":3,"sense_key":"name%1:14:00::","synset_id":"name.n.03"},{"index":4,"sense_key":"name%1:18:00::","synset_id":"name.n.04"},{"index":5,"sense_key":"name%1:04:00::","synset_id":"name.n.05"},{"index":6,"sense_key":"name%1:10:01::","synset_id":"name.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"name%1:10:00::","call_ids":["senseval2.d000.s022.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"name%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":840,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0018,"input_uncached_usd":0.0016799999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0608354590367526}
{"item_id":"senseval2.d000.s023.t000","gold_sense_keys":["series%1:14:00::"],"candidates":[{"index":1,"sense_key":"series%1:14:00::","synset_id":"series.n.01"},{"index":2,"sense_key":"series%1:10:01::","synset_id":"serial.n.01"},{"index":3,"sense_key":"series%1:10:00::","synset_id":"series.n.03"},{"index":4,"sense_key":"series%1:11:00::","synset_id":"series.n.04"},{"index":5,"sense_key":"series%1:24:00::","synset_id":"series.n.05"},{"index":6,"sense_key":"series%1:14:01::","synset_id":"series.n.06"},{"index":7,"sense_key":"series%1:09:00::","synset_id":"series.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"series%1:14:00::","call_ids":["senseval2.d000.s023.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"series%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":942,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0020039999999999997,"input_uncached_usd":0.0018839999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0210553749930114}
{"item_id":"senseval2.d000.s023.t003","gold_sense_keys":["take%2:40:06::"],"candidates":[{"index":1,"sense_key":"take%2:41:04::","synset_id":"take.v.01"},{"index":2,"sense_key":"take%2:40:06::","synset_id":"take.v.02"},{"index":3,"sense_key":"take%2:38:09::","synset_id":"lead.v.01"},{"index":4,"sense_key":"take%2:35:00::","synset_id":"take.v.04"},{"index":5,"sense_key":"take%2:30:01::","synset_id":"assume.v.03"},{"index":6,"sense_key":"take%2:31:07::","synset_id":"take.v.06"},{"index":7,"sense_key":"take%2:38:10::","synset_id":"bring.v.01"},{"index":8,"sense_key":"take%2:40:00::","synset_id":"take.v.08"},{"index":9,"sense_key":"take%2:38:02::","synset_id":"take.v.09"},{"index":10,"sense_key":"take%2:31:01::","synset_id":"choose.v.01"},{"index":11,"sense_key":"take%2:40:05::","synset_id":"accept.v.02"},{"index":12,"sense_key":"take%2:41:00::","synset_id":"fill.v.04"},{"index":13,"sense_key":"take%2:31:03::","synset_id":"consider.v.03"},{"index":14,"sense_key":"take%2:42:00::","synset_id":"necessitate.v.01"},{"index":15,"sense_key":"take%2:39:00::","synset_id":"take.v.15"},{"index":16,"sense_key":"take%2:32:02::","synset_id":"film.v.01"},{"index":17,"sense_key":"take%2:30:00::","synset_id":"remove.v.01"},{"index":18,"sense_key":"take%2:34:00::","synset_id":"consume.v.02"},{"index":19,"sense_key":"take%2:31:09::","synset_id":"take.v.19"},{"index":20,"sense_key":"take%2:40:15::","synset_id":"take.v.20"},{"index":21,"sense_key":"take%2:40:01::","synset_id":"take.v.21"},{"index":22,"sense_key":"take%2:38:00::","synset_id":"assume.v.05"},{"index":23,"sense_key":"take%2:40:07::","synset_id":"accept.v.05"},{"index":24,"sense_key":"take%2:30:08::","synset_id":"take.v.24"},{"index":25,"sense_key":"take%2:31:02::","synset_id":"learn.v.04"},{"index":26,"sense_key":"take%2:32:00::","synset_id":"claim.v.05"},{"index":27,"sense_key":"take%2:38:05::","synset_id":"take.v.27"},{"index":28,"sense_key":"take%2:33:09::","synset_id":"aim.v.01"},{"index":29,"sense_key":"take%2:30:09::","synset_id":"take.v.29"},{"index":30,"sense_key":"take%2:42:10::","synset_id":"carry.v.02"},{"index":31,"sense_key":"take%2:40:03::","synset_id":"lease.v.04"},{"index":32,"sense_key":"take%2:40:04::","synset_id":"subscribe.v.05"},{"index":33,"sense_key":"take%2:40:02::","synset_id":"take.v.33"},{"index":34,"sense_key":"take%2:40:08::","synset_id":"take.v.34"},{"index":35,"sense_key":"take%2:35:14::","synset_id":"take.v.35"},{"index":36,"sense_key":"take%2:32:09::","synset_id":"claim.v.04"},{"index":37,"sense_key":"take%2:42:03::","synset_id":"accept.v.08"},{"index":38,"sense_key":"take%2:42:15::","synset_id":"contain.v.05"},{"index":39,"sense_key":"take%2:41:13::","synset_id":"take.v.39"},{"index":40,"sense_key":"take%2:38:11::","synset_id":"drive.v.16"},{"index":41,"sense_key":"take%2:33:08::","synset_id":"take.v.41"},{"index":42,"sense_key":"take%2:29:08::","synset_id":"contract.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"take%2:40:06::","call_ids":["senseval2.d000.s023.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"take%2:40:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":3130,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00638,"input_uncached_usd":0.00626,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4996632499387488}
{"item_id":"senseval2.d000.s023.t004","gold_sense_keys":["about%4:02:00::"],"candidates":[{"index":1,"sense_key":"about%4:02:00::","synset_id":"approximately.r.01"},{"index":2,"sense_key":"about%4:02:06::","synset_id":"about.r.02"},{"index":3,"sense_key":"about%4:02:01::","synset_id":"about.r.03"},{"index":4,"sense_key":"about%4:02:04::","synset_id":"about.r.04"},{"index":5,"sense_key":"about%4:02:02::","synset_id":"about.r.05"},{"index":6,"sense_key":"about%4:02:03::","synset_id":"about.r.06"},{"index":7,"sense_key":"about%4:02:05::","synset_id":"about.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"about%4:02:00::","call_ids":["senseval2.d000.s023.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"about%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":978,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0020759999999999997,"input_uncached_usd":0.001956,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1265762500697747}
{"item_id":"senseval2.d000.s023.t005","gold_sense_keys":["hour%1:28:00::"],"candidates":[{"index":1,"sense_key":"hour%1:28:00::","synset_id":"hour.n.01"},{"index":2,"sense_key":"hour%1:28:01::","synset_id":"hour.n.02"},{"index":3,"sense_key":"hour%1:28:02::","synset_id":"hour.n.03"},{"index":4,"sense_key":"hour%1:07:00::","synset_id":"hour.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"hour%1:28:00::","call_ids":["senseval2.d000.s023.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"hour%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":745,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00161,"input_uncached_usd":0.00149,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1806797080207616}
{"item_id":"senseval2.d000.s024.t000","gold_sense_keys":["look%1:04:00::"],"candidates":[{"index":1,"sense_key":"look%1:07:01::","synset_id":"expression.n.01"},{"index":2,"sense_key":"look%1:04:00::","synset_id":"look.n.02"},{"index":3,"sense_key":"look%1:07:00::","synset_id":"look.n.03"},{"index":4,"sense_key":"look%1:26:00::","synset_id":"spirit.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"look%1:04:00::","call_ids":["senseval2.d000.s024.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"look%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":822,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001764,"input_uncached_usd":0.0016439999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.786127292085439}
{"item_id":"senseval2.d000.s024.t002","gold_sense_keys":["night%1:28:03::"],"candidates":[{"index":1,"sense_key":"night%1:28:00::","synset_id":"night.n.01"},{"index":2,"sense_key":"night%1:28:01::","synset_id":"night.n.02"},{"index":3,"sense_key":"night%1:28:04::","synset_id":"night.n.03"},{"index":4,"sense_key":"night%1:28:02::","synset_id":"night.n.04"},{"index":5,"sense_key":"night%1:26:00::","synset_id":"night.n.05"},{"index":6,"sense_key":"night%1:28:05::","synset_id":"night.n.06"},{"index":7,"sense_key":"night%1:28:03::","synset_id":"night.n.07"},{"index":8,"sense_key":"night%1:18:00::","synset_id":"nox.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"night%1:28:03::","call_ids":["senseval2.d000.s024.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"night%1:28:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":899,"cached_input_tokens":0,"output_tokens":150,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0032979999999999997,"input_uncached_usd":0.001798,"input_cached_usd":0.0,"output_usd":0.0015,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7508959580445662}
{"item_id":"senseval2.d000.s024.t003","gold_sense_keys":["practice%1:04:02::"],"candidates":[{"index":1,"sense_key":"practice%1:04:00::","synset_id":"practice.n.01"},{"index":2,"sense_key":"practice%1:04:02::","synset_id":"exercise.n.03"},{"index":3,"sense_key":"practice%1:04:04::","synset_id":"practice.n.03"},{"index":4,"sense_key":"practice%1:04:03::","synset_id":"practice.n.04"},{"index":5,"sense_key":"practice%1:09:00::","synset_id":"practice.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"practice%1:04:02::","call_ids":["senseval2.d000.s024.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"practice%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":844,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001808,"input_uncached_usd":0.0016879999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4102327080909163}
{"item_id":"senseval2.d000.s024.t004","gold_sense_keys":["church%1:06:00::"],"candidates":[{"index":1,"sense_key":"church%1:14:00::","synset_id":"church.n.01"},{"index":2,"sense_key":"church%1:06:00::","synset_id":"church.n.02"},{"index":3,"sense_key":"church%1:04:00::","synset_id":"church_service.n.01"},{"index":4,"sense_key":"church%1:14:01::","synset_id":"church.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"church%1:06:00::","call_ids":["senseval2.d000.s024.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"church%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":747,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001614,"input_uncached_usd":0.0014939999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.7824835409410298}
{"item_id":"senseval2.d000.s024.t006","gold_sense_keys":["give%2:32:08::","give%2:40:03::","give%2:40:05::"],"candidates":[{"index":1,"sense_key":"give%2:40:03::","synset_id":"give.v.01"},{"index":2,"sense_key":"give%2:40:05::","synset_id":"yield.v.01"},{"index":3,"sense_key":"give%2:40:00::","synset_id":"give.v.03"},{"index":4,"sense_key":"give%2:40:07::","synset_id":"give.v.04"},{"index":5,"sense_key":"give%2:32:01::","synset_id":"give.v.05"},{"index":6,"sense_key":"give%2:36:02::","synset_id":"hold.v.03"},{"index":7,"sense_key":"give%2:32:00::","synset_id":"give.v.07"},{"index":8,"sense_key":"give%2:40:01::","synset_id":"give.v.08"},{"index":9,"sense_key":"give%2:36:03::","synset_id":"give.v.09"},{"index":10,"sense_key":"give%2:31:02::","synset_id":"give.v.10"},{"index":11,"sense_key":"give%2:36:00::","synset_id":"render.v.04"},{"index":12,"sense_key":"give%2:40:09::","synset_id":"impart.v.01"},{"index":13,"sense_key":"give%2:36:01::","synset_id":"establish.v.05"},{"index":14,"sense_key":"give%2:40:02::","synset_id":"give.v.14"},{"index":15,"sense_key":"give%2:41:00::","synset_id":"give.v.15"},{"index":16,"sense_key":"give%2:40:10::","synset_id":"sacrifice.v.01"},{"index":17,"sense_key":"give%2:40:06::","synset_id":"pass.v.05"},{"index":18,"sense_key":"give%2:32:03::","synset_id":"give.v.18"},{"index":19,"sense_key":"give%2:40:13::","synset_id":"give.v.19"},{"index":20,"sense_key":"give%2:40:14::","synset_id":"give.v.20"},{"index":21,"sense_key":"give%2:32:02::","synset_id":"give.v.21"},{"index":22,"sense_key":"give%2:40:11::","synset_id":"grant.v.05"},{"index":23,"sense_key":"give%2:38:00::","synset_id":"move_over.v.01"},{"index":24,"sense_key":"give%2:34:00::","synset_id":"feed.v.02"},{"index":25,"sense_key":"give%2:40:08::","synset_id":"contribute.v.02"},{"index":26,"sense_key":"give%2:38:13::","synset_id":"collapse.v.01"},{"index":27,"sense_key":"give%2:31:00::","synset_id":"give.v.27"},{"index":28,"sense_key":"give%2:41:10::","synset_id":"give.v.28"},{"index":29,"sense_key":"give%2:40:12::","synset_id":"give.v.29"},{"index":30,"sense_key":"give%2:40:15::","synset_id":"afford.v.04"},{"index":31,"sense_key":"give%2:36:13::","synset_id":"give.v.31"},{"index":32,"sense_key":"give%2:36:12::","synset_id":"give.v.32"},{"index":33,"sense_key":"give%2:35:00::","synset_id":"give.v.33"},{"index":34,"sense_key":"give%2:34:10::","synset_id":"give.v.34"},{"index":35,"sense_key":"give%2:32:11::","synset_id":"give.v.35"},{"index":36,"sense_key":"give%2:32:08::","synset_id":"give.v.36"},{"index":37,"sense_key":"give%2:32:15::","synset_id":"give.v.37"},{"index":38,"sense_key":"give%2:32:14::","synset_id":"give.v.38"},{"index":39,"sense_key":"give%2:32:09::","synset_id":"give.v.39"},{"index":40,"sense_key":"give%2:32:07::","synset_id":"give.v.40"},{"index":41,"sense_key":"give%2:32:10::","synset_id":"give.v.41"},{"index":42,"sense_key":"give%2:30:13::","synset_id":"give.v.42"},{"index":43,"sense_key":"give%2:29:11::","synset_id":"give.v.43"},{"index":44,"sense_key":"give%2:29:10::","synset_id":"give.v.44"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"give%2:40:07::","call_ids":["senseval2.d000.s024.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"give%2:40:07::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":2940,"cached_input_tokens":0,"output_tokens":120,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0070799999999999995,"input_uncached_usd":0.00588,"input_cached_usd":0.0,"output_usd":0.0012000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2260405418928713}
{"item_id":"senseval2.d000.s024.t007","gold_sense_keys":["idea%1:09:02::"],"candidates":[{"index":1,"sense_key":"idea%1:09:00::","synset_id":"idea.n.01"},{"index":2,"sense_key":"idea%1:09:01::","synset_id":"mind.n.06"},{"index":3,"sense_key":"idea%1:09:03::","synset_id":"idea.n.03"},{"index":4,"sense_key":"idea%1:09:02::","synset_id":"estimate.n.01"},{"index":5,"sense_key":"idea%1:10:00::","synset_id":"theme.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"idea%1:09:02::","call_ids":["senseval2.d000.s024.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"idea%1:09:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":891,"cached_input_tokens":0,"output_tokens":142,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003202,"input_uncached_usd":0.001782,"input_cached_usd":0.0,"output_usd":0.00142,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7777626250172034}
{"item_id":"senseval2.d000.s024.t008","gold_sense_keys":["work%1:04:00::"],"candidates":[{"index":1,"sense_key":"work%1:04:00::","synset_id":"work.n.01"},{"index":2,"sense_key":"work%1:06:00::","synset_id":"work.n.02"},{"index":3,"sense_key":"work%1:04:01::","synset_id":"employment.n.02"},{"index":4,"sense_key":"work%1:09:00::","synset_id":"study.n.02"},{"index":5,"sense_key":"work%1:19:00::","synset_id":"work.n.05"},{"index":6,"sense_key":"work%1:06:01::","synset_id":"workplace.n.01"},{"index":7,"sense_key":"work%1:06:02::","synset_id":"oeuvre.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"work%1:04:00::","call_ids":["senseval2.d000.s024.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"work%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1041,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002762,"input_uncached_usd":0.002082,"input_cached_usd":0.0,"output_usd":0.00068,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0255196670768782}
{"item_id":"senseval2.d000.s024.t009","gold_sense_keys":["involved%3:00:00::"],"candidates":[{"index":1,"sense_key":"involved%3:00:00::","synset_id":"involved.a.01"},{"index":2,"sense_key":"involved%5:00:00:encumbered:00","synset_id":"involved.s.01"},{"index":3,"sense_key":"involved%5:00:00:attached:01","synset_id":"involved.s.02"},{"index":4,"sense_key":"involved%5:00:00:complex:00","synset_id":"byzantine.s.01"},{"index":5,"sense_key":"involved%5:00:00:enclosed:00","synset_id":"involved.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"involved%3:00:00::","call_ids":["senseval2.d000.s024.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"involved%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":903,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001926,"input_uncached_usd":0.0018059999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4743231660686433}
{"item_id":"senseval2.d000.s025.t000","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"},{"index":2,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"},{"index":3,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":4,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s025.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":728,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001576,"input_uncached_usd":0.0014559999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5342789590358734}
{"item_id":"senseval2.d000.s025.t001","gold_sense_keys":["stand%2:35:00::"],"candidates":[{"index":1,"sense_key":"stand%2:35:00::","synset_id":"stand.v.01"},{"index":2,"sense_key":"stand%2:42:04::","synset_id":"stand.v.02"},{"index":3,"sense_key":"stand%2:42:03::","synset_id":"stand.v.03"},{"index":4,"sense_key":"stand%2:42:00::","synset_id":"stand.v.04"},{"index":5,"sense_key":"stand%2:31:00::","synset_id":"digest.v.03"},{"index":6,"sense_key":"stand%2:31:01::","synset_id":"stand.v.06"},{"index":7,"sense_key":"stand%2:42:06::","synset_id":"stand.v.07"},{"index":8,"sense_key":"stand%2:42:02::","synset_id":"stand.v.08"},{"index":9,"sense_key":"stand%2:42:01::","synset_id":"stand.v.09"},{"index":10,"sense_key":"stand%2:35:01::","synset_id":"stand.v.10"},{"index":11,"sense_key":"stand%2:33:00::","synset_id":"resist.v.04"},{"index":12,"sense_key":"stand%2:35:10::","synset_id":"stand.v.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"stand%2:35:00::","call_ids":["senseval2.d000.s025.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"stand%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1179,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0024779999999999997,"input_uncached_usd":0.002358,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5101610419806093}
{"item_id":"senseval2.d000.s025.t002","gold_sense_keys":["circle%1:25:01::"],"candidates":[{"index":1,"sense_key":"circle%1:25:00::","synset_id":"circle.n.01"},{"index":2,"sense_key":"circle%1:14:00::","synset_id":"set.n.05"},{"index":3,"sense_key":"circle%1:25:01::","synset_id":"circle.n.03"},{"index":4,"sense_key":"circle%1:04:00::","synset_id":"lap.n.05"},{"index":5,"sense_key":"circle%1:06:01::","synset_id":"traffic_circle.n.01"},{"index":6,"sense_key":"circle%1:06:03::","synset_id":"r-2.n.01"},{"index":7,"sense_key":"circle%1:06:02::","synset_id":"circle.n.07"},{"index":8,"sense_key":"circle%1:06:00::","synset_id":"circle.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"circle%1:25:01::","call_ids":["senseval2.d000.s025.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"circle%1:25:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1052,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002224,"input_uncached_usd":0.002104,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8384146670578048}
{"item_id":"senseval2.d000.s025.t003","gold_sense_keys":["foot%1:08:01::"],"candidates":[{"index":1,"sense_key":"foot%1:08:01::","synset_id":"foot.n.01"},{"index":2,"sense_key":"foot%1:23:00::","synset_id":"foot.n.02"},{"index":3,"sense_key":"foot%1:15:00::","synset_id":"foot.n.03"},{"index":4,"sense_key":"foot%1:05:01::","synset_id":"animal_foot.n.01"},{"index":5,"sense_key":"foot%1:06:00::","synset_id":"foundation.n.03"},{"index":6,"sense_key":"foot%1:05:00::","synset_id":"foot.n.06"},{"index":7,"sense_key":"foot%1:04:00::","synset_id":"foot.n.07"},{"index":8,"sense_key":"foot%1:18:00::","synset_id":"foot.n.08"},{"index":9,"sense_key":"foot%1:14:00::","synset_id":"infantry.n.01"},{"index":10,"sense_key":"foot%1:10:00::","synset_id":"metrical_foot.n.01"},{"index":11,"sense_key":"foot%1:06:01::","synset_id":"foot.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"foot%1:08:01::","call_ids":["senseval2.d000.s025.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"foot%1:08:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1192,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0025039999999999997,"input_uncached_usd":0.002384,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8174130829283968}
{"item_id":"senseval2.d000.s025.t004","gold_sense_keys":["ahead%4:02:00::"],"candidates":[{"index":1,"sense_key":"ahead%4:02:00::","synset_id":"ahead.r.01"},{"index":2,"sense_key":"ahead%4:02:06::","synset_id":"ahead.r.02"},{"index":3,"sense_key":"ahead%4:02:02::","synset_id":"ahead.r.03"},{"index":4,"sense_key":"ahead%4:02:01::","synset_id":"ahead.r.04"},{"index":5,"sense_key":"ahead%4:02:05::","synset_id":"ahead.r.05"},{"index":6,"sense_key":"ahead%4:02:04::","synset_id":"ahead.r.06"},{"index":7,"sense_key":"ahead%4:02:03::","synset_id":"ahead.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ahead%4:02:00::","call_ids":["senseval2.d000.s025.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ahead%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1047,"cached_input_tokens":0,"output_tokens":97,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003064,"input_uncached_usd":0.002094,"input_cached_usd":0.0,"output_usd":0.00097,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5282842500600964}
{"item_id":"senseval2.d000.s025.t006","gold_sense_keys":["stance%1:07:00::"],"candidates":[{"index":1,"sense_key":"stance%1:07:00::","synset_id":"stance.n.01"},{"index":2,"sense_key":"stance%1:09:00::","synset_id":"position.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"stance%1:07:00::","call_ids":["senseval2.d000.s025.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"stance%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":594,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0013080000000000001,"input_uncached_usd":0.001188,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9896612080046907}
{"item_id":"senseval2.d000.s025.t007","gold_sense_keys":["pull%2:35:04::"],"candidates":[{"index":1,"sense_key":"pull%2:35:00::","synset_id":"pull.v.01"},{"index":2,"sense_key":"pull%2:35:02::","synset_id":"attract.v.01"},{"index":3,"sense_key":"pull%2:38:01::","synset_id":"pull.v.03"},{"index":4,"sense_key":"pull%2:35:04::","synset_id":"pull.v.04"},{"index":5,"sense_key":"pull%2:41:00::","synset_id":"perpetrate.v.01"},{"index":6,"sense_key":"pull%2:38:03::","synset_id":"draw.v.05"},{"index":7,"sense_key":"pull%2:38:00::","synset_id":"pull.v.07"},{"index":8,"sense_key":"pull%2:29:00::","synset_id":"pull.v.08"},{"index":9,"sense_key":"pull%2:38:13::","synset_id":"pull.v.09"},{"index":10,"sense_key":"pull%2:38:02::","synset_id":"pull.v.10"},{"index":11,"sense_key":"pull%2:38:04::","synset_id":"pull.v.11"},{"index":12,"sense_key":"pull%2:35:01::","synset_id":"rend.v.01"},{"index":13,"sense_key":"pull%2:35:03::","synset_id":"pull.v.13"},{"index":14,"sense_key":"pull%2:35:05::","synset_id":"pluck.v.05"},{"index":15,"sense_key":"pull%2:35:10::","synset_id":"extract.v.01"},{"index":16,"sense_key":"pull%2:33:13::","synset_id":"pull.v.16"},{"index":17,"sense_key":"pull%2:30:11::","synset_id":"pull.v.17"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"pull%2:35:04::","call_ids":["senseval2.d000.s025.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"pull%2:35:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1615,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0038799999999999998,"input_uncached_usd":0.00323,"input_cached_usd":0.0,"output_usd":0.0006500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2361696669831872}
{"item_id":"senseval2.d000.s025.t008","gold_sense_keys":["rope%1:06:00::"],"candidates":[{"index":1,"sense_key":"rope%1:06:00::","synset_id":"rope.n.01"},{"index":2,"sense_key":"rope%1:06:01::","synset_id":"r-2.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"rope%1:06:00::","call_ids":["senseval2.d000.s025.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"rope%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":617,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001354,"input_uncached_usd":0.0012339999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9534415000816807}
{"item_id":"senseval2.d000.s025.t009","gold_sense_keys":["small%3:00:00::"],"candidates":[{"index":1,"sense_key":"small%3:00:00::","synset_id":"small.a.01"},{"index":2,"sense_key":"small%5:00:00:limited:00","synset_id":"minor.s.03"},{"index":3,"sense_key":"small%5:00:00:young:00","synset_id":"little.s.01"},{"index":4,"sense_key":"small%5:00:00:little:03","synset_id":"small.s.03"},{"index":5,"sense_key":"small%5:00:00:inferior:01","synset_id":"humble.s.01"},{"index":6,"sense_key":"small%5:00:00:lowercase:00","synset_id":"little.s.04"},{"index":7,"sense_key":"small%5:00:00:soft:04","synset_id":"little.s.03"},{"index":8,"sense_key":"small%5:00:00:fine:00","synset_id":"small.s.07"},{"index":9,"sense_key":"small%5:00:00:moderate:00","synset_id":"modest.s.01"},{"index":10,"sense_key":"small%5:00:00:decreased:00","synset_id":"belittled.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"small%3:00:00::","call_ids":["senseval2.d000.s025.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"small%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1132,"cached_input_tokens":0,"output_tokens":117,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003434,"input_uncached_usd":0.002264,"input_cached_usd":0.0,"output_usd":0.00117,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.393767041969113}
{"item_id":"senseval2.d000.s025.t010","gold_sense_keys":["hole%1:06:00::","hole%1:17:01::"],"candidates":[{"index":1,"sense_key":"hole%1:17:01::","synset_id":"hole.n.01"},{"index":2,"sense_key":"hole%1:06:00::","synset_id":"hole.n.02"},{"index":3,"sense_key":"hole%1:06:01::","synset_id":"hole.n.03"},{"index":4,"sense_key":"hole%1:25:00::","synset_id":"hole.n.04"},{"index":5,"sense_key":"hole%1:17:02::","synset_id":"hole.n.05"},{"index":6,"sense_key":"hole%1:26:01::","synset_id":"hole.n.06"},{"index":7,"sense_key":"hole%1:26:00::","synset_id":"fix.n.01"},{"index":8,"sense_key":"hole%1:08:00::","synset_id":"trap.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"hole%1:06:00::","call_ids":["senseval2.d000.s025.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"hole%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":899,"cached_input_tokens":0,"output_tokens":226,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004058,"input_uncached_usd":0.001798,"input_cached_usd":0.0,"output_usd":0.0022600000000000003,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.773096999968402}
{"item_id":"senseval2.d000.s025.t011","gold_sense_keys":["high%3:00:01::"],"candidates":[{"index":1,"sense_key":"high%3:00:02::","synset_id":"high.a.01"},{"index":2,"sense_key":"high%3:00:01::","synset_id":"high.a.02"},{"index":3,"sense_key":"high%5:00:00:superior:01","synset_id":"eminent.s.01"},{"index":4,"sense_key":"high%3:00:03::","synset_id":"high.a.04"},{"index":5,"sense_key":"high%5:00:00:elated:00","synset_id":"high.s.02"},{"index":6,"sense_key":"high%5:00:00:malodorous:00","synset_id":"gamey.s.02"},{"index":7,"sense_key":"high%5:00:00:intoxicated:00","synset_id":"high.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"high%3:00:01::","call_ids":["senseval2.d000.s025.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"high%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":933,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001986,"input_uncached_usd":0.0018659999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9946398329921067}
{"item_id":"senseval2.d000.s025.t012","gold_sense_keys":["ceiling%1:06:00::"],"candidates":[{"index":1,"sense_key":"ceiling%1:06:00::","synset_id":"ceiling.n.01"},{"index":2,"sense_key":"ceiling%1:07:00::","synset_id":"ceiling.n.02"},{"index":3,"sense_key":"ceiling%1:10:00::","synset_id":"ceiling.n.03"},{"index":4,"sense_key":"ceiling%1:07:01::","synset_id":"ceiling.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ceiling%1:06:00::","call_ids":["senseval2.d000.s025.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ceiling%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":748,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001616,"input_uncached_usd":0.001496,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.107410000055097}
{"item_id":"senseval2.d000.s025.t013","gold_sense_keys":["chamber%1:06:00::"],"candidates":[{"index":1,"sense_key":"chamber%1:06:00::","synset_id":"chamber.n.01"},{"index":2,"sense_key":"chamber%1:08:00::","synset_id":"chamber.n.02"},{"index":3,"sense_key":"chamber%1:06:02::","synset_id":"chamber.n.03"},{"index":4,"sense_key":"chamber%1:14:00::","synset_id":"chamber.n.04"},{"index":5,"sense_key":"chamber%1:06:01::","synset_id":"bedroom.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"chamber%1:06:00::","call_ids":["senseval2.d000.s025.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"chamber%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":767,"cached_input_tokens":0,"output_tokens":109,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002624,"input_uncached_usd":0.001534,"input_cached_usd":0.0,"output_usd":0.00109,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6271435839589685}
{"item_id":"senseval2.d000.s026.t000","gold_sense_keys":["speak%2:32:02::"],"candidates":[{"index":1,"sense_key":"speak%2:32:00::","synset_id":"talk.v.02"},{"index":2,"sense_key":"speak%2:32:01::","synset_id":"talk.v.01"},{"index":3,"sense_key":"speak%2:32:02::","synset_id":"speak.v.03"},{"index":4,"sense_key":"speak%2:32:03::","synset_id":"address.v.02"},{"index":5,"sense_key":"speak%2:39:00::","synset_id":"speak.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"speak%2:32:02::","call_ids":["senseval2.d000.s026.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"speak%2:32:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":828,"cached_input_tokens":0,"output_tokens":107,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002726,"input_uncached_usd":0.001656,"input_cached_usd":0.0,"output_usd":0.00107,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9747935420600697}
{"item_id":"senseval2.d000.s026.t002","gold_sense_keys":["rope%1:06:00::"],"candidates":[{"index":1,"sense_key":"rope%1:06:00::","synset_id":"rope.n.01"},{"index":2,"sense_key":"rope%1:06:01::","synset_id":"r-2.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"rope%1:06:00::","call_ids":["senseval2.d000.s026.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"rope%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":634,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0013880000000000001,"input_uncached_usd":0.001268,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5629052500007674}
{"item_id":"senseval2.d000.s026.t003","gold_sense_keys":["seem%2:39:00::"],"candidates":[{"index":1,"sense_key":"seem%2:39:00::","synset_id":"look.v.02"},{"index":2,"sense_key":"seem%2:39:01::","synset_id":"appear.v.04"},{"index":3,"sense_key":"seem%2:42:00::","synset_id":"seem.v.03"},{"index":4,"sense_key":"seem%2:39:02::","synset_id":"seem.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"seem%2:39:00::","call_ids":["senseval2.d000.s026.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"seem%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":801,"cached_input_tokens":0,"output_tokens":150,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0031019999999999997,"input_uncached_usd":0.001602,"input_cached_usd":0.0,"output_usd":0.0015,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.142010874929838}
{"item_id":"senseval2.d000.s026.t004","gold_sense_keys":["make%2:36:08::"],"candidates":[{"index":1,"sense_key":"make%2:41:00::","synset_id":"make.v.01"},{"index":2,"sense_key":"make%2:30:00::","synset_id":"make.v.02"},{"index":3,"sense_key":"make%2:36:00::","synset_id":"make.v.03"},{"index":4,"sense_key":"make%2:32:00::","synset_id":"induce.v.02"},{"index":5,"sense_key":"make%2:36:08::","synset_id":"cause.v.01"},{"index":6,"sense_key":"make%2:36:01::","synset_id":"produce.v.02"},{"index":7,"sense_key":"make%2:31:00::","synset_id":"draw.v.04"},{"index":8,"sense_key":"make%2:36:15::","synset_id":"make.v.08"},{"index":9,"sense_key":"make%2:36:09::","synset_id":"create.v.05"},{"index":10,"sense_key":"make%2:40:01::","synset_id":"gain.v.08"},{"index":11,"sense_key":"make%2:36:05::","synset_id":"do.v.08"},{"index":12,"sense_key":"make%2:42:05::","synset_id":"form.v.02"},{"index":13,"sense_key":"make%2:38:02::","synset_id":"reach.v.07"},{"index":14,"sense_key":"make%2:42:00::","synset_id":"make.v.14"},{"index":15,"sense_key":"make%2:36:11::","synset_id":"make.v.15"},{"index":16,"sense_key":"make%2:36:12::","synset_id":"make.v.16"},{"index":17,"sense_key":"make%2:36:13::","synset_id":"construct.v.01"},{"index":18,"sense_key":"make%2:30:15::","synset_id":"make.v.18"},{"index":19,"sense_key":"make%2:29:08::","synset_id":"make.v.19"},{"index":20,"sense_key":"make%2:41:03::","synset_id":"name.v.03"},{"index":21,"sense_key":"make%2:40:02::","synset_id":"have.v.17"},{"index":22,"sense_key":"make%2:38:00::","synset_id":"reach.v.01"},{"index":23,"sense_key":"make%2:31:13::","synset_id":"lay_down.v.01"},{"index":24,"sense_key":"make%2:41:13::","synset_id":"make.v.24"},{"index":25,"sense_key":"make%2:41:01::","synset_id":"make.v.25"},{"index":26,"sense_key":"make%2:36:04::","synset_id":"hold.v.03"},{"index":27,"sense_key":"make%2:30:02::","synset_id":"make.v.27"},{"index":28,"sense_key":"make%2:38:05::","synset_id":"take.v.27"},{"index":29,"sense_key":"make%2:29:00::","synset_id":"stool.v.04"},{"index":30,"sense_key":"make%2:42:09::","synset_id":"make.v.30"},{"index":31,"sense_key":"make%2:42:13::","synset_id":"make.v.31"},{"index":32,"sense_key":"make%2:42:12::","synset_id":"make.v.32"},{"index":33,"sense_key":"make%2:42:11::","synset_id":"make.v.33"},{"index":34,"sense_key":"make%2:42:15::","synset_id":"make.v.34"},{"index":35,"sense_key":"make%2:39:12::","synset_id":"make.v.35"},{"index":36,"sense_key":"make%2:38:10::","synset_id":"make.v.36"},{"index":37,"sense_key":"make%2:38:15::","synset_id":"make.v.37"},{"index":38,"sense_key":"make%2:36:10::","synset_id":"make.v.38"},{"index":39,"sense_key":"make%2:36:07::","synset_id":"cook.v.02"},{"index":40,"sense_key":"make%2:35:13::","synset_id":"seduce.v.01"},{"index":41,"sense_key":"make%2:32:14::","synset_id":"make.v.41"},{"index":42,"sense_key":"make%2:32:09::","synset_id":"make.v.42"},{"index":43,"sense_key":"make%2:31:10::","synset_id":"make.v.43"},{"index":44,"sense_key":"make%2:31:12::","synset_id":"make.v.44"},{"index":45,"sense_key":"make%2:30:10::","synset_id":"make.v.45"},{"index":46,"sense_key":"make%2:30:14::","synset_id":"make.v.46"},{"index":47,"sense_key":"make%2:30:12::","synset_id":"make.v.47"},{"index":48,"sense_key":"make%2:29:06::","synset_id":"make.v.48"},{"index":49,"sense_key":"make%2:29:01::","synset_id":"make.v.49"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"make%2:36:08::","call_ids":["senseval2.d000.s026.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"make%2:36:08::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":3207,"cached_input_tokens":0,"output_tokens":131,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.007724,"input_uncached_usd":0.0064139999999999996,"input_cached_usd":0.0,"output_usd":0.0013100000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.360862208995968}
{"item_id":"senseval2.d000.s026.t006","gold_sense_keys":["sound%1:07:00::"],"candidates":[{"index":1,"sense_key":"sound%1:07:00::","synset_id":"sound.n.01"},{"index":2,"sense_key":"sound%1:09:00::","synset_id":"sound.n.02"},{"index":3,"sense_key":"sound%1:19:00::","synset_id":"sound.n.03"},{"index":4,"sense_key":"sound%1:11:00::","synset_id":"sound.n.04"},{"index":5,"sense_key":"sound%1:10:00::","synset_id":"audio.n.01"},{"index":6,"sense_key":"sound%1:10:01::","synset_id":"phone.n.02"},{"index":7,"sense_key":"sound%1:17:01::","synset_id":"strait.n.01"},{"index":8,"sense_key":"sound%1:17:00::","synset_id":"sound.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"sound%1:07:00::","call_ids":["senseval2.d000.s026.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"sound%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1010,"cached_input_tokens":0,"output_tokens":139,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0034100000000000003,"input_uncached_usd":0.00202,"input_cached_usd":0.0,"output_usd":0.0013900000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9611663749674335}
{"item_id":"senseval2.d000.s026.t007","gold_sense_keys":["bell%1:06:00::"],"candidates":[{"index":1,"sense_key":"bell%1:06:00::","synset_id":"bell.n.01"},{"index":2,"sense_key":"bell%1:06:02::","synset_id":"doorbell.n.01"},{"index":3,"sense_key":"bell%1:11:00::","synset_id":"bell.n.03"},{"index":4,"sense_key":"bell%1:28:00::","synset_id":"bell.n.04"},{"index":5,"sense_key":"bell%1:25:00::","synset_id":"bell.n.05"},{"index":6,"sense_key":"bell%1:18:02::","synset_id":"bell.n.06"},{"index":7,"sense_key":"bell%1:18:01::","synset_id":"bell.n.07"},{"index":8,"sense_key":"bell%1:18:00::","synset_id":"bell.n.08"},{"index":9,"sense_key":"bell%1:06:04::","synset_id":"chime.n.01"},{"index":10,"sense_key":"bell%1:06:03::","synset_id":"bell.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bell%1:06:00::","call_ids":["senseval2.d000.s026.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bell%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1188,"cached_input_tokens":0,"output_tokens":93,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003306,"input_uncached_usd":0.002376,"input_cached_usd":0.0,"output_usd":0.00093,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1029278329806402}
{"item_id":"senseval2.d000.s026.t008","gold_sense_keys":["muffle%2:39:00::"],"candidates":[{"index":1,"sense_key":"muffle%2:35:00::","synset_id":"smother.v.03"},{"index":2,"sense_key":"muffle%2:39:00::","synset_id":"muffle.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"muffle%2:39:00::","call_ids":["senseval2.d000.s026.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"muffle%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":673,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001466,"input_uncached_usd":0.001346,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.16059225006029}
{"item_id":"senseval2.d000.s026.t009","gold_sense_keys":["ceiling%1:06:00::"],"candidates":[{"index":1,"sense_key":"ceiling%1:06:00::","synset_id":"ceiling.n.01"},{"index":2,"sense_key":"ceiling%1:07:00::","synset_id":"ceiling.n.02"},{"index":3,"sense_key":"ceiling%1:10:00::","synset_id":"ceiling.n.03"},{"index":4,"sense_key":"ceiling%1:07:01::","synset_id":"ceiling.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ceiling%1:06:00::","call_ids":["senseval2.d000.s026.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ceiling%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":764,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001648,"input_uncached_usd":0.0015279999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.146227083983831}
{"item_id":"senseval2.d000.s027.t001","gold_sense_keys":["absorbed%5:00:00:attentive:00"],"candidates":[{"index":1,"sense_key":"absorbed%5:00:00:attentive:00","synset_id":"captive.s.02"},{"index":2,"sense_key":"absorbed%5:00:00:unreflected:00","synset_id":"absorbed.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"absorbed%5:00:00:attentive:00","call_ids":["senseval2.d000.s027.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"absorbed%5:00:00:attentive:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":749,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001618,"input_uncached_usd":0.001498,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0796359169762582}
{"item_id":"senseval2.d000.s027.t002","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"},{"index":2,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"},{"index":3,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":4,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s027.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":791,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001702,"input_uncached_usd":0.0015819999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0633131250506267}
{"item_id":"senseval2.d000.s027.t003","gold_sense_keys":["stare%2:29:00::"],"candidates":[{"index":1,"sense_key":"stare%2:39:00::","synset_id":"gaze.v.01"},{"index":2,"sense_key":"stare%2:29:00::","synset_id":"stare.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"stare%2:29:00::","call_ids":["senseval2.d000.s027.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"stare%2:29:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":697,"cached_input_tokens":0,"output_tokens":134,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002734,"input_uncached_usd":0.0013939999999999998,"input_cached_usd":0.0,"output_usd":0.00134,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9223568339366466}
{"item_id":"senseval2.d000.s027.t004","gold_sense_keys":["straight%4:02:05::"],"candidates":[{"index":1,"sense_key":"straight%4:02:05::","synset_id":"directly.r.01"},{"index":2,"sense_key":"straight%4:02:04::","synset_id":"directly.r.04"},{"index":3,"sense_key":"straight%4:02:00::","synset_id":"straight.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"straight%4:02:00::","call_ids":["senseval2.d000.s027.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"straight%4:02:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":756,"cached_input_tokens":0,"output_tokens":171,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003222,"input_uncached_usd":0.0015119999999999999,"input_cached_usd":0.0,"output_usd":0.0017100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9450090000173077}
{"item_id":"senseval2.d000.s027.t005","gold_sense_keys":["ahead%4:02:00::"],"candidates":[{"index":1,"sense_key":"ahead%4:02:00::","synset_id":"ahead.r.01"},{"index":2,"sense_key":"ahead%4:02:06::","synset_id":"ahead.r.02"},{"index":3,"sense_key":"ahead%4:02:02::","synset_id":"ahead.r.03"},{"index":4,"sense_key":"ahead%4:02:01::","synset_id":"ahead.r.04"},{"index":5,"sense_key":"ahead%4:02:05::","synset_id":"ahead.r.05"},{"index":6,"sense_key":"ahead%4:02:04::","synset_id":"ahead.r.06"},{"index":7,"sense_key":"ahead%4:02:03::","synset_id":"ahead.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ahead%4:02:00::","call_ids":["senseval2.d000.s027.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ahead%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1110,"cached_input_tokens":0,"output_tokens":110,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00332,"input_uncached_usd":0.0022199999999999998,"input_cached_usd":0.0,"output_usd":0.0011,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1809258749708533}
{"item_id":"senseval2.d000.s027.t006","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":2,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":3,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":4,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":5,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":6,"sense_key":"use%2:41:03::","synset_id":"use.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%2:34:01::","call_ids":["senseval2.d000.s027.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":974,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002068,"input_uncached_usd":0.0019479999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1454302909551188}
{"item_id":"senseval2.d000.s027.t008","gold_sense_keys":["call%2:32:02::"],"candidates":[{"index":1,"sense_key":"call%2:32:02::","synset_id":"name.v.01"},{"index":2,"sense_key":"call%2:32:00::","synset_id":"call.v.02"},{"index":3,"sense_key":"call%2:32:01::","synset_id":"call.v.03"},{"index":4,"sense_key":"call%2:32:09::","synset_id":"shout.v.02"},{"index":5,"sense_key":"call%2:32:05::","synset_id":"call.v.05"},{"index":6,"sense_key":"call%2:41:00::","synset_id":"visit.v.03"},{"index":7,"sense_key":"call%2:41:04::","synset_id":"call.v.07"},{"index":8,"sense_key":"call%2:32:13::","synset_id":"call.v.08"},{"index":9,"sense_key":"call%2:32:04::","synset_id":"call.v.09"},{"index":10,"sense_key":"call%2:32:03::","synset_id":"call.v.10"},{"index":11,"sense_key":"call%2:42:09::","synset_id":"call.v.11"},{"index":12,"sense_key":"call%2:41:14::","synset_id":"address.v.06"},{"index":13,"sense_key":"call%2:41:12::","synset_id":"call.v.13"},{"index":14,"sense_key":"call%2:40:09::","synset_id":"call.v.14"},{"index":15,"sense_key":"call%2:40:05::","synset_id":"bid.v.04"},{"index":16,"sense_key":"call%2:38:00::","synset_id":"call.v.16"},{"index":17,"sense_key":"call%2:32:06::","synset_id":"call.v.17"},{"index":18,"sense_key":"call%2:32:10::","synset_id":"predict.v.01"},{"index":19,"sense_key":"call%2:32:12::","synset_id":"call.v.19"},{"index":20,"sense_key":"call%2:32:07::","synset_id":"call.v.20"},{"index":21,"sense_key":"call%2:32:08::","synset_id":"call.v.21"},{"index":22,"sense_key":"call%2:32:11::","synset_id":"call.v.22"},{"index":23,"sense_key":"call%2:32:15::","synset_id":"call.v.23"},{"index":24,"sense_key":"call%2:32:14::","synset_id":"call.v.24"},{"index":25,"sense_key":"call%2:31:13::","synset_id":"call.v.25"},{"index":26,"sense_key":"call%2:31:00::","synset_id":"call.v.26"},{"index":27,"sense_key":"call%2:31:05::","synset_id":"call.v.27"},{"index":28,"sense_key":"call%2:29:05::","synset_id":"call.v.28"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"call%2:32:00::","call_ids":["senseval2.d000.s027.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"call%2:32:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":2210,"cached_input_tokens":0,"output_tokens":226,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00668,"input_uncached_usd":0.0044199999999999995,"input_cached_usd":0.0,"output_usd":0.0022600000000000003,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.535610208986327}
{"item_id":"senseval2.d000.s027.t009","gold_sense_keys":["watch%2:39:00::","watch%2:39:02::","watch%2:41:00::"],"candidates":[{"index":1,"sense_key":"watch%2:39:00::","synset_id":"watch.v.01"},{"index":2,"sense_key":"watch%2:41:00::","synset_id":"watch.v.02"},{"index":3,"sense_key":"watch%2:39:05::","synset_id":"watch.v.03"},{"index":4,"sense_key":"watch%2:39:02::","synset_id":"watch.v.04"},{"index":5,"sense_key":"watch%2:39:01::","synset_id":"watch.v.05"},{"index":6,"sense_key":"watch%2:39:03::","synset_id":"watch.v.06"},{"index":7,"sense_key":"watch%2:32:09::","synset_id":"determine.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"watch%2:39:00::","call_ids":["senseval2.d000.s027.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"watch%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1026,"cached_input_tokens":0,"output_tokens":203,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0040820000000000006,"input_uncached_usd":0.002052,"input_cached_usd":0.0,"output_usd":0.00203,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.5161247910000384}
{"item_id":"senseval2.d000.s027.t010","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":2,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":3,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"other%3:00:00::","call_ids":["senseval2.d000.s027.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":822,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001764,"input_uncached_usd":0.0016439999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1752040829742327}
{"item_id":"senseval2.d000.s027.t011","gold_sense_keys":["rope%1:06:00::"],"candidates":[{"index":1,"sense_key":"rope%1:06:00::","synset_id":"rope.n.01"},{"index":2,"sense_key":"rope%1:06:01::","synset_id":"r-2.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"rope%1:06:00::","call_ids":["senseval2.d000.s027.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"rope%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":681,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001482,"input_uncached_usd":0.001362,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1253815840464085}
{"item_id":"senseval2.d000.s027.t012","gold_sense_keys":["thus%4:02:00::"],"candidates":[{"index":1,"sense_key":"thus%4:02:00::","synset_id":"therefore.r.01"},{"index":2,"sense_key":"thus%4:02:01::","synset_id":"thus.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"thus%4:02:00::","call_ids":["senseval2.d000.s027.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"thus%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":760,"cached_input_tokens":0,"output_tokens":69,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00221,"input_uncached_usd":0.0015199999999999999,"input_cached_usd":0.0,"output_usd":0.0006900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3568183749448508}
{"item_id":"senseval2.d000.s027.t013","gold_sense_keys":["time%2:30:06::"],"candidates":[{"index":1,"sense_key":"time%2:30:00::","synset_id":"clock.v.01"},{"index":2,"sense_key":"time%2:31:00::","synset_id":"time.v.02"},{"index":3,"sense_key":"time%2:31:01::","synset_id":"time.v.03"},{"index":4,"sense_key":"time%2:30:01::","synset_id":"time.v.04"},{"index":5,"sense_key":"time%2:30:06::","synset_id":"time.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"time%2:30:06::","call_ids":["senseval2.d000.s027.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"time%2:30:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":863,"cached_input_tokens":0,"output_tokens":82,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002546,"input_uncached_usd":0.0017259999999999999,"input_cached_usd":0.0,"output_usd":0.0008200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5888759999070317}
{"item_id":"senseval2.d000.s027.t014","gold_sense_keys":["pull%1:04:00::"],"candidates":[{"index":1,"sense_key":"pull%1:04:00::","synset_id":"pull.n.01"},{"index":2,"sense_key":"pull%1:19:00::","synset_id":"pull.n.02"},{"index":3,"sense_key":"pull%1:07:00::","synset_id":"pull.n.03"},{"index":4,"sense_key":"pull%1:06:01::","synset_id":"pull.n.04"},{"index":5,"sense_key":"pull%1:26:00::","synset_id":"wrench.n.01"},{"index":6,"sense_key":"pull%1:04:01::","synset_id":"puff.n.07"},{"index":7,"sense_key":"pull%1:04:02::","synset_id":"pull.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"pull%1:04:00::","call_ids":["senseval2.d000.s027.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"pull%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1035,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0026699999999999996,"input_uncached_usd":0.00207,"input_cached_usd":0.0,"output_usd":0.0006000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7385247501078993}
{"item_id":"senseval2.d000.s028.t000","gold_sense_keys":["far%4:02:00::"],"candidates":[{"index":1,"sense_key":"far%4:02:02::","synset_id":"far.r.01"},{"index":2,"sense_key":"far%4:02:00::","synset_id":"far.r.02"},{"index":3,"sense_key":"far%4:02:04::","synset_id":"far.r.03"},{"index":4,"sense_key":"far%4:02:01::","synset_id":"far.r.04"},{"index":5,"sense_key":"far%4:02:05::","synset_id":"far.r.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"far%4:02:00::","call_ids":["senseval2.d000.s028.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"far%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":827,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002424,"input_uncached_usd":0.0016539999999999999,"input_cached_usd":0.0,"output_usd":0.0007700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3686480410397053}
{"item_id":"senseval2.d000.s028.t001","gold_sense_keys":["above%4:02:01::"],"candidates":[{"index":1,"sense_key":"above%4:02:00::","synset_id":"above.r.01"},{"index":2,"sense_key":"above%4:02:01::","synset_id":"above.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"above%4:02:01::","call_ids":["senseval2.d000.s028.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"above%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":666,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001452,"input_uncached_usd":0.0013319999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9689690410159528}
{"item_id":"senseval2.d000.s028.t004","gold_sense_keys":["bronze%5:00:00:metallic:00"],"candidates":[{"index":1,"sense_key":"bronze%5:00:00:chromatic:00","synset_id":"bronze.s.01"},{"index":2,"sense_key":"bronze%5:00:00:metallic:00","synset_id":"bronze.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bronze%5:00:00:metallic:00","call_ids":["senseval2.d000.s028.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bronze%5:00:00:metallic:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":662,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001444,"input_uncached_usd":0.0013239999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9775958749232814}
{"item_id":"senseval2.d000.s028.t005","gold_sense_keys":["bell%1:06:00::"],"candidates":[{"index":1,"sense_key":"bell%1:06:00::","synset_id":"bell.n.01"},{"index":2,"sense_key":"bell%1:06:02::","synset_id":"doorbell.n.01"},{"index":3,"sense_key":"bell%1:11:00::","synset_id":"bell.n.03"},{"index":4,"sense_key":"bell%1:28:00::","synset_id":"bell.n.04"},{"index":5,"sense_key":"bell%1:25:00::","synset_id":"bell.n.05"},{"index":6,"sense_key":"bell%1:18:02::","synset_id":"bell.n.06"},{"index":7,"sense_key":"bell%1:18:01::","synset_id":"bell.n.07"},{"index":8,"sense_key":"bell%1:18:00::","synset_id":"bell.n.08"},{"index":9,"sense_key":"bell%1:06:04::","synset_id":"chime.n.01"},{"index":10,"sense_key":"bell%1:06:03::","synset_id":"bell.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bell%1:06:00::","call_ids":["senseval2.d000.s028.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bell%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1222,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002564,"input_uncached_usd":0.002444,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.866888332995586}
{"item_id":"senseval2.d000.s028.t006","gold_sense_keys":["mount%2:35:00::"],"candidates":[{"index":1,"sense_key":"mount%2:35:00::","synset_id":"mount.v.01"},{"index":2,"sense_key":"mount%2:30:00::","synset_id":"wax.v.02"},{"index":3,"sense_key":"mount%2:30:01::","synset_id":"mount.v.03"},{"index":4,"sense_key":"mount%2:36:00::","synset_id":"mount.v.04"},{"index":5,"sense_key":"mount%2:38:01::","synset_id":"hop_on.v.01"},{"index":6,"sense_key":"mount%2:38:00::","synset_id":"climb.v.01"},{"index":7,"sense_key":"mount%2:36:01::","synset_id":"mount.v.07"},{"index":8,"sense_key":"mount%2:35:07::","synset_id":"ride.v.14"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"mount%2:35:00::","call_ids":["senseval2.d000.s028.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"mount%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1020,"cached_input_tokens":0,"output_tokens":133,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0033699999999999997,"input_uncached_usd":0.0020399999999999997,"input_cached_usd":0.0,"output_usd":0.00133,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7114032909739763}
{"item_id":"senseval2.d000.s028.t007","gold_sense_keys":["wheel%1:06:00::"],"candidates":[{"index":1,"sense_key":"wheel%1:06:00::","synset_id":"wheel.n.01"},{"index":2,"sense_key":"wheel%1:06:05::","synset_id":"steering_wheel.n.01"},{"index":3,"sense_key":"wheel%1:07:00::","synset_id":"wheel.n.03"},{"index":4,"sense_key":"wheel%1:06:01::","synset_id":"wheel.n.04"},{"index":5,"sense_key":"wheel%1:06:03::","synset_id":"roulette_wheel.n.01"},{"index":6,"sense_key":"wheel%1:06:02::","synset_id":"rack.n.04"},{"index":7,"sense_key":"wheel%1:06:06::","synset_id":"bicycle.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"wheel%1:06:00::","call_ids":["senseval2.d000.s028.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"wheel%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":989,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0020979999999999996,"input_uncached_usd":0.0019779999999999997,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1316661250311881}
{"item_id":"senseval2.d000.s028.t008","gold_sense_keys":["swing%2:38:03::"],"candidates":[{"index":1,"sense_key":"swing%2:38:02::","synset_id":"swing.v.01"},{"index":2,"sense_key":"swing%2:38:00::","synset_id":"swing.v.02"},{"index":3,"sense_key":"swing%2:38:03::","synset_id":"swing.v.03"},{"index":4,"sense_key":"swing%2:41:01::","synset_id":"swing.v.04"},{"index":5,"sense_key":"swing%2:35:00::","synset_id":"swing.v.05"},{"index":6,"sense_key":"swing%2:42:00::","synset_id":"dangle.v.01"},{"index":7,"sense_key":"swing%2:33:00::","synset_id":"swing.v.07"},{"index":8,"sense_key":"swing%2:30:08::","synset_id":"swing.v.08"},{"index":9,"sense_key":"swing%2:42:01::","synset_id":"swing.v.09"},{"index":10,"sense_key":"swing%2:42:10::","synset_id":"swing.v.10"},{"index":11,"sense_key":"swing%2:41:00::","synset_id":"swing.v.11"},{"index":12,"sense_key":"swing%2:36:03::","synset_id":"swing.v.12"},{"index":13,"sense_key":"swing%2:35:08::","synset_id":"swing.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"swing%2:38:00::","call_ids":["senseval2.d000.s028.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"swing%2:38:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1257,"cached_input_tokens":0,"output_tokens":174,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004254,"input_uncached_usd":0.0025139999999999997,"input_cached_usd":0.0,"output_usd":0.0017400000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.7778131660306826}
{"item_id":"senseval2.d000.s028.t009","gold_sense_keys":["madly%4:02:02::"],"candidates":[{"index":1,"sense_key":"madly%4:02:02::","synset_id":"madly.r.01"},{"index":2,"sense_key":"madly%4:02:01::","synset_id":"insanely.r.01"},{"index":3,"sense_key":"madly%4:02:00::","synset_id":"madly.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"madly%4:02:02::","call_ids":["senseval2.d000.s028.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"madly%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":770,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00166,"input_uncached_usd":0.00154,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.909301667008549}
{"item_id":"senseval2.d000.s028.t010","gold_sense_keys":["full%5:00:01:whole:00"],"candidates":[{"index":1,"sense_key":"full%3:00:00::","synset_id":"full.a.01"},{"index":2,"sense_key":"full%5:00:01:whole:00","synset_id":"entire.s.01"},{"index":3,"sense_key":"full%5:00:00:complete:00","synset_id":"full.s.02"},{"index":4,"sense_key":"full%5:00:00:nourished:00","synset_id":"full.s.03"},{"index":5,"sense_key":"full%3:00:01::","synset_id":"full.a.05"},{"index":6,"sense_key":"full%5:00:00:ample:00","synset_id":"full.s.04"},{"index":7,"sense_key":"full%5:00:00:high:02","synset_id":"broad.s.04"},{"index":8,"sense_key":"full%5:00:02:ample:00","synset_id":"wide.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"full%5:00:01:whole:00","call_ids":["senseval2.d000.s028.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"full%5:00:01:whole:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1033,"cached_input_tokens":0,"output_tokens":105,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003116,"input_uncached_usd":0.0020659999999999997,"input_cached_usd":0.0,"output_usd":0.0010500000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6490853329887614}
{"item_id":"senseval2.d000.s028.t011","gold_sense_keys":["degree%1:23:00::"],"candidates":[{"index":1,"sense_key":"degree%1:07:00::","synset_id":"degree.n.01"},{"index":2,"sense_key":"degree%1:26:01::","synset_id":"degree.n.02"},{"index":3,"sense_key":"degree%1:10:00::","synset_id":"academic_degree.n.01"},{"index":4,"sense_key":"degree%1:23:00::","synset_id":"degree.n.04"},{"index":5,"sense_key":"degree%1:09:00::","synset_id":"degree.n.05"},{"index":6,"sense_key":"degree%1:23:03::","synset_id":"degree.n.06"},{"index":7,"sense_key":"degree%1:07:01::","synset_id":"degree.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"degree%1:23:00::","call_ids":["senseval2.d000.s028.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"degree%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1020,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0021599999999999996,"input_uncached_usd":0.0020399999999999997,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1501545829232782}
{"item_id":"senseval2.d000.s028.t012","gold_sense_keys":["start%2:42:00::"],"candidates":[{"index":1,"sense_key":"start%2:30:00::","synset_id":"get_down.v.07"},{"index":2,"sense_key":"start%2:30:01::","synset_id":"begin.v.03"},{"index":3,"sense_key":"start%2:38:02::","synset_id":"depart.v.03"},{"index":4,"sense_key":"start%2:42:00::","synset_id":"begin.v.02"},{"index":5,"sense_key":"start%2:36:00::","synset_id":"originate.v.02"},{"index":6,"sense_key":"start%2:36:01::","synset_id":"start.v.06"},{"index":7,"sense_key":"start%2:38:04::","synset_id":"startle.v.02"},{"index":8,"sense_key":"start%2:38:01::","synset_id":"start.v.08"},{"index":9,"sense_key":"start%2:38:00::","synset_id":"start.v.09"},{"index":10,"sense_key":"start%2:41:00::","synset_id":"start.v.10"},{"index":11,"sense_key":"start%2:33:00::","synset_id":"start.v.11"},{"index":12,"sense_key":"start%2:42:03::","synset_id":"begin.v.07"},{"index":13,"sense_key":"start%2:41:04::","synset_id":"begin.v.08"},{"index":14,"sense_key":"start%2:38:03::","synset_id":"start.v.14"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"start%2:42:00::","call_ids":["senseval2.d000.s028.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"start%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1460,"cached_input_tokens":0,"output_tokens":139,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00431,"input_uncached_usd":0.00292,"input_cached_usd":0.0,"output_usd":0.0013900000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2985384999774396}
{"item_id":"senseval2.d000.s028.t013","gold_sense_keys":["end%2:42:00::"],"candidates":[{"index":1,"sense_key":"end%2:42:00::","synset_id":"end.v.01"},{"index":2,"sense_key":"end%2:30:01::","synset_id":"end.v.02"},{"index":3,"sense_key":"end%2:42:01::","synset_id":"end.v.03"},{"index":4,"sense_key":"end%2:36:13::","synset_id":"end.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"end%2:42:00::","call_ids":["senseval2.d000.s028.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"end%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":877,"cached_input_tokens":0,"output_tokens":177,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0035240000000000002,"input_uncached_usd":0.001754,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0934837920358405}
{"item_id":"senseval2.d000.s028.t015","gold_sense_keys":["inverted%5:00:00:turned:00"],"candidates":[{"index":1,"sense_key":"inverted%5:00:00:turned:00","synset_id":"inverted.s.01"},{"index":2,"sense_key":"inverted%3:00:04::","synset_id":"anatropous.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"inverted%5:00:00:turned:00","call_ids":["senseval2.d000.s028.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"inverted%5:00:00:turned:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":722,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001564,"input_uncached_usd":0.001444,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0853271660162136}
{"item_id":"senseval2.d000.s028.t016","gold_sense_keys":["position%1:07:00::"],"candidates":[{"index":1,"sense_key":"position%1:15:00::","synset_id":"position.n.01"},{"index":2,"sense_key":"position%1:15:02::","synset_id":"military_position.n.01"},{"index":3,"sense_key":"position%1:09:00::","synset_id":"position.n.03"},{"index":4,"sense_key":"position%1:07:01::","synset_id":"position.n.04"},{"index":5,"sense_key":"position%1:26:00::","synset_id":"status.n.01"},{"index":6,"sense_key":"position%1:04:01::","synset_id":"position.n.06"},{"index":7,"sense_key":"position%1:07:00::","synset_id":"position.n.07"},{"index":8,"sense_key":"position%1:15:03::","synset_id":"position.n.08"},{"index":9,"sense_key":"position%1:04:02::","synset_id":"position.n.09"},{"index":10,"sense_key":"position%1:04:00::","synset_id":"placement.n.03"},{"index":11,"sense_key":"position%1:26:02::","synset_id":"situation.n.02"},{"index":12,"sense_key":"position%1:09:01::","synset_id":"position.n.12"},{"index":13,"sense_key":"position%1:10:00::","synset_id":"side.n.10"},{"index":14,"sense_key":"position%1:10:01::","synset_id":"place.n.15"},{"index":15,"sense_key":"position%1:04:03::","synset_id":"stead.n.01"},{"index":16,"sense_key":"position%1:04:04::","synset_id":"position.n.16"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"position%1:07:00::","call_ids":["senseval2.d000.s028.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"position%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1501,"cached_input_tokens":0,"output_tokens":128,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004282,"input_uncached_usd":0.003002,"input_cached_usd":0.0,"output_usd":0.00128,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.888918707962148}
{"item_id":"senseval2.d000.s029.t001","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"},{"index":2,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"},{"index":3,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":4,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s029.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":818,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001756,"input_uncached_usd":0.0016359999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.019995708949864}
{"item_id":"senseval2.d000.s029.t002","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":2,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":3,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":4,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":5,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":6,"sense_key":"use%2:41:03::","synset_id":"use.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%2:34:01::","call_ids":["senseval2.d000.s029.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1001,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0021219999999999998,"input_uncached_usd":0.002002,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.555493792053312}
{"item_id":"senseval2.d000.s029.t005","gold_sense_keys":["retard%2:30:01::"],"candidates":[{"index":1,"sense_key":"retard%2:30:01::","synset_id":"retard.v.01"},{"index":2,"sense_key":"retard%2:42:00::","synset_id":"retard.v.02"},{"index":3,"sense_key":"retard%2:30:07::","synset_id":"check.v.07"},{"index":4,"sense_key":"retard%2:30:00::","synset_id":"decelerate.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"retard%2:30:01::","call_ids":["senseval2.d000.s029.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"retard%2:30:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":817,"cached_input_tokens":0,"output_tokens":84,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0024739999999999996,"input_uncached_usd":0.0016339999999999998,"input_cached_usd":0.0,"output_usd":0.00084,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.782982417033054}
{"item_id":"senseval2.d000.s029.t006","gold_sense_keys":["swing%1:04:00::"],"candidates":[{"index":1,"sense_key":"swing%1:26:01::","synset_id":"swing.n.01"},{"index":2,"sense_key":"swing%1:06:00::","synset_id":"swing.n.02"},{"index":3,"sense_key":"swing%1:04:06::","synset_id":"swing.n.03"},{"index":4,"sense_key":"swing%1:04:00::","synset_id":"swing.n.04"},{"index":5,"sense_key":"swing%1:10:01::","synset_id":"swing.n.05"},{"index":6,"sense_key":"swing%1:07:00::","synset_id":"lilt.n.01"},{"index":7,"sense_key":"swing%1:04:03::","synset_id":"golf_stroke.n.01"},{"index":8,"sense_key":"swing%1:04:02::","synset_id":"baseball_swing.n.01"},{"index":9,"sense_key":"swing%1:04:04::","synset_id":"swing.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"swing%1:04:00::","call_ids":["senseval2.d000.s029.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"swing%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1116,"cached_input_tokens":0,"output_tokens":69,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002922,"input_uncached_usd":0.002232,"input_cached_usd":0.0,"output_usd":0.0006900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2556953750317916}
{"item_id":"senseval2.d000.s029.t007","gold_sense_keys":["so%4:02:06::"],"candidates":[{"index":1,"sense_key":"so%4:02:02::","synset_id":"so.r.01"},{"index":2,"sense_key":"so%4:02:06::","synset_id":"so.r.02"},{"index":3,"sense_key":"so%4:02:05::","synset_id":"so.r.03"},{"index":4,"sense_key":"so%4:02:07::","synset_id":"so.r.04"},{"index":5,"sense_key":"so%4:02:08::","synset_id":"so.r.05"},{"index":6,"sense_key":"so%4:02:04::","synset_id":"thus.r.02"},{"index":7,"sense_key":"so%4:02:03::","synset_id":"so.r.07"},{"index":8,"sense_key":"so%4:02:09::","synset_id":"then.r.01"},{"index":9,"sense_key":"so%4:02:10::","synset_id":"therefore.r.01"},{"index":10,"sense_key":"so%4:02:00::","synset_id":"indeed.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"so%4:02:06::","call_ids":["senseval2.d000.s029.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"so%4:02:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1276,"cached_input_tokens":0,"output_tokens":86,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003412,"input_uncached_usd":0.002552,"input_cached_usd":0.0,"output_usd":0.0008600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5818887080531567}
{"item_id":"senseval2.d000.s029.t008","gold_sense_keys":["bell%1:06:00::"],"candidates":[{"index":1,"sense_key":"bell%1:06:00::","synset_id":"bell.n.01"},{"index":2,"sense_key":"bell%1:06:02::","synset_id":"doorbell.n.01"},{"index":3,"sense_key":"bell%1:11:00::","synset_id":"bell.n.03"},{"index":4,"sense_key":"bell%1:28:00::","synset_id":"bell.n.04"},{"index":5,"sense_key":"bell%1:25:00::","synset_id":"bell.n.05"},{"index":6,"sense_key":"bell%1:18:02::","synset_id":"bell.n.06"},{"index":7,"sense_key":"bell%1:18:01::","synset_id":"bell.n.07"},{"index":8,"sense_key":"bell%1:18:00::","synset_id":"bell.n.08"},{"index":9,"sense_key":"bell%1:06:04::","synset_id":"chime.n.01"},{"index":10,"sense_key":"bell%1:06:03::","synset_id":"bell.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bell%1:06:00::","call_ids":["senseval2.d000.s029.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bell%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1262,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0026439999999999996,"input_uncached_usd":0.0025239999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9930002079345286}
{"item_id":"senseval2.d000.s029.t009","gold_sense_keys":["swap%2:40:00::"],"candidates":[{"index":1,"sense_key":"swap%2:40:00::","synset_id":"trade.v.04"},{"index":2,"sense_key":"swap%2:38:00::","synset_id":"swap.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"swap%2:40:00::","call_ids":["senseval2.d000.s029.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"swap%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":704,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001528,"input_uncached_usd":0.001408,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9585279999300838}
{"item_id":"senseval2.d000.s029.t010","gold_sense_keys":["place%1:10:02::"],"candidates":[{"index":1,"sense_key":"place%1:15:00::","synset_id":"topographic_point.n.01"},{"index":2,"sense_key":"place%1:15:04::","synset_id":"place.n.02"},{"index":3,"sense_key":"place%1:09:00::","synset_id":"place.n.03"},{"index":4,"sense_key":"place%1:15:01::","synset_id":"place.n.04"},{"index":5,"sense_key":"place%1:04:01::","synset_id":"stead.n.01"},{"index":6,"sense_key":"place%1:26:01::","synset_id":"place.n.06"},{"index":7,"sense_key":"place%1:15:06::","synset_id":"home.n.01"},{"index":8,"sense_key":"place%1:04:00::","synset_id":"position.n.06"},{"index":9,"sense_key":"place%1:15:05::","synset_id":"position.n.01"},{"index":10,"sense_key":"place%1:26:00::","synset_id":"place.n.10"},{"index":11,"sense_key":"place%1:15:03::","synset_id":"seat.n.01"},{"index":12,"sense_key":"place%1:10:00::","synset_id":"place.n.12"},{"index":13,"sense_key":"place%1:26:02::","synset_id":"place.n.13"},{"index":14,"sense_key":"place%1:15:02::","synset_id":"plaza.n.01"},{"index":15,"sense_key":"place%1:10:02::","synset_id":"place.n.15"},{"index":16,"sense_key":"place%1:10:01::","synset_id":"space.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":15,"chosen_sense_key":"place%1:10:02::","call_ids":["senseval2.d000.s029.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":15,"predicted_sense_key":"place%1:10:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1536,"cached_input_tokens":0,"output_tokens":123,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004302,"input_uncached_usd":0.003072,"input_cached_usd":0.0,"output_usd":0.0012300000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.976397666032426}
{"item_id":"senseval2.d000.s030.t000","gold_sense_keys":["well-known%5:00:00:known:00"],"candidates":[{"index":1,"sense_key":"well-known%5:00:00:known:00","synset_id":"well-known.s.01"},{"index":2,"sense_key":"well-known%5:00:00:familiar:00","synset_id":"long-familiar.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"well-known%5:00:00:known:00","call_ids":["senseval2.d000.s030.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"well-known%5:00:00:known:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":741,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0016020000000000001,"input_uncached_usd":0.001482,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.373142124968581}
{"item_id":"senseval2.d000.s030.t002","gold_sense_keys":["involve%2:42:02::"],"candidates":[{"index":1,"sense_key":"involve%2:42:01::","synset_id":"involve.v.01"},{"index":2,"sense_key":"involve%2:42:04::","synset_id":"involve.v.02"},{"index":3,"sense_key":"involve%2:42:00::","synset_id":"imply.v.05"},{"index":4,"sense_key":"involve%2:42:07::","synset_id":"necessitate.v.01"},{"index":5,"sense_key":"involve%2:42:02::","synset_id":"involve.v.05"},{"index":6,"sense_key":"involve%2:31:00::","synset_id":"involve.v.06"},{"index":7,"sense_key":"involve%2:30:00::","synset_id":"involve.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"involve%2:42:02::","call_ids":["senseval2.d000.s030.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"involve%2:42:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":991,"cached_input_tokens":0,"output_tokens":215,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004132,"input_uncached_usd":0.001982,"input_cached_usd":0.0,"output_usd":0.00215,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.3672577920369804}
{"item_id":"senseval2.d000.s030.t003","gold_sense_keys":["church%1:06:00::"],"candidates":[{"index":1,"sense_key":"church%1:14:00::","synset_id":"church.n.01"},{"index":2,"sense_key":"church%1:06:00::","synset_id":"church.n.02"},{"index":3,"sense_key":"church%1:04:00::","synset_id":"church_service.n.01"},{"index":4,"sense_key":"church%1:14:01::","synset_id":"church.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"church%1:06:00::","call_ids":["senseval2.d000.s030.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"church%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":821,"cached_input_tokens":0,"output_tokens":52,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002162,"input_uncached_usd":0.001642,"input_cached_usd":0.0,"output_usd":0.0005200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2894524579169229}
{"item_id":"senseval2.d000.s030.t004","gold_sense_keys":["bell%1:06:00::"],"candidates":[{"index":1,"sense_key":"bell%1:06:00::","synset_id":"bell.n.01"},{"index":2,"sense_key":"bell%1:06:02::","synset_id":"doorbell.n.01"},{"index":3,"sense_key":"bell%1:11:00::","synset_id":"bell.n.03"},{"index":4,"sense_key":"bell%1:28:00::","synset_id":"bell.n.04"},{"index":5,"sense_key":"bell%1:25:00::","synset_id":"bell.n.05"},{"index":6,"sense_key":"bell%1:18:02::","synset_id":"bell.n.06"},{"index":7,"sense_key":"bell%1:18:01::","synset_id":"bell.n.07"},{"index":8,"sense_key":"bell%1:18:00::","synset_id":"bell.n.08"},{"index":9,"sense_key":"bell%1:06:04::","synset_id":"chime.n.01"},{"index":10,"sense_key":"bell%1:06:03::","synset_id":"bell.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bell%1:06:00::","call_ids":["senseval2.d000.s030.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bell%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1261,"cached_input_tokens":0,"output_tokens":46,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002982,"input_uncached_usd":0.002522,"input_cached_usd":0.0,"output_usd":0.00046,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3603349999757484}
{"item_id":"senseval2.d000.s030.t005","gold_sense_keys":["english%3:01:00::"],"candidates":[{"index":1,"sense_key":"english%3:01:00::","synset_id":"english.a.01"},{"index":2,"sense_key":"english%3:01:01::","synset_id":"english.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"english%3:01:00::","call_ids":["senseval2.d000.s030.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"english%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":697,"cached_input_tokens":0,"output_tokens":49,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0018839999999999998,"input_uncached_usd":0.0013939999999999998,"input_cached_usd":0.0,"output_usd":0.0004900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2772477499675006}
{"item_id":"senseval2.d000.s030.t007","gold_sense_keys":["describe%2:32:00::","describe%2:32:01::"],"candidates":[{"index":1,"sense_key":"describe%2:32:00::","synset_id":"describe.v.01"},{"index":2,"sense_key":"describe%2:32:01::","synset_id":"report.v.01"},{"index":3,"sense_key":"describe%2:35:00::","synset_id":"trace.v.02"},{"index":4,"sense_key":"describe%2:31:00::","synset_id":"identify.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"describe%2:32:01::","call_ids":["senseval2.d000.s030.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"describe%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":860,"cached_input_tokens":0,"output_tokens":109,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00281,"input_uncached_usd":0.00172,"input_cached_usd":0.0,"output_usd":0.00109,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.791209541959688}
{"item_id":"senseval2.d000.s030.t008","gold_sense_keys":["passion%1:09:00::"],"candidates":[{"index":1,"sense_key":"passion%1:12:00::","synset_id":"passion.n.01"},{"index":2,"sense_key":"passion%1:07:00::","synset_id":"heat.n.04"},{"index":3,"sense_key":"passion%1:26:01::","synset_id":"rage.n.03"},{"index":4,"sense_key":"passion%1:16:00::","synset_id":"mania.n.01"},{"index":5,"sense_key":"passion%1:12:02::","synset_id":"passion.n.05"},{"index":6,"sense_key":"passion%1:09:00::","synset_id":"love.n.02"},{"index":7,"sense_key":"passion%1:26:02::","synset_id":"passion.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"passion%1:09:00::","call_ids":["senseval2.d000.s030.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"passion%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":957,"cached_input_tokens":0,"output_tokens":301,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0049239999999999996,"input_uncached_usd":0.001914,"input_cached_usd":0.0,"output_usd":0.00301,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.152805749909021}
{"item_id":"senseval2.d000.s030.t009","gold_sense_keys":["find%2:30:00::","find%2:38:10::"],"candidates":[{"index":1,"sense_key":"find%2:40:02::","synset_id":"find.v.01"},{"index":2,"sense_key":"find%2:39:02::","synset_id":"detect.v.01"},{"index":3,"sense_key":"find%2:40:00::","synset_id":"find.v.03"},{"index":4,"sense_key":"find%2:32:00::","synset_id":"determine.v.01"},{"index":5,"sense_key":"find%2:31:10::","synset_id":"find.v.05"},{"index":6,"sense_key":"find%2:39:01::","synset_id":"witness.v.02"},{"index":7,"sense_key":"find%2:40:01::","synset_id":"line_up.v.02"},{"index":8,"sense_key":"find%2:36:00::","synset_id":"discover.v.03"},{"index":9,"sense_key":"find%2:31:09::","synset_id":"discover.v.04"},{"index":10,"sense_key":"find%2:40:03::","synset_id":"find.v.10"},{"index":11,"sense_key":"find%2:32:01::","synset_id":"rule.v.04"},{"index":12,"sense_key":"find%2:30:00::","synset_id":"receive.v.02"},{"index":13,"sense_key":"find%2:39:05::","synset_id":"find.v.13"},{"index":14,"sense_key":"find%2:40:15::","synset_id":"recover.v.01"},{"index":15,"sense_key":"find%2:38:10::","synset_id":"find.v.15"},{"index":16,"sense_key":"find%2:30:13::","synset_id":"find_oneself.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":12,"chosen_sense_key":"find%2:30:00::","call_ids":["senseval2.d000.s030.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":12,"predicted_sense_key":"find%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1765,"cached_input_tokens":0,"output_tokens":103,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00456,"input_uncached_usd":0.0035299999999999997,"input_cached_usd":0.0,"output_usd":0.00103,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8902895000064746}
{"item_id":"senseval2.d000.s030.t010","gold_sense_keys":["satisfaction%1:12:00::","satisfaction%1:26:00::"],"candidates":[{"index":1,"sense_key":"satisfaction%1:12:00::","synset_id":"satisfaction.n.01"},{"index":2,"sense_key":"satisfaction%1:26:00::","synset_id":"gratification.n.01"},{"index":3,"sense_key":"satisfaction%1:21:00::","synset_id":"atonement.n.01"},{"index":4,"sense_key":"satisfaction%1:21:01::","synset_id":"satisfaction.n.04"},{"index":5,"sense_key":"satisfaction%1:04:00::","synset_id":"satisfaction.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"satisfaction%1:12:00::","call_ids":["senseval2.d000.s030.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"satisfaction%1:12:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":954,"cached_input_tokens":0,"output_tokens":236,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004268,"input_uncached_usd":0.001908,"input_cached_usd":0.0,"output_usd":0.00236,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.100633708992973}
{"item_id":"senseval2.d000.s030.t011","gold_sense_keys":["mathematical%5:00:00:exact:00"],"candidates":[{"index":1,"sense_key":"mathematical%3:01:00::","synset_id":"mathematical.a.01"},{"index":2,"sense_key":"mathematical%3:00:00::","synset_id":"numerical.a.03"},{"index":3,"sense_key":"mathematical%5:00:00:unquestionable:00","synset_id":"mathematical.s.01"},{"index":4,"sense_key":"mathematical%5:00:00:possible:00","synset_id":"mathematical.s.02"},{"index":5,"sense_key":"mathematical%5:00:00:exact:00","synset_id":"mathematical.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"mathematical%5:00:00:exact:00","call_ids":["senseval2.d000.s030.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"mathematical%5:00:00:exact:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":919,"cached_input_tokens":0,"output_tokens":48,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002318,"input_uncached_usd":0.001838,"input_cached_usd":0.0,"output_usd":0.00048000000000000007,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4150709999958053}
{"item_id":"senseval2.d000.s030.t012","gold_sense_keys":["completeness%1:26:00::"],"candidates":[{"index":1,"sense_key":"completeness%1:26:00::","synset_id":"completeness.n.01"},{"index":2,"sense_key":"completeness%1:07:00::","synset_id":"completeness.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"completeness%1:26:00::","call_ids":["senseval2.d000.s030.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"completeness%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":731,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001582,"input_uncached_usd":0.001462,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5230986250098795}
{"item_id":"senseval2.d000.s030.t013","gold_sense_keys":["mechanical%3:00:00::","mechanical%3:01:00::"],"candidates":[{"index":1,"sense_key":"mechanical%3:00:00::","synset_id":"mechanical.a.01"},{"index":2,"sense_key":"mechanical%3:01:01::","synset_id":"mechanical.a.02"},{"index":3,"sense_key":"mechanical%3:01:00::","synset_id":"mechanical.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"mechanical%3:00:00::","call_ids":["senseval2.d000.s030.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"mechanical%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":825,"cached_input_tokens":0,"output_tokens":121,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00286,"input_uncached_usd":0.00165,"input_cached_usd":0.0,"output_usd":0.0012100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.641897541936487}
{"item_id":"senseval2.d000.s030.t014","gold_sense_keys":["perfection%1:26:00::"],"candidates":[{"index":1,"sense_key":"perfection%1:26:00::","synset_id":"perfection.n.01"},{"index":2,"sense_key":"perfection%1:09:00::","synset_id":"paragon.n.01"},{"index":3,"sense_key":"perfection%1:04:00::","synset_id":"perfection.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"perfection%1:26:00::","call_ids":["senseval2.d000.s030.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"perfection%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":762,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001644,"input_uncached_usd":0.001524,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9027050420409068}
{"item_id":"senseval2.d000.s031.t000","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"},{"index":2,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"},{"index":3,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":4,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s031.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":798,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001716,"input_uncached_usd":0.001596,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1915129169356078}
{"item_id":"senseval2.d000.s031.t001","gold_sense_keys":["add%2:32:01::"],"candidates":[{"index":1,"sense_key":"add%2:30:00::","synset_id":"add.v.01"},{"index":2,"sense_key":"add%2:32:01::","synset_id":"add.v.02"},{"index":3,"sense_key":"add%2:40:00::","synset_id":"lend.v.01"},{"index":4,"sense_key":"add%2:31:00::","synset_id":"add.v.04"},{"index":5,"sense_key":"add%2:32:00::","synset_id":"total.v.02"},{"index":6,"sense_key":"add%2:42:00::","synset_id":"add.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"add%2:32:01::","call_ids":["senseval2.d000.s031.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"add%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":970,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0020599999999999998,"input_uncached_usd":0.0019399999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3489831669721752}
{"item_id":"senseval2.d000.s031.t002","gold_sense_keys":["filled%5:00:01:full:00"],"candidates":[{"index":1,"sense_key":"filled%5:00:01:full:00","synset_id":"filled.s.01"},{"index":2,"sense_key":"filled%3:44:00::","synset_id":"filled.a.02"},{"index":3,"sense_key":"filled%5:00:00:occupied:00","synset_id":"filled.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"filled%5:00:01:full:00","call_ids":["senseval2.d000.s031.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"filled%5:00:01:full:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":749,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001618,"input_uncached_usd":0.001498,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4665186670608819}
{"item_id":"senseval2.d000.s031.t003","gold_sense_keys":["solemn%5:00:00:serious:00"],"candidates":[{"index":1,"sense_key":"solemn%5:00:00:serious:00","synset_id":"grave.s.01"},{"index":2,"sense_key":"solemn%5:00:01:serious:00","synset_id":"earnest.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"solemn%5:00:00:serious:00","call_ids":["senseval2.d000.s031.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"solemn%5:00:00:serious:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":780,"cached_input_tokens":0,"output_tokens":112,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00268,"input_uncached_usd":0.00156,"input_cached_usd":0.0,"output_usd":0.0011200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.221056667040102}
{"item_id":"senseval2.d000.s031.t004","gold_sense_keys":["come%2:30:13::"],"candidates":[{"index":1,"sense_key":"come%2:38:00::","synset_id":"come.v.01"},{"index":2,"sense_key":"come%2:38:04::","synset_id":"arrive.v.01"},{"index":3,"sense_key":"come%2:30:01::","synset_id":"come.v.03"},{"index":4,"sense_key":"come%2:30:03::","synset_id":"come.v.04"},{"index":5,"sense_key":"come%2:42:04::","synset_id":"come.v.05"},{"index":6,"sense_key":"come%2:30:02::","synset_id":"come.v.06"},{"index":7,"sense_key":"come%2:42:13::","synset_id":"issue_forth.v.01"},{"index":8,"sense_key":"come%2:42:09::","synset_id":"hail.v.02"},{"index":9,"sense_key":"come%2:42:03::","synset_id":"come.v.09"},{"index":10,"sense_key":"come%2:42:05::","synset_id":"come.v.10"},{"index":11,"sense_key":"come%2:38:03::","synset_id":"come.v.11"},{"index":12,"sense_key":"come%2:42:00::","synset_id":"fall.v.04"},{"index":13,"sense_key":"come%2:30:13::","synset_id":"come.v.13"},{"index":14,"sense_key":"come%2:42:12::","synset_id":"total.v.01"},{"index":15,"sense_key":"come%2:42:11::","synset_id":"come.v.15"},{"index":16,"sense_key":"come%2:38:02::","synset_id":"come.v.16"},{"index":17,"sense_key":"come%2:31:14::","synset_id":"occur.v.02"},{"index":18,"sense_key":"come%2:42:15::","synset_id":"derive.v.05"},{"index":19,"sense_key":"come%2:42:02::","synset_id":"do.v.04"},{"index":20,"sense_key":"come%2:39:00::","synset_id":"come.v.20"},{"index":21,"sense_key":"come%2:31:13::","synset_id":"come.v.21"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"come%2:42:04::","call_ids":["senseval2.d000.s031.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"come%2:42:04::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1769,"cached_input_tokens":0,"output_tokens":161,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.005148,"input_uncached_usd":0.003538,"input_cached_usd":0.0,"output_usd":0.00161,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.739499249961227}
{"item_id":"senseval2.d000.s031.t006","gold_sense_keys":["ritual%1:04:00::"],"candidates":[{"index":1,"sense_key":"ritual%1:04:00::","synset_id":"ritual.n.01"},{"index":2,"sense_key":"ritual%1:04:01::","synset_id":"ritual.n.02"},{"index":3,"sense_key":"ritual%1:04:02::","synset_id":"ritual.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ritual%1:04:00::","call_ids":["senseval2.d000.s031.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ritual%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":709,"cached_input_tokens":0,"output_tokens":150,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002918,"input_uncached_usd":0.001418,"input_cached_usd":0.0,"output_usd":0.0015,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8813589579658583}
{"item_id":"senseval2.d000.s031.t008","gold_sense_keys":["perform%2:36:00::"],"candidates":[{"index":1,"sense_key":"perform%2:36:00::","synset_id":"perform.v.01"},{"index":2,"sense_key":"perform%2:41:02::","synset_id":"perform.v.02"},{"index":3,"sense_key":"perform%2:36:01::","synset_id":"perform.v.03"},{"index":4,"sense_key":"perform%2:41:00::","synset_id":"do.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"perform%2:36:00::","call_ids":["senseval2.d000.s031.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"perform%2:36:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":830,"cached_input_tokens":0,"output_tokens":174,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0034000000000000002,"input_uncached_usd":0.00166,"input_cached_usd":0.0,"output_usd":0.0017400000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.027101374929771}
{"item_id":"senseval2.d000.s032.t000","gold_sense_keys":["become%2:42:01::"],"candidates":[{"index":1,"sense_key":"become%2:30:00::","synset_id":"become.v.01"},{"index":2,"sense_key":"become%2:42:01::","synset_id":"become.v.02"},{"index":3,"sense_key":"become%2:42:00::","synset_id":"become.v.03"},{"index":4,"sense_key":"become%2:42:03::","synset_id":"become.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"become%2:42:01::","call_ids":["senseval2.d000.s032.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"become%2:42:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":797,"cached_input_tokens":0,"output_tokens":149,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003084,"input_uncached_usd":0.001594,"input_cached_usd":0.0,"output_usd":0.0014900000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0105333339888602}
{"item_id":"senseval2.d000.s032.t002","gold_sense_keys":["obsession%1:09:00::"],"candidates":[{"index":1,"sense_key":"obsession%1:16:00::","synset_id":"compulsion.n.02"},{"index":2,"sense_key":"obsession%1:09:00::","synset_id":"obsession.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"obsession%1:09:00::","call_ids":["senseval2.d000.s032.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"obsession%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":702,"cached_input_tokens":0,"output_tokens":109,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0024939999999999997,"input_uncached_usd":0.0014039999999999999,"input_cached_usd":0.0,"output_usd":0.00109,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5655232089338824}
{"item_id":"senseval2.d000.s032.t003","gold_sense_keys":["admit%2:32:00::"],"candidates":[{"index":1,"sense_key":"admit%2:32:00::","synset_id":"admit.v.01"},{"index":2,"sense_key":"admit%2:41:01::","synset_id":"admit.v.02"},{"index":3,"sense_key":"admit%2:41:00::","synset_id":"admit.v.03"},{"index":4,"sense_key":"admit%2:40:00::","synset_id":"accept.v.05"},{"index":5,"sense_key":"admit%2:42:06::","synset_id":"admit.v.05"},{"index":6,"sense_key":"admit%2:42:00::","synset_id":"admit.v.06"},{"index":7,"sense_key":"admit%2:42:04::","synset_id":"accommodate.v.04"},{"index":8,"sense_key":"admit%2:42:03::","synset_id":"admit.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"admit%2:32:00::","call_ids":["senseval2.d000.s032.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"admit%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1113,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0029460000000000003,"input_uncached_usd":0.002226,"input_cached_usd":0.0,"output_usd":0.00072,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5417502080090344}
{"item_id":"senseval2.d000.s032.t004","gold_sense_keys":["master%1:18:03::"],"candidates":[{"index":1,"sense_key":"master%1:18:00::","synset_id":"maestro.n.01"},{"index":2,"sense_key":"master%1:18:04::","synset_id":"overlord.n.01"},{"index":3,"sense_key":"master%1:18:01::","synset_id":"victor.n.01"},{"index":4,"sense_key":"master%1:18:03::","synset_id":"master.n.04"},{"index":5,"sense_key":"master%1:18:06::","synset_id":"headmaster.n.01"},{"index":6,"sense_key":"master%1:06:00::","synset_id":"master.n.06"},{"index":7,"sense_key":"master%1:18:07::","synset_id":"master.n.07"},{"index":8,"sense_key":"master%1:18:05::","synset_id":"master.n.08"},{"index":9,"sense_key":"master%1:18:02::","synset_id":"master.n.09"},{"index":10,"sense_key":"master%1:06:01::","synset_id":"passkey.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"master%1:18:03::","call_ids":["senseval2.d000.s032.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"master%1:18:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1021,"cached_input_tokens":0,"output_tokens":228,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004322,"input_uncached_usd":0.002042,"input_cached_usd":0.0,"output_usd":0.0022800000000000003,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.023597958032042}
{"item_id":"senseval2.d000.s032.t007","gold_sense_keys":["best%3:00:00::"],"candidates":[{"index":1,"sense_key":"best%3:00:00::","synset_id":"best.a.01"},{"index":2,"sense_key":"best%5:00:00:advisable:00","synset_id":"better.s.01"},{"index":3,"sense_key":"good%3:00:01::","synset_id":"good.a.01"},{"index":4,"sense_key":"full%5:00:00:ample:00","synset_id":"full.s.04"},{"index":5,"sense_key":"good%3:00:02::","synset_id":"good.a.03"},{"index":6,"sense_key":"estimable%5:00:00:reputable:00","synset_id":"estimable.s.01"},{"index":7,"sense_key":"beneficial%5:00:00:advantageous:00","synset_id":"beneficial.s.01"},{"index":8,"sense_key":"good%5:00:00:nice:00","synset_id":"good.s.04"},{"index":9,"sense_key":"good%5:00:00:righteous:00","synset_id":"good.s.05"},{"index":10,"sense_key":"adept%5:00:00:skilled:00","synset_id":"adept.s.01"},{"index":11,"sense_key":"good%5:00:00:complete:00","synset_id":"good.s.07"},{"index":12,"sense_key":"dear%5:00:00:close:02","synset_id":"dear.s.02"},{"index":13,"sense_key":"dependable%5:00:00:sound:00","synset_id":"dependable.s.03"},{"index":14,"sense_key":"good%5:00:00:opportune:00","synset_id":"good.s.10"},{"index":15,"sense_key":"good%5:00:00:fortunate:00","synset_id":"good.s.11"},{"index":16,"sense_key":"effective%5:00:00:operative:00","synset_id":"effective.s.03"},{"index":17,"sense_key":"good%5:00:00:pleasing:00","synset_id":"good.s.13"},{"index":18,"sense_key":"good%5:00:00:intellectual:00","synset_id":"good.s.14"},{"index":19,"sense_key":"good%5:00:00:healthy:00","synset_id":"good.s.15"},{"index":20,"sense_key":"good%5:00:00:healthful:00","synset_id":"good.s.16"},{"index":21,"sense_key":"good%5:00:00:genuine:00","synset_id":"good.s.17"},{"index":22,"sense_key":"good%5:00:00:fresh:01","synset_id":"good.s.18"},{"index":23,"sense_key":"good%5:00:00:discriminating:00","synset_id":"good.s.19"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"best%3:00:00::","call_ids":["senseval2.d000.s032.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"best%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2037,"cached_input_tokens":0,"output_tokens":231,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.006383999999999999,"input_uncached_usd":0.0040739999999999995,"input_cached_usd":0.0,"output_usd":0.00231,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.8724824159871787}
{"item_id":"senseval2.d000.s032.t008","gold_sense_keys":["female%3:00:00::"],"candidates":[{"index":1,"sense_key":"female%3:00:00::","synset_id":"female.a.01"},{"index":2,"sense_key":"female%5:00:02:feminine:01","synset_id":"female.s.01"},{"index":3,"sense_key":"female%5:00:01:female:00","synset_id":"female.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"female%5:00:01:female:00","call_ids":["senseval2.d000.s032.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"female%5:00:01:female:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":764,"cached_input_tokens":0,"output_tokens":269,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0042179999999999995,"input_uncached_usd":0.0015279999999999998,"input_cached_usd":0.0,"output_usd":0.00269,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.010136832948774}
{"item_id":"senseval2.d000.s032.t009","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"},{"index":2,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"},{"index":3,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":4,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s032.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":772,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0016640000000000001,"input_uncached_usd":0.001544,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9230891249608248}
{"item_id":"senseval2.d000.s033.t000","gold_sense_keys":["passion%1:09:00::"],"candidates":[{"index":1,"sense_key":"passion%1:12:00::","synset_id":"passion.n.01"},{"index":2,"sense_key":"passion%1:07:00::","synset_id":"heat.n.04"},{"index":3,"sense_key":"passion%1:26:01::","synset_id":"rage.n.03"},{"index":4,"sense_key":"passion%1:16:00::","synset_id":"mania.n.01"},{"index":5,"sense_key":"passion%1:12:02::","synset_id":"passion.n.05"},{"index":6,"sense_key":"passion%1:09:00::","synset_id":"love.n.02"},{"index":7,"sense_key":"passion%1:26:02::","synset_id":"passion.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"passion%1:09:00::","call_ids":["senseval2.d000.s033.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"passion%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":884,"cached_input_tokens":0,"output_tokens":245,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004218,"input_uncached_usd":0.0017679999999999998,"input_cached_usd":0.0,"output_usd":0.0024500000000000004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.5831328340573236}
{"item_id":"senseval2.d000.s033.t002","gold_sense_keys":["stay%2:42:04::"],"candidates":[{"index":1,"sense_key":"stay%2:30:00::","synset_id":"stay.v.01"},{"index":2,"sense_key":"stay%2:38:00::","synset_id":"stay.v.02"},{"index":3,"sense_key":"stay%2:42:01::","synset_id":"bide.v.01"},{"index":4,"sense_key":"stay%2:42:02::","synset_id":"stay.v.04"},{"index":5,"sense_key":"stay%2:38:01::","synset_id":"stay.v.05"},{"index":6,"sense_key":"stay%2:30:01::","synset_id":"stay.v.06"},{"index":7,"sense_key":"stay%2:42:04::","synset_id":"persist.v.03"},{"index":8,"sense_key":"stay%2:42:03::","synset_id":"last_out.v.01"},{"index":9,"sense_key":"stay%2:41:00::","synset_id":"stay.v.09"},{"index":10,"sense_key":"stay%2:35:00::","synset_id":"stay.v.10"},{"index":11,"sense_key":"stay%2:34:00::","synset_id":"quell.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"stay%2:42:04::","call_ids":["senseval2.d000.s033.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"stay%2:42:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1181,"cached_input_tokens":0,"output_tokens":156,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003922,"input_uncached_usd":0.002362,"input_cached_usd":0.0,"output_usd":0.0015600000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1735127079300582}
{"item_id":"senseval2.d000.s033.t003","gold_sense_keys":["tower%1:06:00::"],"candidates":[{"index":1,"sense_key":"tower%1:06:00::","synset_id":"tower.n.01"},{"index":2,"sense_key":"tower%1:25:00::","synset_id":"column.n.04"},{"index":3,"sense_key":"tower%1:06:01::","synset_id":"tugboat.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"tower%1:06:00::","call_ids":["senseval2.d000.s033.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"tower%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":723,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0015660000000000001,"input_uncached_usd":0.001446,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.119063499965705}
{"item_id":"senseval2.d000.s033.t004","gold_sense_keys":["however%4:02:00::"],"candidates":[{"index":1,"sense_key":"however%4:02:00::","synset_id":"however.r.01"},{"index":2,"sense_key":"however%4:02:04::","synset_id":"however.r.02"},{"index":3,"sense_key":"however%4:02:02::","synset_id":"however.r.03"},{"index":4,"sense_key":"however%4:02:01::","synset_id":"however.r.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"however%4:02:00::","call_ids":["senseval2.d000.s033.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"however%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":831,"cached_input_tokens":0,"output_tokens":247,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004132,"input_uncached_usd":0.0016619999999999998,"input_cached_usd":0.0,"output_usd":0.0024700000000000004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.644340040977113}
{"item_id":"senseval2.d000.s034.t000","gold_sense_keys":["more%4:02:00::"],"candidates":[{"index":1,"sense_key":"more%4:02:00::","synset_id":"more.r.01"},{"index":2,"sense_key":"more%4:02:01::","synset_id":"more.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"more%4:02:00::","call_ids":["senseval2.d000.s034.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"more%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":633,"cached_input_tokens":0,"output_tokens":106,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002326,"input_uncached_usd":0.001266,"input_cached_usd":0.0,"output_usd":0.0010600000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5527673750184476}
{"item_id":"senseval2.d000.s034.t001","gold_sense_keys":["often%4:02:00::","often%4:02:02::"],"candidates":[{"index":1,"sense_key":"often%4:02:00::","synset_id":"frequently.r.01"},{"index":2,"sense_key":"often%4:02:01::","synset_id":"much.r.05"},{"index":3,"sense_key":"often%4:02:02::","synset_id":"often.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"often%4:02:02::","call_ids":["senseval2.d000.s034.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"often%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":670,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0019500000000000001,"input_uncached_usd":0.00134,"input_cached_usd":0.0,"output_usd":0.0006100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6588620420079678}
{"item_id":"senseval2.d000.s034.t002","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"},{"index":2,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"},{"index":3,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":4,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s034.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":720,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00156,"input_uncached_usd":0.0014399999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5989842920098454}
{"item_id":"senseval2.d000.s034.t003","gold_sense_keys":["think_of%2:31:01::"],"candidates":[{"index":1,"sense_key":"think_of%2:31:00::","synset_id":"remember.v.02"},{"index":2,"sense_key":"think_of%2:31:03::","synset_id":"entertain.v.02"},{"index":3,"sense_key":"think_of%2:31:01::","synset_id":"think_of.v.03"},{"index":4,"sense_key":"think_of%2:31:02::","synset_id":"think_of.v.04"},{"index":5,"sense_key":"think_of%2:36:00::","synset_id":"think_up.v.01"},{"index":6,"sense_key":"think_of%2:31:04::","synset_id":"think_of.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"think_of%2:31:01::","call_ids":["senseval2.d000.s034.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"think_of%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":955,"cached_input_tokens":0,"output_tokens":89,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0028,"input_uncached_usd":0.0019099999999999998,"input_cached_usd":0.0,"output_usd":0.0008900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3972016249317676}
{"item_id":"senseval2.d000.s034.t004","gold_sense_keys":["church%1:06:00::"],"candidates":[{"index":1,"sense_key":"church%1:14:00::","synset_id":"church.n.01"},{"index":2,"sense_key":"church%1:06:00::","synset_id":"church.n.02"},{"index":3,"sense_key":"church%1:04:00::","synset_id":"church_service.n.01"},{"index":4,"sense_key":"church%1:14:01::","synset_id":"church.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"church%1:06:00::","call_ids":["senseval2.d000.s034.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"church%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":724,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002118,"input_uncached_usd":0.0014479999999999999,"input_cached_usd":0.0,"output_usd":0.00067,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4362866670126095}
{"item_id":"senseval2.d000.s034.t006","gold_sense_keys":["bottom%1:15:01::"],"candidates":[{"index":1,"sense_key":"bottom%1:15:00::","synset_id":"bottom.n.01"},{"index":2,"sense_key":"bottom%1:15:01::","synset_id":"bottom.n.02"},{"index":3,"sense_key":"bottom%1:08:00::","synset_id":"buttocks.n.01"},{"index":4,"sense_key":"bottom%1:28:00::","synset_id":"bottom.n.04"},{"index":5,"sense_key":"bottom%1:17:00::","synset_id":"bed.n.03"},{"index":6,"sense_key":"bottom%1:17:01::","synset_id":"bottomland.n.01"},{"index":7,"sense_key":"bottom%1:06:00::","synset_id":"bottom.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bottom%1:15:01::","call_ids":["senseval2.d000.s034.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bottom%1:15:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":936,"cached_input_tokens":0,"output_tokens":70,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002572,"input_uncached_usd":0.001872,"input_cached_usd":0.0,"output_usd":0.0007000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.487657624995336}
{"item_id":"senseval2.d000.s035.t001","gold_sense_keys":["complete%2:30:02::"],"candidates":[{"index":1,"sense_key":"complete%2:30:02::","synset_id":"complete.v.01"},{"index":2,"sense_key":"complete%2:30:00::","synset_id":"complete.v.02"},{"index":3,"sense_key":"complete%2:41:00::","synset_id":"dispatch.v.02"},{"index":4,"sense_key":"complete%2:33:00::","synset_id":"complete.v.04"},{"index":5,"sense_key":"complete%2:32:00::","synset_id":"complete.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"complete%2:30:02::","call_ids":["senseval2.d000.s035.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"complete%2:30:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":766,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002162,"input_uncached_usd":0.001532,"input_cached_usd":0.0,"output_usd":0.00063,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9204762920271605}
{"item_id":"senseval2.d000.s035.t002","gold_sense_keys":["work_up%2:29:00::"],"candidates":[{"index":1,"sense_key":"work_up%2:30:00::","synset_id":"build_up.v.02"},{"index":2,"sense_key":"work_up%2:29:00::","synset_id":"work_up.v.02"},{"index":3,"sense_key":"work_up%2:30:06::","synset_id":"build_up.v.04"},{"index":4,"sense_key":"work_up%2:30:03::","synset_id":"work_out.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"work_up%2:29:00::","call_ids":["senseval2.d000.s035.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"work_up%2:29:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":751,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002142,"input_uncached_usd":0.0015019999999999999,"input_cached_usd":0.0,"output_usd":0.00064,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.510041332920082}
{"item_id":"senseval2.d000.s035.t003","gold_sense_keys":["sweat%1:08:00::"],"candidates":[{"index":1,"sense_key":"sweat%1:08:00::","synset_id":"perspiration.n.01"},{"index":2,"sense_key":"sweat%1:26:01::","synset_id":"fret.n.01"},{"index":3,"sense_key":"sweat%1:19:00::","synset_id":"sweat.n.03"},{"index":4,"sense_key":"sweat%1:04:00::","synset_id":"effort.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"sweat%1:08:00::","call_ids":["senseval2.d000.s035.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"sweat%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":761,"cached_input_tokens":0,"output_tokens":52,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002042,"input_uncached_usd":0.001522,"input_cached_usd":0.0,"output_usd":0.0005200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7481180829927325}
{"item_id":"senseval2.d000.s035.t004","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"},{"index":2,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"},{"index":3,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":4,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s035.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":693,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001506,"input_uncached_usd":0.0013859999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0929142920067534}
{"item_id":"senseval2.d000.s035.t005","gold_sense_keys":["often%4:02:00::","often%4:02:02::"],"candidates":[{"index":1,"sense_key":"often%4:02:00::","synset_id":"frequently.r.01"},{"index":2,"sense_key":"often%4:02:01::","synset_id":"much.r.05"},{"index":3,"sense_key":"often%4:02:02::","synset_id":"often.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"often%4:02:02::","call_ids":["senseval2.d000.s035.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"often%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":643,"cached_input_tokens":0,"output_tokens":273,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0040160000000000005,"input_uncached_usd":0.001286,"input_cached_usd":0.0,"output_usd":0.0027300000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.804825291968882}
{"item_id":"senseval2.d000.s035.t006","gold_sense_keys":["skip%2:38:02::"],"candidates":[{"index":1,"sense_key":"skip%2:31:01::","synset_id":"jump.v.13"},{"index":2,"sense_key":"skip%2:42:00::","synset_id":"cut.v.11"},{"index":3,"sense_key":"skip%2:38:00::","synset_id":"hop.v.01"},{"index":4,"sense_key":"skip%2:38:02::","synset_id":"decamp.v.03"},{"index":5,"sense_key":"skip%2:38:03::","synset_id":"skip.v.05"},{"index":6,"sense_key":"skip%2:35:00::","synset_id":"skim.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"skip%2:38:02::","call_ids":["senseval2.d000.s035.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"skip%2:38:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":800,"cached_input_tokens":0,"output_tokens":109,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0026899999999999997,"input_uncached_usd":0.0015999999999999999,"input_cached_usd":0.0,"output_usd":0.00109,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6288092080503702}
{"item_id":"senseval2.d000.s035.t007","gold_sense_keys":["local%3:01:01::"],"candidates":[{"index":1,"sense_key":"local%3:00:01::","synset_id":"local.a.01"},{"index":2,"sense_key":"local%3:01:01::","synset_id":"local.a.02"},{"index":3,"sense_key":"local%3:00:02::","synset_id":"local.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"local%3:01:01::","call_ids":["senseval2.d000.s035.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"local%3:01:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":653,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001426,"input_uncached_usd":0.0013059999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.153692416031845}
{"item_id":"senseval2.d000.s035.t009","gold_sense_keys":["leave%2:40:06::"],"candidates":[{"index":1,"sense_key":"leave%2:38:01::","synset_id":"leave.v.01"},{"index":2,"sense_key":"leave%2:31:05::","synset_id":"leave.v.02"},{"index":3,"sense_key":"leave%2:42:02::","synset_id":"leave.v.03"},{"index":4,"sense_key":"leave%2:30:02::","synset_id":"leave.v.04"},{"index":5,"sense_key":"leave%2:38:00::","synset_id":"exit.v.01"},{"index":6,"sense_key":"leave%2:42:01::","synset_id":"leave.v.06"},{"index":7,"sense_key":"leave%2:42:00::","synset_id":"leave.v.07"},{"index":8,"sense_key":"leave%2:41:00::","synset_id":"leave.v.08"},{"index":9,"sense_key":"leave%2:40:06::","synset_id":"entrust.v.02"},{"index":10,"sense_key":"leave%2:40:01::","synset_id":"bequeath.v.01"},{"index":11,"sense_key":"leave%2:42:03::","synset_id":"leave.v.11"},{"index":12,"sense_key":"leave%2:30:03::","synset_id":"leave.v.12"},{"index":13,"sense_key":"leave%2:40:02::","synset_id":"impart.v.01"},{"index":14,"sense_key":"leave%2:31:02::","synset_id":"forget.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"leave%2:40:06::","call_ids":["senseval2.d000.s035.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"leave%2:40:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1453,"cached_input_tokens":0,"output_tokens":171,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004616,"input_uncached_usd":0.0029059999999999997,"input_cached_usd":0.0,"output_usd":0.0017100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.580695749958977}
{"item_id":"senseval2.d000.s035.t010","gold_sense_keys":["worship%1:04:00::"],"candidates":[{"index":1,"sense_key":"worship%1:04:00::","synset_id":"worship.n.01"},{"index":2,"sense_key":"worship%1:12:01::","synset_id":"worship.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"worship%1:04:00::","call_ids":["senseval2.d000.s035.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"worship%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":564,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001248,"input_uncached_usd":0.0011279999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9700480829924345}
{"item_id":"senseval2.d000.s035.t011","gold_sense_keys":["below%4:02:06::"],"candidates":[{"index":1,"sense_key":"below%4:02:01::","synset_id":"below.r.01"},{"index":2,"sense_key":"below%4:02:00::","synset_id":"below.r.02"},{"index":3,"sense_key":"below%4:02:02::","synset_id":"below.r.03"},{"index":4,"sense_key":"below%4:02:06::","synset_id":"downstairs.r.01"},{"index":5,"sense_key":"below%4:02:03::","synset_id":"under.r.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"below%4:02:06::","call_ids":["senseval2.d000.s035.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"below%4:02:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":718,"cached_input_tokens":0,"output_tokens":158,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003016,"input_uncached_usd":0.001436,"input_cached_usd":0.0,"output_usd":0.00158,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":4.8654209999367595}
{"item_id":"senseval2.d000.s036.t000","gold_sense_keys":["well%4:02:00::"],"candidates":[{"index":1,"sense_key":"well%4:02:00::","synset_id":"well.r.01"},{"index":2,"sense_key":"well%4:02:05::","synset_id":"well.r.02"},{"index":3,"sense_key":"well%4:02:03::","synset_id":"well.r.03"},{"index":4,"sense_key":"well%4:02:15::","synset_id":"well.r.04"},{"index":5,"sense_key":"well%4:02:08::","synset_id":"well.r.05"},{"index":6,"sense_key":"well%4:02:07::","synset_id":"well.r.06"},{"index":7,"sense_key":"well%4:02:11::","synset_id":"well.r.07"},{"index":8,"sense_key":"well%4:02:14::","synset_id":"well.r.08"},{"index":9,"sense_key":"well%4:02:13::","synset_id":"well.r.09"},{"index":10,"sense_key":"well%4:02:12::","synset_id":"well.r.10"},{"index":11,"sense_key":"well%4:02:10::","synset_id":"well.r.11"},{"index":12,"sense_key":"well%4:02:09::","synset_id":"well.r.12"},{"index":13,"sense_key":"well%4:02:01::","synset_id":"well.r.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":13,"chosen_sense_key":"well%4:02:01::","call_ids":["senseval2.d000.s036.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":13,"predicted_sense_key":"well%4:02:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1281,"cached_input_tokens":0,"output_tokens":123,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0037920000000000002,"input_uncached_usd":0.002562,"input_cached_usd":0.0,"output_usd":0.0012300000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.247272541979328}
{"item_id":"senseval2.d000.s037.t000","gold_sense_keys":["membership%1:14:00::"],"candidates":[{"index":1,"sense_key":"membership%1:14:00::","synset_id":"membership.n.01"},{"index":2,"sense_key":"membership%1:26:00::","synset_id":"membership.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"membership%1:14:00::","call_ids":["senseval2.d000.s037.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"membership%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":604,"cached_input_tokens":0,"output_tokens":75,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0019579999999999997,"input_uncached_usd":0.0012079999999999999,"input_cached_usd":0.0,"output_usd":0.00075,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7392867079470307}
{"item_id":"senseval2.d000.s037.t002","gold_sense_keys":["steadily%4:02:01::"],"candidates":[{"index":1,"sense_key":"steadily%4:02:01::","synset_id":"steadily.r.01"},{"index":2,"sense_key":"steadily%4:02:00::","synset_id":"steadily.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"steadily%4:02:01::","call_ids":["senseval2.d000.s037.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"steadily%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":610,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00134,"input_uncached_usd":0.00122,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.156824417063035}
{"item_id":"senseval2.d000.s037.t005","gold_sense_keys":["vicar%1:18:01::"],"candidates":[{"index":1,"sense_key":"vicar%1:18:00::","synset_id":"vicar.n.01"},{"index":2,"sense_key":"vicar%1:18:02::","synset_id":"vicar.n.02"},{"index":3,"sense_key":"vicar%1:18:01::","synset_id":"vicar.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"vicar%1:18:01::","call_ids":["senseval2.d000.s037.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"vicar%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":655,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00143,"input_uncached_usd":0.00131,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0616535000735894}
{"item_id":"senseval2.d000.s037.t006","gold_sense_keys":["press%2:32:00::"],"candidates":[{"index":1,"sense_key":"press%2:35:00::","synset_id":"press.v.01"},{"index":2,"sense_key":"press%2:32:00::","synset_id":"urge.v.01"},{"index":3,"sense_key":"press%2:42:01::","synset_id":"weigh.v.05"},{"index":4,"sense_key":"press%2:35:01::","synset_id":"press.v.04"},{"index":5,"sense_key":"press%2:35:02::","synset_id":"compress.v.02"},{"index":6,"sense_key":"press%2:38:00::","synset_id":"press.v.06"},{"index":7,"sense_key":"press%2:36:01::","synset_id":"press.v.07"},{"index":8,"sense_key":"press%2:42:00::","synset_id":"press.v.08"},{"index":9,"sense_key":"press%2:41:12::","synset_id":"crusade.v.01"},{"index":10,"sense_key":"press%2:36:00::","synset_id":"press.v.10"},{"index":11,"sense_key":"press%2:35:06::","synset_id":"press.v.11"},{"index":12,"sense_key":"press%2:35:03::","synset_id":"iron.v.01"},{"index":13,"sense_key":"press%2:33:00::","synset_id":"weight-lift.v.01"},{"index":14,"sense_key":"press%2:32:01::","synset_id":"bid.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"press%2:32:00::","call_ids":["senseval2.d000.s037.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"press%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1348,"cached_input_tokens":0,"output_tokens":117,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003866,"input_uncached_usd":0.002696,"input_cached_usd":0.0,"output_usd":0.00117,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6182893750956282}
{"item_id":"senseval2.d000.s037.t007","gold_sense_keys":["equally%4:02:01::"],"candidates":[{"index":1,"sense_key":"equally%4:02:01::","synset_id":"equally.r.01"},{"index":2,"sense_key":"equally%4:02:02::","synset_id":"evenly.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"equally%4:02:01::","call_ids":["senseval2.d000.s037.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"equally%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":665,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00195,"input_uncached_usd":0.00133,"input_cached_usd":0.0,"output_usd":0.00062,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3901042089564726}
{"item_id":"senseval2.d000.s037.t009","gold_sense_keys":["often%4:02:02::"],"candidates":[{"index":1,"sense_key":"often%4:02:00::","synset_id":"frequently.r.01"},{"index":2,"sense_key":"often%4:02:01::","synset_id":"much.r.05"},{"index":3,"sense_key":"often%4:02:02::","synset_id":"often.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"often%4:02:02::","call_ids":["senseval2.d000.s037.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"often%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":666,"cached_input_tokens":0,"output_tokens":91,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002242,"input_uncached_usd":0.0013319999999999999,"input_cached_usd":0.0,"output_usd":0.0009100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.5658068329794332}
{"item_id":"senseval2.d000.s037.t010","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"},{"index":2,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"},{"index":3,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":4,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s037.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":716,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001552,"input_uncached_usd":0.001432,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0584742079954594}
{"item_id":"senseval2.d000.s037.t011","gold_sense_keys":["attend%2:42:00::"],"candidates":[{"index":1,"sense_key":"attend%2:42:00::","synset_id":"attend.v.01"},{"index":2,"sense_key":"attend%2:41:12::","synset_id":"attend.v.02"},{"index":3,"sense_key":"attend%2:42:01::","synset_id":"attend.v.03"},{"index":4,"sense_key":"attend%2:41:00::","synset_id":"serve.v.10"},{"index":5,"sense_key":"attend%2:39:00::","synset_id":"attend.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"attend%2:42:00::","call_ids":["senseval2.d000.s037.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"attend%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":859,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001838,"input_uncached_usd":0.001718,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.541613584035076}
{"item_id":"senseval2.d000.s037.t012","gold_sense_keys":["service%1:04:01::"],"candidates":[{"index":1,"sense_key":"service%1:04:08::","synset_id":"service.n.01"},{"index":2,"sense_key":"service%1:04:00::","synset_id":"service.n.02"},{"index":3,"sense_key":"service%1:04:01::","synset_id":"service.n.03"},{"index":4,"sense_key":"service%1:14:05::","synset_id":"service.n.04"},{"index":5,"sense_key":"service%1:04:07::","synset_id":"service.n.05"},{"index":6,"sense_key":"service%1:14:00::","synset_id":"military_service.n.01"},{"index":7,"sense_key":"service%1:18:00::","synset_id":"service.n.07"},{"index":8,"sense_key":"service%1:07:00::","synset_id":"avail.n.01"},{"index":9,"sense_key":"service%1:06:00::","synset_id":"service.n.09"},{"index":10,"sense_key":"service%1:04:03::","synset_id":"servicing.n.01"},{"index":11,"sense_key":"service%1:04:09::","synset_id":"service.n.11"},{"index":12,"sense_key":"service%1:04:02::","synset_id":"serve.n.01"},{"index":13,"sense_key":"service%1:04:04::","synset_id":"service.n.13"},{"index":14,"sense_key":"service%1:04:05::","synset_id":"overhaul.n.01"},{"index":15,"sense_key":"service%1:04:06::","synset_id":"service.n.15"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"service%1:04:01::","call_ids":["senseval2.d000.s037.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"service%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1311,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0027419999999999996,"input_uncached_usd":0.0026219999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9984412920894101}
{"item_id":"senseval2.d000.s038.t000","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["senseval2.d000.s038.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":760,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00164,"input_uncached_usd":0.0015199999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5397454160265625}
{"item_id":"senseval2.d000.s038.t002","gold_sense_keys":["vicar%1:18:01::"],"candidates":[{"index":1,"sense_key":"vicar%1:18:00::","synset_id":"vicar.n.01"},{"index":2,"sense_key":"vicar%1:18:02::","synset_id":"vicar.n.02"},{"index":3,"sense_key":"vicar%1:18:01::","synset_id":"vicar.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"vicar%1:18:01::","call_ids":["senseval2.d000.s038.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"vicar%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":643,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0014060000000000001,"input_uncached_usd":0.001286,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0296520829433575}
{"item_id":"senseval2.d000.s038.t003","gold_sense_keys":["get%2:30:00::"],"candidates":[{"index":1,"sense_key":"get%2:40:00::","synset_id":"get.v.01"},{"index":2,"sense_key":"get%2:30:00::","synset_id":"become.v.01"},{"index":3,"sense_key":"get%2:30:02::","synset_id":"get.v.03"},{"index":4,"sense_key":"get%2:30:03::","synset_id":"receive.v.02"},{"index":5,"sense_key":"get%2:38:00::","synset_id":"arrive.v.01"},{"index":6,"sense_key":"get%2:35:03::","synset_id":"bring.v.04"},{"index":7,"sense_key":"get%2:39:00::","synset_id":"experience.v.03"},{"index":8,"sense_key":"get%2:33:00::","synset_id":"pay_back.v.02"},{"index":9,"sense_key":"get%2:40:01::","synset_id":"have.v.17"},{"index":10,"sense_key":"get%2:32:00::","synset_id":"induce.v.02"},{"index":11,"sense_key":"get%2:35:09::","synset_id":"get.v.11"},{"index":12,"sense_key":"get%2:29:01::","synset_id":"grow.v.08"},{"index":13,"sense_key":"get%2:29:00::","synset_id":"contract.v.04"},{"index":14,"sense_key":"get%2:32:03::","synset_id":"get.v.14"},{"index":15,"sense_key":"get%2:30:01::","synset_id":"make.v.02"},{"index":16,"sense_key":"get%2:32:08::","synset_id":"drive.v.11"},{"index":17,"sense_key":"get%2:31:09::","synset_id":"catch.v.18"},{"index":18,"sense_key":"get%2:35:00::","synset_id":"catch.v.07"},{"index":19,"sense_key":"get%2:35:10::","synset_id":"get.v.19"},{"index":20,"sense_key":"get%2:31:03::","synset_id":"get.v.20"},{"index":21,"sense_key":"get%2:40:14::","synset_id":"get.v.21"},{"index":22,"sense_key":"get%2:40:13::","synset_id":"get.v.22"},{"index":23,"sense_key":"get%2:39:13::","synset_id":"catch.v.21"},{"index":24,"sense_key":"get%2:39:09::","synset_id":"catch.v.22"},{"index":25,"sense_key":"get%2:39:14::","synset_id":"get.v.25"},{"index":26,"sense_key":"get%2:38:15::","synset_id":"scram.v.01"},{"index":27,"sense_key":"get%2:38:12::","synset_id":"get.v.27"},{"index":28,"sense_key":"get%2:37:12::","synset_id":"get.v.28"},{"index":29,"sense_key":"get%2:37:11::","synset_id":"get.v.29"},{"index":30,"sense_key":"get%2:36:14::","synset_id":"catch.v.24"},{"index":31,"sense_key":"get%2:36:00::","synset_id":"draw.v.15"},{"index":32,"sense_key":"get%2:35:12::","synset_id":"get.v.32"},{"index":33,"sense_key":"get%2:31:10::","synset_id":"perplex.v.01"},{"index":34,"sense_key":"get%2:30:12::","synset_id":"get_down.v.07"},{"index":35,"sense_key":"get%2:29:11::","synset_id":"suffer.v.02"},{"index":36,"sense_key":"get%2:29:08::","synset_id":"beget.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"get%2:30:00::","call_ids":["senseval2.d000.s038.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"get%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2843,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.006456,"input_uncached_usd":0.005686,"input_cached_usd":0.0,"output_usd":0.0007700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0283491669688374}
{"item_id":"senseval2.d000.s038.t004","gold_sense_keys":["so%4:02:03::"],"candidates":[{"index":1,"sense_key":"so%4:02:02::","synset_id":"so.r.01"},{"index":2,"sense_key":"so%4:02:06::","synset_id":"so.r.02"},{"index":3,"sense_key":"so%4:02:05::","synset_id":"so.r.03"},{"index":4,"sense_key":"so%4:02:07::","synset_id":"so.r.04"},{"index":5,"sense_key":"so%4:02:08::","synset_id":"so.r.05"},{"index":6,"sense_key":"so%4:02:04::","synset_id":"thus.r.02"},{"index":7,"sense_key":"so%4:02:03::","synset_id":"so.r.07"},{"index":8,"sense_key":"so%4:02:09::","synset_id":"then.r.01"},{"index":9,"sense_key":"so%4:02:10::","synset_id":"therefore.r.01"},{"index":10,"sense_key":"so%4:02:00::","synset_id":"indeed.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"so%4:02:03::","call_ids":["senseval2.d000.s038.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"so%4:02:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1162,"cached_input_tokens":0,"output_tokens":142,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003744,"input_uncached_usd":0.002324,"input_cached_usd":0.0,"output_usd":0.00142,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.080633291043341}
{"item_id":"senseval2.d000.s038.t005","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"},{"index":2,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"},{"index":3,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":4,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s038.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":704,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001528,"input_uncached_usd":0.001408,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0573224580148235}
{"item_id":"senseval2.d000.s038.t006","gold_sense_keys":["attend%2:42:00::"],"candidates":[{"index":1,"sense_key":"attend%2:42:00::","synset_id":"attend.v.01"},{"index":2,"sense_key":"attend%2:41:12::","synset_id":"attend.v.02"},{"index":3,"sense_key":"attend%2:42:01::","synset_id":"attend.v.03"},{"index":4,"sense_key":"attend%2:41:00::","synset_id":"serve.v.10"},{"index":5,"sense_key":"attend%2:39:00::","synset_id":"attend.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"attend%2:42:00::","call_ids":["senseval2.d000.s038.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"attend%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":847,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001814,"input_uncached_usd":0.001694,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0499290829757228}
{"item_id":"senseval2.d000.s038.t007","gold_sense_keys":["service%1:04:01::"],"candidates":[{"index":1,"sense_key":"service%1:04:08::","synset_id":"service.n.01"},{"index":2,"sense_key":"service%1:04:00::","synset_id":"service.n.02"},{"index":3,"sense_key":"service%1:04:01::","synset_id":"service.n.03"},{"index":4,"sense_key":"service%1:14:05::","synset_id":"service.n.04"},{"index":5,"sense_key":"service%1:04:07::","synset_id":"service.n.05"},{"index":6,"sense_key":"service%1:14:00::","synset_id":"military_service.n.01"},{"index":7,"sense_key":"service%1:18:00::","synset_id":"service.n.07"},{"index":8,"sense_key":"service%1:07:00::","synset_id":"avail.n.01"},{"index":9,"sense_key":"service%1:06:00::","synset_id":"service.n.09"},{"index":10,"sense_key":"service%1:04:03::","synset_id":"servicing.n.01"},{"index":11,"sense_key":"service%1:04:09::","synset_id":"service.n.11"},{"index":12,"sense_key":"service%1:04:02::","synset_id":"serve.n.01"},{"index":13,"sense_key":"service%1:04:04::","synset_id":"service.n.13"},{"index":14,"sense_key":"service%1:04:05::","synset_id":"overhaul.n.01"},{"index":15,"sense_key":"service%1:04:06::","synset_id":"service.n.15"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"service%1:04:01::","call_ids":["senseval2.d000.s038.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"service%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1299,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002718,"input_uncached_usd":0.002598,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5057332910364494}
{"item_id":"senseval2.d000.s038.t008","gold_sense_keys":["sack%2:41:00::"],"candidates":[{"index":1,"sense_key":"sack%2:40:00::","synset_id":"sack.v.01"},{"index":2,"sense_key":"sack%2:41:00::","synset_id":"displace.v.03"},{"index":3,"sense_key":"sack%2:40:01::","synset_id":"net.v.01"},{"index":4,"sense_key":"sack%2:35:00::","synset_id":"sack.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"sack%2:41:00::","call_ids":["senseval2.d000.s038.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"sack%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":747,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001614,"input_uncached_usd":0.0014939999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9992277909768745}
{"item_id":"senseval2.d000.s038.t009","gold_sense_keys":["entire%5:00:01:whole:00"],"candidates":[{"index":1,"sense_key":"entire%5:00:01:whole:00","synset_id":"entire.s.01"},{"index":2,"sense_key":"entire%5:00:02:whole:00","synset_id":"integral.s.02"},{"index":3,"sense_key":"entire%5:00:00:smooth:02","synset_id":"entire.s.03"},{"index":4,"sense_key":"entire%5:00:00:uncastrated:00","synset_id":"entire.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"entire%5:00:01:whole:00","call_ids":["senseval2.d000.s038.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"entire%5:00:01:whole:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":793,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001706,"input_uncached_usd":0.001586,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0634872089140117}
{"item_id":"senseval2.d000.s038.t011","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"},{"index":2,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"},{"index":3,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":4,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s038.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":704,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001528,"input_uncached_usd":0.001408,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0635019999463111}
{"item_id":"senseval2.d000.s038.t012","gold_sense_keys":["promptly%4:02:02::"],"candidates":[{"index":1,"sense_key":"promptly%4:02:02::","synset_id":"promptly.r.01"},{"index":2,"sense_key":"promptly%4:02:00::","synset_id":"promptly.r.02"},{"index":3,"sense_key":"promptly%4:02:01::","synset_id":"promptly.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"promptly%4:02:01::","call_ids":["senseval2.d000.s038.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"promptly%4:02:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":684,"cached_input_tokens":0,"output_tokens":109,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002458,"input_uncached_usd":0.0013679999999999999,"input_cached_usd":0.0,"output_usd":0.00109,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4578292079968378}
{"item_id":"senseval2.d000.s038.t015","gold_sense_keys":["protest%1:04:00::"],"candidates":[{"index":1,"sense_key":"protest%1:10:00::","synset_id":"protest.n.01"},{"index":2,"sense_key":"protest%1:04:00::","synset_id":"protest.n.02"},{"index":3,"sense_key":"protest%1:10:01::","synset_id":"protest.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"protest%1:04:00::","call_ids":["senseval2.d000.s038.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"protest%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":705,"cached_input_tokens":0,"output_tokens":83,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0022400000000000002,"input_uncached_usd":0.00141,"input_cached_usd":0.0,"output_usd":0.0008300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3903893339447677}
{"item_id":"senseval2.d000.s039.t000","gold_sense_keys":["club%1:14:00::"],"candidates":[{"index":1,"sense_key":"club%1:14:01::","synset_id":"baseball_club.n.01"},{"index":2,"sense_key":"club%1:14:00::","synset_id":"club.n.02"},{"index":3,"sense_key":"club%1:06:00::","synset_id":"club.n.03"},{"index":4,"sense_key":"club%1:06:02::","synset_id":"clubhouse.n.01"},{"index":5,"sense_key":"club%1:06:01::","synset_id":"golf_club.n.02"},{"index":6,"sense_key":"club%1:06:03::","synset_id":"club.n.06"},{"index":7,"sense_key":"club%1:06:04::","synset_id":"cabaret.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"club%1:14:00::","call_ids":["senseval2.d000.s039.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"club%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1002,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0021239999999999996,"input_uncached_usd":0.0020039999999999997,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3993805419886485}
{"item_id":"senseval2.d000.s039.t001","gold_sense_keys":["treat%2:31:00::"],"candidates":[{"index":1,"sense_key":"treat%2:41:00::","synset_id":"treat.v.01"},{"index":2,"sense_key":"treat%2:30:01::","synset_id":"process.v.01"},{"index":3,"sense_key":"treat%2:29:00::","synset_id":"treat.v.03"},{"index":4,"sense_key":"treat%2:32:00::","synset_id":"cover.v.05"},{"index":5,"sense_key":"treat%2:40:00::","synset_id":"treat.v.05"},{"index":6,"sense_key":"treat%2:34:00::","synset_id":"regale.v.01"},{"index":7,"sense_key":"treat%2:32:03::","synset_id":"treat.v.07"},{"index":8,"sense_key":"treat%2:31:00::","synset_id":"treat.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"treat%2:31:00::","call_ids":["senseval2.d000.s039.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"treat%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1038,"cached_input_tokens":0,"output_tokens":107,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0031459999999999995,"input_uncached_usd":0.0020759999999999997,"input_cached_usd":0.0,"output_usd":0.00107,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5794890829129145}
{"item_id":"senseval2.d000.s039.t002","gold_sense_keys":["tower%1:06:00::"],"candidates":[{"index":1,"sense_key":"tower%1:06:00::","synset_id":"tower.n.01"},{"index":2,"sense_key":"tower%1:25:00::","synset_id":"column.n.04"},{"index":3,"sense_key":"tower%1:06:01::","synset_id":"tugboat.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"tower%1:06:00::","call_ids":["senseval2.d000.s039.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"tower%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":695,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00151,"input_uncached_usd":0.00139,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.033943415968679}
{"item_id":"senseval2.d000.s039.t003","gold_sense_keys":["sort%1:07:00::"],"candidates":[{"index":1,"sense_key":"sort%1:09:00::","synset_id":"kind.n.01"},{"index":2,"sense_key":"sort%1:07:00::","synset_id":"sort.n.02"},{"index":3,"sense_key":"sort%1:18:00::","synset_id":"sort.n.03"},{"index":4,"sense_key":"sort%1:22:00::","synset_id":"sort.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"sort%1:07:00::","call_ids":["senseval2.d000.s039.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"sort%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":777,"cached_input_tokens":0,"output_tokens":80,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0023539999999999998,"input_uncached_usd":0.0015539999999999998,"input_cached_usd":0.0,"output_usd":0.0008,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8515432080021128}
{"item_id":"senseval2.d000.s039.t004","gold_sense_keys":["separate%3:00:00::","separate%5:00:00:detached:00"],"candidates":[{"index":1,"sense_key":"separate%3:00:00::","synset_id":"separate.a.01"},{"index":2,"sense_key":"separate%5:00:00:detached:00","synset_id":"freestanding.s.01"},{"index":3,"sense_key":"separate%5:00:02:segregated:00","synset_id":"separate.s.02"},{"index":4,"sense_key":"separate%5:00:02:unconnected:00","synset_id":"disjoined.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"separate%5:00:00:detached:00","call_ids":["senseval2.d000.s039.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"separate%5:00:00:detached:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":751,"cached_input_tokens":0,"output_tokens":165,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0031520000000000003,"input_uncached_usd":0.0015019999999999999,"input_cached_usd":0.0,"output_usd":0.0016500000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.666710374993272}
{"item_id":"senseval2.d000.s039.t006","gold_sense_keys":["say%2:32:00::","say%2:32:15::"],"candidates":[{"index":1,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":6,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":9,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":10,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":11,"sense_key":"say%2:32:09::","synset_id":"say.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"say%2:32:00::","call_ids":["senseval2.d000.s039.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"say%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1171,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004422000000000001,"input_uncached_usd":0.002342,"input_cached_usd":0.0,"output_usd":0.0020800000000000003,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.0961677089799196}
{"item_id":"senseval2.d000.s040.t000","gold_sense_keys":["new%3:00:00::","new%5:00:00:other:00"],"candidates":[{"index":1,"sense_key":"new%3:00:00::","synset_id":"new.a.01"},{"index":2,"sense_key":"new%5:00:00:original:00","synset_id":"fresh.s.03"},{"index":3,"sense_key":"new%5:00:00:inexperienced:00","synset_id":"raw.s.11"},{"index":4,"sense_key":"new%5:00:00:unprecedented:00","synset_id":"new.s.03"},{"index":5,"sense_key":"new%5:00:00:other:00","synset_id":"new.s.04"},{"index":6,"sense_key":"new%3:00:09::","synset_id":"new.a.06"},{"index":7,"sense_key":"new%5:00:02:original:00","synset_id":"newfangled.s.01"},{"index":8,"sense_key":"new%5:00:02:late:01","synset_id":"new.s.06"},{"index":9,"sense_key":"new%5:00:01:late:01","synset_id":"modern.s.04"},{"index":10,"sense_key":"new%5:00:00:early:02","synset_id":"new.s.08"},{"index":11,"sense_key":"new%5:00:00:unaccustomed:00","synset_id":"new.s.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"new%5:00:00:other:00","call_ids":["senseval2.d000.s040.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"new%5:00:00:other:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1198,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0030759999999999997,"input_uncached_usd":0.0023959999999999997,"input_cached_usd":0.0,"output_usd":0.00068,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2489961669780314}
{"item_id":"senseval2.d000.s040.t002","gold_sense_keys":["today%1:28:00::"],"candidates":[{"index":1,"sense_key":"today%1:28:00::","synset_id":"today.n.01"},{"index":2,"sense_key":"today%1:28:01::","synset_id":"today.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"today%1:28:00::","call_ids":["senseval2.d000.s040.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"today%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":616,"cached_input_tokens":0,"output_tokens":82,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002052,"input_uncached_usd":0.001232,"input_cached_usd":0.0,"output_usd":0.0008200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3483897080877796}
{"item_id":"senseval2.d000.s040.t003","gold_sense_keys":["member%1:18:00::"],"candidates":[{"index":1,"sense_key":"member%1:18:00::","synset_id":"member.n.01"},{"index":2,"sense_key":"member%1:24:00::","synset_id":"member.n.02"},{"index":3,"sense_key":"member%1:08:00::","synset_id":"extremity.n.01"},{"index":4,"sense_key":"member%1:14:00::","synset_id":"member.n.04"},{"index":5,"sense_key":"member%1:08:01::","synset_id":"penis.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"member%1:18:00::","call_ids":["senseval2.d000.s040.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"member%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":837,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001794,"input_uncached_usd":0.001674,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.094164166948758}
{"item_id":"senseval2.d000.s040.t004","gold_sense_keys":["congregation%1:14:00::"],"candidates":[{"index":1,"sense_key":"congregation%1:14:00::","synset_id":"congregation.n.01"},{"index":2,"sense_key":"congregation%1:14:01::","synset_id":"congregation.n.02"},{"index":3,"sense_key":"congregation%1:04:00::","synset_id":"congregation.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"congregation%1:14:00::","call_ids":["senseval2.d000.s040.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"congregation%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":675,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00147,"input_uncached_usd":0.0013499999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1452168750111014}
{"item_id":"senseval2.d000.s041.t000","gold_sense_keys":["still%4:02:01::"],"candidates":[{"index":1,"sense_key":"still%4:02:01::","synset_id":"still.r.01"},{"index":2,"sense_key":"still%4:02:04::","synset_id":"however.r.01"},{"index":3,"sense_key":"still%4:02:00::","synset_id":"even.r.03"},{"index":4,"sense_key":"still%4:02:02::","synset_id":"still.r.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"still%4:02:01::","call_ids":["senseval2.d000.s041.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"still%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":807,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002294,"input_uncached_usd":0.001614,"input_cached_usd":0.0,"output_usd":0.00068,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5105756249977276}
{"item_id":"senseval2.d000.s041.t002","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"},{"index":2,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"},{"index":3,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":4,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s041.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":711,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001542,"input_uncached_usd":0.0014219999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9896244590636343}
{"item_id":"senseval2.d000.s041.t003","gold_sense_keys":["ring%2:39:01::"],"candidates":[{"index":1,"sense_key":"ring%2:39:00::","synset_id":"ring.v.01"},{"index":2,"sense_key":"ring%2:39:04::","synset_id":"resound.v.01"},{"index":3,"sense_key":"ring%2:39:01::","synset_id":"ring.v.03"},{"index":4,"sense_key":"ring%2:32:00::","synset_id":"call.v.03"},{"index":5,"sense_key":"ring%2:35:03::","synset_id":"surround.v.01"},{"index":6,"sense_key":"ring%2:35:00::","synset_id":"ring.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"ring%2:39:01::","call_ids":["senseval2.d000.s041.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"ring%2:39:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":912,"cached_input_tokens":0,"output_tokens":91,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002734,"input_uncached_usd":0.0018239999999999999,"input_cached_usd":0.0,"output_usd":0.0009100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5815152920549735}
{"item_id":"senseval2.d000.s041.t005","gold_sense_keys":["bell%1:06:00::"],"candidates":[{"index":1,"sense_key":"bell%1:06:00::","synset_id":"bell.n.01"},{"index":2,"sense_key":"bell%1:06:02::","synset_id":"doorbell.n.01"},{"index":3,"sense_key":"bell%1:11:00::","synset_id":"bell.n.03"},{"index":4,"sense_key":"bell%1:28:00::","synset_id":"bell.n.04"},{"index":5,"sense_key":"bell%1:25:00::","synset_id":"bell.n.05"},{"index":6,"sense_key":"bell%1:18:02::","synset_id":"bell.n.06"},{"index":7,"sense_key":"bell%1:18:01::","synset_id":"bell.n.07"},{"index":8,"sense_key":"bell%1:18:00::","synset_id":"bell.n.08"},{"index":9,"sense_key":"bell%1:06:04::","synset_id":"chime.n.01"},{"index":10,"sense_key":"bell%1:06:03::","synset_id":"bell.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bell%1:06:00::","call_ids":["senseval2.d000.s041.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bell%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1155,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00243,"input_uncached_usd":0.00231,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9778310409747064}
{"item_id":"senseval2.d000.s042.t000","gold_sense_keys":["bell%1:06:00::"],"candidates":[{"index":1,"sense_key":"bell%1:06:00::","synset_id":"bell.n.01"},{"index":2,"sense_key":"bell%1:06:02::","synset_id":"doorbell.n.01"},{"index":3,"sense_key":"bell%1:11:00::","synset_id":"bell.n.03"},{"index":4,"sense_key":"bell%1:28:00::","synset_id":"bell.n.04"},{"index":5,"sense_key":"bell%1:25:00::","synset_id":"bell.n.05"},{"index":6,"sense_key":"bell%1:18:02::","synset_id":"bell.n.06"},{"index":7,"sense_key":"bell%1:18:01::","synset_id":"bell.n.07"},{"index":8,"sense_key":"bell%1:18:00::","synset_id":"bell.n.08"},{"index":9,"sense_key":"bell%1:06:04::","synset_id":"chime.n.01"},{"index":10,"sense_key":"bell%1:06:03::","synset_id":"bell.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bell%1:06:00::","call_ids":["senseval2.d000.s042.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bell%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1178,"cached_input_tokens":0,"output_tokens":108,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003436,"input_uncached_usd":0.002356,"input_cached_usd":0.0,"output_usd":0.00108,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0128252080176026}
{"item_id":"senseval2.d000.s042.t002","gold_sense_keys":["silent%5:00:00:quiet:01"],"candidates":[{"index":1,"sense_key":"silent%5:00:00:quiet:01","synset_id":"silent.s.01"},{"index":2,"sense_key":"silent%5:00:00:uncommunicative:00","synset_id":"mum.s.01"},{"index":3,"sense_key":"silent%5:00:00:implicit:00","synset_id":"silent.s.03"},{"index":4,"sense_key":"silent%5:00:02:inaudible:00","synset_id":"silent.s.04"},{"index":5,"sense_key":"silent%5:00:00:inaudible:00","synset_id":"silent.s.05"},{"index":6,"sense_key":"silent%5:00:00:inarticulate:00","synset_id":"dumb.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"silent%5:00:00:quiet:01","call_ids":["senseval2.d000.s042.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"silent%5:00:00:quiet:01","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":918,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002366,"input_uncached_usd":0.001836,"input_cached_usd":0.0,"output_usd":0.0005300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7372154999757186}
{"item_id":"senseval2.d000.s042.t003","gold_sense_keys":["follow%2:42:01::"],"candidates":[{"index":1,"sense_key":"follow%2:38:00::","synset_id":"follow.v.01"},{"index":2,"sense_key":"follow%2:42:00::","synset_id":"postdate.v.01"},{"index":3,"sense_key":"follow%2:42:05::","synset_id":"follow.v.03"},{"index":4,"sense_key":"follow%2:38:01::","synset_id":"follow.v.04"},{"index":5,"sense_key":"follow%2:41:00::","synset_id":"comply.v.01"},{"index":6,"sense_key":"follow%2:42:01::","synset_id":"follow.v.06"},{"index":7,"sense_key":"follow%2:30:00::","synset_id":"follow.v.07"},{"index":8,"sense_key":"follow%2:42:02::","synset_id":"follow.v.08"},{"index":9,"sense_key":"follow%2:40:00::","synset_id":"adopt.v.01"},{"index":10,"sense_key":"follow%2:42:03::","synset_id":"follow.v.10"},{"index":11,"sense_key":"follow%2:36:00::","synset_id":"take_after.v.02"},{"index":12,"sense_key":"follow%2:31:00::","synset_id":"trace.v.01"},{"index":13,"sense_key":"follow%2:41:02::","synset_id":"watch.v.02"},{"index":14,"sense_key":"follow%2:41:01::","synset_id":"succeed.v.02"},{"index":15,"sense_key":"follow%2:36:01::","synset_id":"play_along.v.02"},{"index":16,"sense_key":"follow%2:30:01::","synset_id":"keep_up.v.04"},{"index":17,"sense_key":"follow%2:42:04::","synset_id":"come.v.05"},{"index":18,"sense_key":"follow%2:41:10::","synset_id":"follow.v.18"},{"index":19,"sense_key":"follow%2:41:11::","synset_id":"follow.v.19"},{"index":20,"sense_key":"follow%2:41:09::","synset_id":"be.v.08"},{"index":21,"sense_key":"follow%2:39:13::","synset_id":"surveil.v.01"},{"index":22,"sense_key":"follow%2:38:13::","synset_id":"pursue.v.02"},{"index":23,"sense_key":"follow%2:31:14::","synset_id":"follow.v.23"},{"index":24,"sense_key":"follow%2:30:12::","synset_id":"stick_to.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"follow%2:42:01::","call_ids":["senseval2.d000.s042.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"follow%2:42:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2058,"cached_input_tokens":0,"output_tokens":113,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.005246,"input_uncached_usd":0.004116,"input_cached_usd":0.0,"output_usd":0.0011300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.4957882500020787}
{"item_id":"senseval2.d000.s042.t005","gold_sense_keys":["church%1:04:00::"],"candidates":[{"index":1,"sense_key":"church%1:14:00::","synset_id":"church.n.01"},{"index":2,"sense_key":"church%1:06:00::","synset_id":"church.n.02"},{"index":3,"sense_key":"church%1:04:00::","synset_id":"church_service.n.01"},{"index":4,"sense_key":"church%1:14:01::","synset_id":"church.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"church%1:04:00::","call_ids":["senseval2.d000.s042.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"church%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":738,"cached_input_tokens":0,"output_tokens":119,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002666,"input_uncached_usd":0.0014759999999999999,"input_cached_usd":0.0,"output_usd":0.00119,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9709644169779494}
{"item_id":"senseval2.d000.s043.t000","gold_sense_keys":["vicar%1:18:01::"],"candidates":[{"index":1,"sense_key":"vicar%1:18:00::","synset_id":"vicar.n.01"},{"index":2,"sense_key":"vicar%1:18:02::","synset_id":"vicar.n.02"},{"index":3,"sense_key":"vicar%1:18:01::","synset_id":"vicar.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"vicar%1:18:01::","call_ids":["senseval2.d000.s043.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"vicar%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":671,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001872,"input_uncached_usd":0.0013419999999999999,"input_cached_usd":0.0,"output_usd":0.0005300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7372400839813054}
{"item_id":"senseval2.d000.s043.t001","gold_sense_keys":["refuse%2:32:00::"],"candidates":[{"index":1,"sense_key":"refuse%2:32:00::","synset_id":"refuse.v.01"},{"index":2,"sense_key":"refuse%2:40:00::","synset_id":"refuse.v.02"},{"index":3,"sense_key":"refuse%2:42:00::","synset_id":"defy.v.02"},{"index":4,"sense_key":"refuse%2:40:01::","synset_id":"deny.v.04"},{"index":5,"sense_key":"refuse%2:42:04::","synset_id":"resist.v.05"},{"index":6,"sense_key":"refuse%2:41:00::","synset_id":"reject.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"refuse%2:32:00::","call_ids":["senseval2.d000.s043.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"refuse%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":917,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002564,"input_uncached_usd":0.001834,"input_cached_usd":0.0,"output_usd":0.0007300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5587923330022022}
{"item_id":"senseval2.d000.s043.t002","gold_sense_keys":["talk_about%2:32:01::"],"candidates":[{"index":1,"sense_key":"talk_about%2:32:00::","synset_id":"discourse.v.01"},{"index":2,"sense_key":"talk_about%2:32:01::","synset_id":"talk_of.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"talk_about%2:32:01::","call_ids":["senseval2.d000.s043.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"talk_about%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":665,"cached_input_tokens":0,"output_tokens":117,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0025,"input_uncached_usd":0.00133,"input_cached_usd":0.0,"output_usd":0.00117,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.252531750011258}
{"item_id":"senseval2.d000.s043.t003","gold_sense_keys":["say%2:32:00::","say%2:32:15::"],"candidates":[{"index":1,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":6,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":9,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":10,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":11,"sense_key":"say%2:32:09::","synset_id":"say.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"say%2:32:00::","call_ids":["senseval2.d000.s043.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"say%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1188,"cached_input_tokens":0,"output_tokens":111,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0034860000000000004,"input_uncached_usd":0.002376,"input_cached_usd":0.0,"output_usd":0.00111,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7609229580266401}
{"item_id":"senseval2.d000.s044.t000","gold_sense_keys":["vicar%1:18:01::"],"candidates":[{"index":1,"sense_key":"vicar%1:18:00::","synset_id":"vicar.n.01"},{"index":2,"sense_key":"vicar%1:18:02::","synset_id":"vicar.n.02"},{"index":3,"sense_key":"vicar%1:18:01::","synset_id":"vicar.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"vicar%1:18:01::","call_ids":["senseval2.d000.s044.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"vicar%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":647,"cached_input_tokens":0,"output_tokens":49,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001784,"input_uncached_usd":0.001294,"input_cached_usd":0.0,"output_usd":0.0004900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4043781670043245}
{"item_id":"senseval2.d000.s044.t002","gold_sense_keys":["church%1:06:00::"],"candidates":[{"index":1,"sense_key":"church%1:14:00::","synset_id":"church.n.01"},{"index":2,"sense_key":"church%1:06:00::","synset_id":"church.n.02"},{"index":3,"sense_key":"church%1:04:00::","synset_id":"church_service.n.01"},{"index":4,"sense_key":"church%1:14:01::","synset_id":"church.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"church%1:06:00::","call_ids":["senseval2.d000.s044.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"church%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":712,"cached_input_tokens":0,"output_tokens":143,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0028539999999999998,"input_uncached_usd":0.001424,"input_cached_usd":0.0,"output_usd":0.00143,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.041540374979377}
{"item_id":"senseval2.d000.s044.t003","gold_sense_keys":["feel%2:31:00::"],"candidates":[{"index":1,"sense_key":"feel%2:37:00::","synset_id":"feel.v.01"},{"index":2,"sense_key":"feel%2:31:00::","synset_id":"find.v.05"},{"index":3,"sense_key":"feel%2:39:00::","synset_id":"feel.v.03"},{"index":4,"sense_key":"feel%2:29:00::","synset_id":"feel.v.04"},{"index":5,"sense_key":"feel%2:31:01::","synset_id":"feel.v.05"},{"index":6,"sense_key":"feel%2:39:01::","synset_id":"feel.v.06"},{"index":7,"sense_key":"feel%2:42:00::","synset_id":"feel.v.07"},{"index":8,"sense_key":"feel%2:39:10::","synset_id":"feel.v.08"},{"index":9,"sense_key":"feel%2:35:00::","synset_id":"feel.v.09"},{"index":10,"sense_key":"feel%2:35:01::","synset_id":"palpate.v.01"},{"index":11,"sense_key":"feel%2:40:00::","synset_id":"feel.v.11"},{"index":12,"sense_key":"feel%2:39:09::","synset_id":"feel.v.12"},{"index":13,"sense_key":"feel%2:35:11::","synset_id":"feel.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"feel%2:31:00::","call_ids":["senseval2.d000.s044.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"feel%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1296,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0027119999999999996,"input_uncached_usd":0.0025919999999999997,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.364752708002925}
{"item_id":"senseval2.d000.s044.t004","gold_sense_keys":["fault%1:07:01::"],"candidates":[{"index":1,"sense_key":"fault%1:04:00::","synset_id":"mistake.n.01"},{"index":2,"sense_key":"fault%1:26:00::","synset_id":"defect.n.03"},{"index":3,"sense_key":"fault%1:07:00::","synset_id":"demerit.n.02"},{"index":4,"sense_key":"fault%1:17:00::","synset_id":"fault.n.04"},{"index":5,"sense_key":"fault%1:11:00::","synset_id":"fault.n.05"},{"index":6,"sense_key":"fault%1:07:01::","synset_id":"fault.n.06"},{"index":7,"sense_key":"fault%1:04:01::","synset_id":"fault.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"fault%1:07:01::","call_ids":["senseval2.d000.s044.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"fault%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1041,"cached_input_tokens":0,"output_tokens":76,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002842,"input_uncached_usd":0.002082,"input_cached_usd":0.0,"output_usd":0.00076,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.5206569170113653}
{"item_id":"senseval2.d000.s044.t008","gold_sense_keys":["next%5:00:00:close:01"],"candidates":[{"index":1,"sense_key":"next%5:00:00:succeeding:00","synset_id":"following.s.02"},{"index":2,"sense_key":"next%5:00:00:close:01","synset_id":"adjacent.s.01"},{"index":3,"sense_key":"next%5:00:00:incoming:00","synset_id":"future.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"next%5:00:00:close:01","call_ids":["senseval2.d000.s044.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"next%5:00:00:close:01","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":692,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0015040000000000001,"input_uncached_usd":0.001384,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5153768749441952}
{"item_id":"senseval2.d000.s044.t009","gold_sense_keys":["altar%1:06:01::"],"candidates":[{"index":1,"sense_key":"altar%1:06:01::","synset_id":"altar.n.01"},{"index":2,"sense_key":"altar%1:06:00::","synset_id":"altar.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"altar%1:06:01::","call_ids":["senseval2.d000.s044.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"altar%1:06:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":599,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001798,"input_uncached_usd":0.0011979999999999998,"input_cached_usd":0.0,"output_usd":0.0006000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4649587919702753}
{"item_id":"senseval2.d000.s045.t000","gold_sense_keys":["so%4:02:10::"],"candidates":[{"index":1,"sense_key":"so%4:02:02::","synset_id":"so.r.01"},{"index":2,"sense_key":"so%4:02:06::","synset_id":"so.r.02"},{"index":3,"sense_key":"so%4:02:05::","synset_id":"so.r.03"},{"index":4,"sense_key":"so%4:02:07::","synset_id":"so.r.04"},{"index":5,"sense_key":"so%4:02:08::","synset_id":"so.r.05"},{"index":6,"sense_key":"so%4:02:04::","synset_id":"thus.r.02"},{"index":7,"sense_key":"so%4:02:03::","synset_id":"so.r.07"},{"index":8,"sense_key":"so%4:02:09::","synset_id":"then.r.01"},{"index":9,"sense_key":"so%4:02:10::","synset_id":"therefore.r.01"},{"index":10,"sense_key":"so%4:02:00::","synset_id":"indeed.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"so%4:02:09::","call_ids":["senseval2.d000.s045.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"so%4:02:09::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1166,"cached_input_tokens":0,"output_tokens":209,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004422000000000001,"input_uncached_usd":0.002332,"input_cached_usd":0.0,"output_usd":0.0020900000000000003,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.7652221670141444}
{"item_id":"senseval2.d000.s045.t007","gold_sense_keys":["here%4:02:01::"],"candidates":[{"index":1,"sense_key":"here%4:02:00::","synset_id":"here.r.01"},{"index":2,"sense_key":"here%4:02:02::","synset_id":"here.r.02"},{"index":3,"sense_key":"here%4:02:01::","synset_id":"here.r.03"},{"index":4,"sense_key":"here%4:02:04::","synset_id":"here.r.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"here%4:02:00::","call_ids":["senseval2.d000.s045.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"here%4:02:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":712,"cached_input_tokens":0,"output_tokens":257,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003994,"input_uncached_usd":0.001424,"input_cached_usd":0.0,"output_usd":0.0025700000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.952501874999143}
{"item_id":"senseval2.d000.s045.t008","gold_sense_keys":["come%2:38:00::","come%2:38:04::"],"candidates":[{"index":1,"sense_key":"come%2:38:00::","synset_id":"come.v.01"},{"index":2,"sense_key":"come%2:38:04::","synset_id":"arrive.v.01"},{"index":3,"sense_key":"come%2:30:01::","synset_id":"come.v.03"},{"index":4,"sense_key":"come%2:30:03::","synset_id":"come.v.04"},{"index":5,"sense_key":"come%2:42:04::","synset_id":"come.v.05"},{"index":6,"sense_key":"come%2:30:02::","synset_id":"come.v.06"},{"index":7,"sense_key":"come%2:42:13::","synset_id":"issue_forth.v.01"},{"index":8,"sense_key":"come%2:42:09::","synset_id":"hail.v.02"},{"index":9,"sense_key":"come%2:42:03::","synset_id":"come.v.09"},{"index":10,"sense_key":"come%2:42:05::","synset_id":"come.v.10"},{"index":11,"sense_key":"come%2:38:03::","synset_id":"come.v.11"},{"index":12,"sense_key":"come%2:42:00::","synset_id":"fall.v.04"},{"index":13,"sense_key":"come%2:30:13::","synset_id":"come.v.13"},{"index":14,"sense_key":"come%2:42:12::","synset_id":"total.v.01"},{"index":15,"sense_key":"come%2:42:11::","synset_id":"come.v.15"},{"index":16,"sense_key":"come%2:38:02::","synset_id":"come.v.16"},{"index":17,"sense_key":"come%2:31:14::","synset_id":"occur.v.02"},{"index":18,"sense_key":"come%2:42:15::","synset_id":"derive.v.05"},{"index":19,"sense_key":"come%2:42:02::","synset_id":"do.v.04"},{"index":20,"sense_key":"come%2:39:00::","synset_id":"come.v.20"},{"index":21,"sense_key":"come%2:31:13::","synset_id":"come.v.21"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"come%2:38:00::","call_ids":["senseval2.d000.s045.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"come%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1679,"cached_input_tokens":0,"output_tokens":113,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004488,"input_uncached_usd":0.003358,"input_cached_usd":0.0,"output_usd":0.0011300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1170009589986876}
{"item_id":"senseval2.d000.s045.t009","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"},{"index":2,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"},{"index":3,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":4,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s045.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":708,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001536,"input_uncached_usd":0.001416,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1430210419930518}
{"item_id":"senseval2.d000.s045.t010","gold_sense_keys":["above%4:02:01::"],"candidates":[{"index":1,"sense_key":"above%4:02:00::","synset_id":"above.r.01"},{"index":2,"sense_key":"above%4:02:01::","synset_id":"above.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"above%4:02:01::","call_ids":["senseval2.d000.s045.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"above%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":596,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001312,"input_uncached_usd":0.001192,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0866719590267166}
{"item_id":"senseval2.d000.s045.t011","gold_sense_keys":["make%2:36:12::"],"candidates":[{"index":1,"sense_key":"make%2:41:00::","synset_id":"make.v.01"},{"index":2,"sense_key":"make%2:30:00::","synset_id":"make.v.02"},{"index":3,"sense_key":"make%2:36:00::","synset_id":"make.v.03"},{"index":4,"sense_key":"make%2:32:00::","synset_id":"induce.v.02"},{"index":5,"sense_key":"make%2:36:08::","synset_id":"cause.v.01"},{"index":6,"sense_key":"make%2:36:01::","synset_id":"produce.v.02"},{"index":7,"sense_key":"make%2:31:00::","synset_id":"draw.v.04"},{"index":8,"sense_key":"make%2:36:15::","synset_id":"make.v.08"},{"index":9,"sense_key":"make%2:36:09::","synset_id":"create.v.05"},{"index":10,"sense_key":"make%2:40:01::","synset_id":"gain.v.08"},{"index":11,"sense_key":"make%2:36:05::","synset_id":"do.v.08"},{"index":12,"sense_key":"make%2:42:05::","synset_id":"form.v.02"},{"index":13,"sense_key":"make%2:38:02::","synset_id":"reach.v.07"},{"index":14,"sense_key":"make%2:42:00::","synset_id":"make.v.14"},{"index":15,"sense_key":"make%2:36:11::","synset_id":"make.v.15"},{"index":16,"sense_key":"make%2:36:12::","synset_id":"make.v.16"},{"index":17,"sense_key":"make%2:36:13::","synset_id":"construct.v.01"},{"index":18,"sense_key":"make%2:30:15::","synset_id":"make.v.18"},{"index":19,"sense_key":"make%2:29:08::","synset_id":"make.v.19"},{"index":20,"sense_key":"make%2:41:03::","synset_id":"name.v.03"},{"index":21,"sense_key":"make%2:40:02::","synset_id":"have.v.17"},{"index":22,"sense_key":"make%2:38:00::","synset_id":"reach.v.01"},{"index":23,"sense_key":"make%2:31:13::","synset_id":"lay_down.v.01"},{"index":24,"sense_key":"make%2:41:13::","synset_id":"make.v.24"},{"index":25,"sense_key":"make%2:41:01::","synset_id":"make.v.25"},{"index":26,"sense_key":"make%2:36:04::","synset_id":"hold.v.03"},{"index":27,"sense_key":"make%2:30:02::","synset_id":"make.v.27"},{"index":28,"sense_key":"make%2:38:05::","synset_id":"take.v.27"},{"index":29,"sense_key":"make%2:29:00::","synset_id":"stool.v.04"},{"index":30,"sense_key":"make%2:42:09::","synset_id":"make.v.30"},{"index":31,"sense_key":"make%2:42:13::","synset_id":"make.v.31"},{"index":32,"sense_key":"make%2:42:12::","synset_id":"make.v.32"},{"index":33,"sense_key":"make%2:42:11::","synset_id":"make.v.33"},{"index":34,"sense_key":"make%2:42:15::","synset_id":"make.v.34"},{"index":35,"sense_key":"make%2:39:12::","synset_id":"make.v.35"},{"index":36,"sense_key":"make%2:38:10::","synset_id":"make.v.36"},{"index":37,"sense_key":"make%2:38:15::","synset_id":"make.v.37"},{"index":38,"sense_key":"make%2:36:10::","synset_id":"make.v.38"},{"index":39,"sense_key":"make%2:36:07::","synset_id":"cook.v.02"},{"index":40,"sense_key":"make%2:35:13::","synset_id":"seduce.v.01"},{"index":41,"sense_key":"make%2:32:14::","synset_id":"make.v.41"},{"index":42,"sense_key":"make%2:32:09::","synset_id":"make.v.42"},{"index":43,"sense_key":"make%2:31:10::","synset_id":"make.v.43"},{"index":44,"sense_key":"make%2:31:12::","synset_id":"make.v.44"},{"index":45,"sense_key":"make%2:30:10::","synset_id":"make.v.45"},{"index":46,"sense_key":"make%2:30:14::","synset_id":"make.v.46"},{"index":47,"sense_key":"make%2:30:12::","synset_id":"make.v.47"},{"index":48,"sense_key":"make%2:29:06::","synset_id":"make.v.48"},{"index":49,"sense_key":"make%2:29:01::","synset_id":"make.v.49"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":16,"chosen_sense_key":"make%2:36:12::","call_ids":["senseval2.d000.s045.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":16,"predicted_sense_key":"make%2:36:12::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":3171,"cached_input_tokens":0,"output_tokens":156,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.007902,"input_uncached_usd":0.0063419999999999995,"input_cached_usd":0.0,"output_usd":0.0015600000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.682267124997452}
{"item_id":"senseval2.d000.s045.t012","gold_sense_keys":["very%4:02:00::"],"candidates":[{"index":1,"sense_key":"very%4:02:00::","synset_id":"very.r.01"},{"index":2,"sense_key":"very%4:02:01::","synset_id":"very.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"very%4:02:00::","call_ids":["senseval2.d000.s045.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"very%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":628,"cached_input_tokens":0,"output_tokens":95,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002206,"input_uncached_usd":0.001256,"input_cached_usd":0.0,"output_usd":0.0009500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":8.722384667024016}
{"item_id":"senseval2.d000.s045.t014","gold_sense_keys":["exit%1:04:00::"],"candidates":[{"index":1,"sense_key":"exit%1:06:00::","synset_id":"exit.n.01"},{"index":2,"sense_key":"exit%1:11:00::","synset_id":"passing.n.02"},{"index":3,"sense_key":"exit%1:04:00::","synset_id":"exit.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"exit%1:04:00::","call_ids":["senseval2.d000.s045.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"exit%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":668,"cached_input_tokens":0,"output_tokens":49,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001826,"input_uncached_usd":0.001336,"input_cached_usd":0.0,"output_usd":0.0004900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2916274999734014}
{"item_id":"senseval2.d000.s045.t015","gold_sense_keys":["congregation%1:14:00::"],"candidates":[{"index":1,"sense_key":"congregation%1:14:00::","synset_id":"congregation.n.01"},{"index":2,"sense_key":"congregation%1:14:01::","synset_id":"congregation.n.02"},{"index":3,"sense_key":"congregation%1:04:00::","synset_id":"congregation.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"congregation%1:14:00::","call_ids":["senseval2.d000.s045.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"congregation%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":672,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001464,"input_uncached_usd":0.001344,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.7729115420952439}
{"item_id":"senseval2.d000.s045.t016","gold_sense_keys":["prayer%1:04:00::"],"candidates":[{"index":1,"sense_key":"prayer%1:04:00::","synset_id":"prayer.n.01"},{"index":2,"sense_key":"prayer%1:10:02::","synset_id":"prayer.n.02"},{"index":3,"sense_key":"prayer%1:10:00::","synset_id":"entreaty.n.01"},{"index":4,"sense_key":"prayer%1:10:01::","synset_id":"prayer.n.04"},{"index":5,"sense_key":"prayer%1:18:00::","synset_id":"prayer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"prayer%1:04:00::","call_ids":["senseval2.d000.s045.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"prayer%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":774,"cached_input_tokens":0,"output_tokens":76,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002308,"input_uncached_usd":0.001548,"input_cached_usd":0.0,"output_usd":0.00076,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5402098749764264}
{"item_id":"senseval2.d000.s045.t017","gold_sense_keys":["say%2:32:00::","say%2:32:15::"],"candidates":[{"index":1,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":6,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":9,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":10,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":11,"sense_key":"say%2:32:09::","synset_id":"say.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"say%2:32:00::","call_ids":["senseval2.d000.s045.t017__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"say%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1163,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003056,"input_uncached_usd":0.002326,"input_cached_usd":0.0,"output_usd":0.0007300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5688169159693643}
{"item_id":"senseval2.d000.s046.t000","gold_sense_keys":["admit%2:32:00::"],"candidates":[{"index":1,"sense_key":"admit%2:32:00::","synset_id":"admit.v.01"},{"index":2,"sense_key":"admit%2:41:01::","synset_id":"admit.v.02"},{"index":3,"sense_key":"admit%2:41:00::","synset_id":"admit.v.03"},{"index":4,"sense_key":"admit%2:40:00::","synset_id":"accept.v.05"},{"index":5,"sense_key":"admit%2:42:06::","synset_id":"admit.v.05"},{"index":6,"sense_key":"admit%2:42:00::","synset_id":"admit.v.06"},{"index":7,"sense_key":"admit%2:42:04::","synset_id":"accommodate.v.04"},{"index":8,"sense_key":"admit%2:42:03::","synset_id":"admit.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"admit%2:32:00::","call_ids":["senseval2.d000.s046.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"admit%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1042,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002204,"input_uncached_usd":0.002084,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0074345839675516}
{"item_id":"senseval2.d000.s046.t001","gold_sense_keys":["mixed%5:00:00:heterogeneous:00"],"candidates":[{"index":1,"sense_key":"mixed%5:00:00:heterogeneous:00","synset_id":"assorted.s.01"},{"index":2,"sense_key":"mixed%5:00:00:integrated:00","synset_id":"interracial.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"mixed%5:00:00:heterogeneous:00","call_ids":["senseval2.d000.s046.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"mixed%5:00:00:heterogeneous:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":680,"cached_input_tokens":0,"output_tokens":85,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00221,"input_uncached_usd":0.0013599999999999999,"input_cached_usd":0.0,"output_usd":0.0008500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7531924999784678}
{"item_id":"senseval2.d000.s046.t002","gold_sense_keys":["feeling%1:03:00::","feeling%1:09:03::"],"candidates":[{"index":1,"sense_key":"feeling%1:03:00::","synset_id":"feeling.n.01"},{"index":2,"sense_key":"feeling%1:09:03::","synset_id":"impression.n.01"},{"index":3,"sense_key":"feeling%1:26:00::","synset_id":"spirit.n.02"},{"index":4,"sense_key":"feeling%1:09:01::","synset_id":"feeling.n.04"},{"index":5,"sense_key":"feeling%1:09:02::","synset_id":"touch.n.10"},{"index":6,"sense_key":"feeling%1:09:00::","synset_id":"feeling.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"feeling%1:09:03::","call_ids":["senseval2.d000.s046.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"feeling%1:09:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":960,"cached_input_tokens":0,"output_tokens":112,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00304,"input_uncached_usd":0.0019199999999999998,"input_cached_usd":0.0,"output_usd":0.0011200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6605329168960452}
{"item_id":"senseval2.d000.s046.t003","gold_sense_keys":["issue%1:09:00::","issue%1:09:01::"],"candidates":[{"index":1,"sense_key":"issue%1:09:01::","synset_id":"issue.n.01"},{"index":2,"sense_key":"issue%1:10:00::","synset_id":"issue.n.02"},{"index":3,"sense_key":"issue%1:09:00::","synset_id":"topic.n.02"},{"index":4,"sense_key":"issue%1:04:01::","synset_id":"issue.n.04"},{"index":5,"sense_key":"issue%1:21:01::","synset_id":"issue.n.05"},{"index":6,"sense_key":"issue%1:21:00::","synset_id":"return.n.06"},{"index":7,"sense_key":"issue%1:19:00::","synset_id":"consequence.n.01"},{"index":8,"sense_key":"issue%1:18:00::","synset_id":"offspring.n.01"},{"index":9,"sense_key":"issue%1:11:00::","synset_id":"emergence.n.02"},{"index":10,"sense_key":"issue%1:06:00::","synset_id":"exit.n.01"},{"index":11,"sense_key":"issue%1:04:00::","synset_id":"issue.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"issue%1:09:01::","call_ids":["senseval2.d000.s046.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"issue%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1276,"cached_input_tokens":0,"output_tokens":136,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0039120000000000005,"input_uncached_usd":0.002552,"input_cached_usd":0.0,"output_usd":0.00136,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7930695420363918}
{"item_id":"senseval2.d000.s046.t004","gold_sense_keys":["vicar%1:18:01::"],"candidates":[{"index":1,"sense_key":"vicar%1:18:00::","synset_id":"vicar.n.01"},{"index":2,"sense_key":"vicar%1:18:02::","synset_id":"vicar.n.02"},{"index":3,"sense_key":"vicar%1:18:01::","synset_id":"vicar.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"vicar%1:18:01::","call_ids":["senseval2.d000.s046.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"vicar%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":640,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0019099999999999998,"input_uncached_usd":0.0012799999999999999,"input_cached_usd":0.0,"output_usd":0.00063,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7364752079593018}
{"item_id":"senseval2.d000.s046.t005","gold_sense_keys":["active%5:00:00:involved:00"],"candidates":[{"index":1,"sense_key":"active%3:00:02::","synset_id":"active.a.01"},{"index":2,"sense_key":"active%5:00:00:operational:00","synset_id":"active.s.01"},{"index":3,"sense_key":"active%3:00:03::","synset_id":"active.a.03"},{"index":4,"sense_key":"active%5:00:00:involved:00","synset_id":"active.s.02"},{"index":5,"sense_key":"active%3:00:01::","synset_id":"active.a.05"},{"index":6,"sense_key":"active%3:00:07::","synset_id":"active.a.06"},{"index":7,"sense_key":"active%3:00:06::","synset_id":"active.a.07"},{"index":8,"sense_key":"active%5:00:00:existent:00","synset_id":"active.s.03"},{"index":9,"sense_key":"active%3:00:10::","synset_id":"active.a.09"},{"index":10,"sense_key":"active%3:00:09::","synset_id":"active.a.10"},{"index":11,"sense_key":"active%3:00:00::","synset_id":"active.a.11"},{"index":12,"sense_key":"active%3:00:05::","synset_id":"active.a.12"},{"index":13,"sense_key":"active%3:00:04::","synset_id":"active.a.13"},{"index":14,"sense_key":"active%3:00:08::","synset_id":"active.a.14"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"active%5:00:00:involved:00","call_ids":["senseval2.d000.s046.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"active%5:00:00:involved:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1254,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003188,"input_uncached_usd":0.002508,"input_cached_usd":0.0,"output_usd":0.00068,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9150760000338778}
{"item_id":"senseval2.d000.s046.t006","gold_sense_keys":["bell_ringer%1:18:01::"],"candidates":[{"index":1,"sense_key":"bell_ringer%1:18:01::","synset_id":"toller.n.01"},{"index":2,"sense_key":"bell_ringer%1:18:00::","synset_id":"bell_ringer.n.02"},{"index":3,"sense_key":"bell_ringer%1:04:00::","synset_id":"bell_ringer.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bell_ringer%1:18:01::","call_ids":["senseval2.d000.s046.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bell_ringer%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":687,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001494,"input_uncached_usd":0.001374,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.141374500002712}
{"item_id":"senseval2.d000.s047.t000","gold_sense_keys":["sound%1:07:00::"],"candidates":[{"index":1,"sense_key":"sound%1:07:00::","synset_id":"sound.n.01"},{"index":2,"sense_key":"sound%1:09:00::","synset_id":"sound.n.02"},{"index":3,"sense_key":"sound%1:19:00::","synset_id":"sound.n.03"},{"index":4,"sense_key":"sound%1:11:00::","synset_id":"sound.n.04"},{"index":5,"sense_key":"sound%1:10:00::","synset_id":"audio.n.01"},{"index":6,"sense_key":"sound%1:10:01::","synset_id":"phone.n.02"},{"index":7,"sense_key":"sound%1:17:01::","synset_id":"strait.n.01"},{"index":8,"sense_key":"sound%1:17:00::","synset_id":"sound.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"sound%1:07:00::","call_ids":["senseval2.d000.s047.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"sound%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":972,"cached_input_tokens":0,"output_tokens":75,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002694,"input_uncached_usd":0.001944,"input_cached_usd":0.0,"output_usd":0.00075,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4168227079790086}
{"item_id":"senseval2.d000.s047.t001","gold_sense_keys":["bell%1:06:00::"],"candidates":[{"index":1,"sense_key":"bell%1:06:00::","synset_id":"bell.n.01"},{"index":2,"sense_key":"bell%1:06:02::","synset_id":"doorbell.n.01"},{"index":3,"sense_key":"bell%1:11:00::","synset_id":"bell.n.03"},{"index":4,"sense_key":"bell%1:28:00::","synset_id":"bell.n.04"},{"index":5,"sense_key":"bell%1:25:00::","synset_id":"bell.n.05"},{"index":6,"sense_key":"bell%1:18:02::","synset_id":"bell.n.06"},{"index":7,"sense_key":"bell%1:18:01::","synset_id":"bell.n.07"},{"index":8,"sense_key":"bell%1:18:00::","synset_id":"bell.n.08"},{"index":9,"sense_key":"bell%1:06:04::","synset_id":"chime.n.01"},{"index":10,"sense_key":"bell%1:06:03::","synset_id":"bell.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bell%1:06:00::","call_ids":["senseval2.d000.s047.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bell%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1150,"cached_input_tokens":0,"output_tokens":140,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0037,"input_uncached_usd":0.0023,"input_cached_usd":0.0,"output_usd":0.0014000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0248266250127926}
{"item_id":"senseval2.d000.s047.t002","gold_sense_keys":["draw%2:35:06::"],"candidates":[{"index":1,"sense_key":"draw%2:35:03::","synset_id":"pull.v.01"},{"index":2,"sense_key":"draw%2:40:00::","synset_id":"reap.v.02"},{"index":3,"sense_key":"draw%2:35:01::","synset_id":"trace.v.02"},{"index":4,"sense_key":"draw%2:31:00::","synset_id":"draw.v.04"},{"index":5,"sense_key":"draw%2:38:01::","synset_id":"draw.v.05"},{"index":6,"sense_key":"draw%2:36:00::","synset_id":"draw.v.06"},{"index":7,"sense_key":"draw%2:38:00::","synset_id":"draw.v.07"},{"index":8,"sense_key":"draw%2:32:00::","synset_id":"describe.v.01"},{"index":9,"sense_key":"draw%2:31:13::","synset_id":"draw.v.09"},{"index":10,"sense_key":"draw%2:37:01::","synset_id":"draw.v.10"},{"index":11,"sense_key":"draw%2:34:01::","synset_id":"puff.v.02"},{"index":12,"sense_key":"draw%2:38:02::","synset_id":"draw.v.12"},{"index":13,"sense_key":"draw%2:40:01::","synset_id":"withdraw.v.09"},{"index":14,"sense_key":"draw%2:33:01::","synset_id":"draw.v.14"},{"index":15,"sense_key":"draw%2:36:01::","synset_id":"draw.v.15"},{"index":16,"sense_key":"draw%2:30:14::","synset_id":"draw.v.16"},{"index":17,"sense_key":"draw%2:38:07::","synset_id":"draw.v.17"},{"index":18,"sense_key":"draw%2:36:07::","synset_id":"draw.v.18"},{"index":19,"sense_key":"draw%2:36:02::","synset_id":"draw.v.19"},{"index":20,"sense_key":"draw%2:35:02::","synset_id":"draw.v.20"},{"index":21,"sense_key":"draw%2:42:00::","synset_id":"draw.v.21"},{"index":22,"sense_key":"draw%2:42:02::","synset_id":"draw.v.22"},{"index":23,"sense_key":"draw%2:41:13::","synset_id":"draw.v.23"},{"index":24,"sense_key":"draw%2:38:11::","synset_id":"pull.v.09"},{"index":25,"sense_key":"draw%2:35:04::","synset_id":"absorb.v.04"},{"index":26,"sense_key":"draw%2:35:06::","synset_id":"attract.v.01"},{"index":27,"sense_key":"draw%2:35:14::","synset_id":"string.v.01"},{"index":28,"sense_key":"draw%2:35:09::","synset_id":"pull_back.v.04"},{"index":29,"sense_key":"draw%2:35:13::","synset_id":"guide.v.05"},{"index":30,"sense_key":"draw%2:33:00::","synset_id":"tie.v.02"},{"index":31,"sense_key":"draw%2:30:00::","synset_id":"draw.v.31"},{"index":32,"sense_key":"draw%2:30:08::","synset_id":"draw.v.32"},{"index":33,"sense_key":"draw%2:30:15::","synset_id":"draw.v.33"},{"index":34,"sense_key":"draw%2:30:05::","synset_id":"disembowel.v.01"},{"index":35,"sense_key":"draw%2:30:04::","synset_id":"draw.v.35"},{"index":36,"sense_key":"draw%2:29:00::","synset_id":"draw.v.36"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":26,"chosen_sense_key":"draw%2:35:06::","call_ids":["senseval2.d000.s047.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":26,"predicted_sense_key":"draw%2:35:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2686,"cached_input_tokens":0,"output_tokens":150,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.006872,"input_uncached_usd":0.005372,"input_cached_usd":0.0,"output_usd":0.0015,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8431203330401331}
{"item_id":"senseval2.d000.s047.t003","gold_sense_keys":["people%1:14:00::"],"candidates":[{"index":1,"sense_key":"people%1:14:00::","synset_id":"people.n.01"},{"index":2,"sense_key":"people%1:14:01::","synset_id":"citizenry.n.01"},{"index":3,"sense_key":"people%1:14:02::","synset_id":"people.n.03"},{"index":4,"sense_key":"people%1:14:03::","synset_id":"multitude.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"people%1:14:00::","call_ids":["senseval2.d000.s047.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"people%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":736,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001592,"input_uncached_usd":0.001472,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9995804580394179}
{"item_id":"senseval2.d000.s047.t004","gold_sense_keys":["church%1:06:00::"],"candidates":[{"index":1,"sense_key":"church%1:14:00::","synset_id":"church.n.01"},{"index":2,"sense_key":"church%1:06:00::","synset_id":"church.n.02"},{"index":3,"sense_key":"church%1:04:00::","synset_id":"church_service.n.01"},{"index":4,"sense_key":"church%1:14:01::","synset_id":"church.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"church%1:06:00::","call_ids":["senseval2.d000.s047.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"church%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":710,"cached_input_tokens":0,"output_tokens":116,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00258,"input_uncached_usd":0.00142,"input_cached_usd":0.0,"output_usd":0.00116,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8059126250445843}
{"item_id":"senseval2.d000.s047.t005","gold_sense_keys":["say%2:32:00::","say%2:32:15::"],"candidates":[{"index":1,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":6,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":9,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":10,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":11,"sense_key":"say%2:32:09::","synset_id":"say.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"say%2:32:00::","call_ids":["senseval2.d000.s047.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"say%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1161,"cached_input_tokens":0,"output_tokens":104,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003362,"input_uncached_usd":0.002322,"input_cached_usd":0.0,"output_usd":0.0010400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.457540791016072}
{"item_id":"senseval2.d000.s048.t000","gold_sense_keys":["live%2:42:06::"],"candidates":[{"index":1,"sense_key":"live%2:42:08::","synset_id":"populate.v.01"},{"index":2,"sense_key":"live%2:42:06::","synset_id":"live.v.02"},{"index":3,"sense_key":"live%2:42:04::","synset_id":"survive.v.01"},{"index":4,"sense_key":"live%2:42:01::","synset_id":"exist.v.02"},{"index":5,"sense_key":"live%2:42:00::","synset_id":"be.v.11"},{"index":6,"sense_key":"live%2:31:00::","synset_id":"know.v.05"},{"index":7,"sense_key":"live%2:42:07::","synset_id":"live.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"live%2:42:06::","call_ids":["senseval2.d000.s048.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"live%2:42:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":998,"cached_input_tokens":0,"output_tokens":78,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002776,"input_uncached_usd":0.001996,"input_cached_usd":0.0,"output_usd":0.0007800000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.558576958021149}
{"item_id":"senseval2.d000.s048.t001","gold_sense_keys":["hope%1:12:01::"],"candidates":[{"index":1,"sense_key":"hope%1:12:01::","synset_id":"hope.n.01"},{"index":2,"sense_key":"hope%1:12:00::","synset_id":"hope.n.02"},{"index":3,"sense_key":"hope%1:09:00::","synset_id":"promise.n.02"},{"index":4,"sense_key":"hope%1:18:00::","synset_id":"hope.n.04"},{"index":5,"sense_key":"hope%1:18:01::","synset_id":"hope.n.05"},{"index":6,"sense_key":"hope%1:07:00::","synset_id":"hope.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"hope%1:12:01::","call_ids":["senseval2.d000.s048.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"hope%1:12:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":844,"cached_input_tokens":0,"output_tokens":119,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002878,"input_uncached_usd":0.0016879999999999998,"input_cached_usd":0.0,"output_usd":0.00119,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.776091375038959}
{"item_id":"senseval2.d000.s048.t002","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"},{"index":2,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"},{"index":3,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":4,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s048.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":718,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001556,"input_uncached_usd":0.001436,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1961025830823928}
{"item_id":"senseval2.d000.s048.t003","gold_sense_keys":["draw%2:30:14::"],"candidates":[{"index":1,"sense_key":"draw%2:35:03::","synset_id":"pull.v.01"},{"index":2,"sense_key":"draw%2:40:00::","synset_id":"reap.v.02"},{"index":3,"sense_key":"draw%2:35:01::","synset_id":"trace.v.02"},{"index":4,"sense_key":"draw%2:31:00::","synset_id":"draw.v.04"},{"index":5,"sense_key":"draw%2:38:01::","synset_id":"draw.v.05"},{"index":6,"sense_key":"draw%2:36:00::","synset_id":"draw.v.06"},{"index":7,"sense_key":"draw%2:38:00::","synset_id":"draw.v.07"},{"index":8,"sense_key":"draw%2:32:00::","synset_id":"describe.v.01"},{"index":9,"sense_key":"draw%2:31:13::","synset_id":"draw.v.09"},{"index":10,"sense_key":"draw%2:37:01::","synset_id":"draw.v.10"},{"index":11,"sense_key":"draw%2:34:01::","synset_id":"puff.v.02"},{"index":12,"sense_key":"draw%2:38:02::","synset_id":"draw.v.12"},{"index":13,"sense_key":"draw%2:40:01::","synset_id":"withdraw.v.09"},{"index":14,"sense_key":"draw%2:33:01::","synset_id":"draw.v.14"},{"index":15,"sense_key":"draw%2:36:01::","synset_id":"draw.v.15"},{"index":16,"sense_key":"draw%2:30:14::","synset_id":"draw.v.16"},{"index":17,"sense_key":"draw%2:38:07::","synset_id":"draw.v.17"},{"index":18,"sense_key":"draw%2:36:07::","synset_id":"draw.v.18"},{"index":19,"sense_key":"draw%2:36:02::","synset_id":"draw.v.19"},{"index":20,"sense_key":"draw%2:35:02::","synset_id":"draw.v.20"},{"index":21,"sense_key":"draw%2:42:00::","synset_id":"draw.v.21"},{"index":22,"sense_key":"draw%2:42:02::","synset_id":"draw.v.22"},{"index":23,"sense_key":"draw%2:41:13::","synset_id":"draw.v.23"},{"index":24,"sense_key":"draw%2:38:11::","synset_id":"pull.v.09"},{"index":25,"sense_key":"draw%2:35:04::","synset_id":"absorb.v.04"},{"index":26,"sense_key":"draw%2:35:06::","synset_id":"attract.v.01"},{"index":27,"sense_key":"draw%2:35:14::","synset_id":"string.v.01"},{"index":28,"sense_key":"draw%2:35:09::","synset_id":"pull_back.v.04"},{"index":29,"sense_key":"draw%2:35:13::","synset_id":"guide.v.05"},{"index":30,"sense_key":"draw%2:33:00::","synset_id":"tie.v.02"},{"index":31,"sense_key":"draw%2:30:00::","synset_id":"draw.v.31"},{"index":32,"sense_key":"draw%2:30:08::","synset_id":"draw.v.32"},{"index":33,"sense_key":"draw%2:30:15::","synset_id":"draw.v.33"},{"index":34,"sense_key":"draw%2:30:05::","synset_id":"disembowel.v.01"},{"index":35,"sense_key":"draw%2:30:04::","synset_id":"draw.v.35"},{"index":36,"sense_key":"draw%2:29:00::","synset_id":"draw.v.36"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":16,"chosen_sense_key":"draw%2:30:14::","call_ids":["senseval2.d000.s048.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":16,"predicted_sense_key":"draw%2:30:14::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2699,"cached_input_tokens":0,"output_tokens":155,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.006948,"input_uncached_usd":0.005398,"input_cached_usd":0.0,"output_usd":0.0015500000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9083415829809383}
{"item_id":"senseval2.d000.s048.t004","gold_sense_keys":["full%3:00:00::"],"candidates":[{"index":1,"sense_key":"full%3:00:00::","synset_id":"full.a.01"},{"index":2,"sense_key":"full%5:00:01:whole:00","synset_id":"entire.s.01"},{"index":3,"sense_key":"full%5:00:00:complete:00","synset_id":"full.s.02"},{"index":4,"sense_key":"full%5:00:00:nourished:00","synset_id":"full.s.03"},{"index":5,"sense_key":"full%3:00:01::","synset_id":"full.a.05"},{"index":6,"sense_key":"full%5:00:00:ample:00","synset_id":"full.s.04"},{"index":7,"sense_key":"full%5:00:00:high:02","synset_id":"broad.s.04"},{"index":8,"sense_key":"full%5:00:02:ample:00","synset_id":"wide.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"full%5:00:00:complete:00","call_ids":["senseval2.d000.s048.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"full%5:00:00:complete:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":975,"cached_input_tokens":0,"output_tokens":112,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00307,"input_uncached_usd":0.00195,"input_cached_usd":0.0,"output_usd":0.0011200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9280640829820186}
{"item_id":"senseval2.d000.s048.t005","gold_sense_keys":["life%1:26:01::"],"candidates":[{"index":1,"sense_key":"life%1:26:01::","synset_id":"life.n.01"},{"index":2,"sense_key":"life%1:09:00::","synset_id":"life.n.02"},{"index":3,"sense_key":"life%1:26:02::","synset_id":"life.n.03"},{"index":4,"sense_key":"life%1:26:00::","synset_id":"animation.n.01"},{"index":5,"sense_key":"life%1:28:00::","synset_id":"life.n.05"},{"index":6,"sense_key":"life%1:28:01::","synset_id":"life.n.06"},{"index":7,"sense_key":"life%1:28:02::","synset_id":"life.n.07"},{"index":8,"sense_key":"life%1:18:00::","synset_id":"life.n.08"},{"index":9,"sense_key":"life%1:07:01::","synset_id":"liveliness.n.02"},{"index":10,"sense_key":"life%1:03:00::","synset_id":"life.n.10"},{"index":11,"sense_key":"life%1:19:00::","synset_id":"life.n.11"},{"index":12,"sense_key":"life%1:10:00::","synset_id":"biography.n.01"},{"index":13,"sense_key":"life%1:16:00::","synset_id":"life.n.13"},{"index":14,"sense_key":"life%1:28:03::","synset_id":"life_sentence.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"life%1:26:01::","call_ids":["senseval2.d000.s048.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"life%1:26:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1275,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0031599999999999996,"input_uncached_usd":0.0025499999999999997,"input_cached_usd":0.0,"output_usd":0.0006100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.301925125066191}
{"item_id":"senseval2.d000.s049.t000","gold_sense_keys":["sort%1:07:00::"],"candidates":[{"index":1,"sense_key":"sort%1:09:00::","synset_id":"kind.n.01"},{"index":2,"sense_key":"sort%1:07:00::","synset_id":"sort.n.02"},{"index":3,"sense_key":"sort%1:18:00::","synset_id":"sort.n.03"},{"index":4,"sense_key":"sort%1:22:00::","synset_id":"sort.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"sort%1:07:00::","call_ids":["senseval2.d000.s049.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"sort%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":804,"cached_input_tokens":0,"output_tokens":71,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0023179999999999997,"input_uncached_usd":0.0016079999999999998,"input_cached_usd":0.0,"output_usd":0.00071,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7687272089533508}
{"item_id":"senseval2.d000.s049.t001","gold_sense_keys":["parliament%1:14:00::"],"candidates":[{"index":1,"sense_key":"parliament%1:14:00::","synset_id":"parliament.n.01"},{"index":2,"sense_key":"parliament%1:04:00::","synset_id":"fantan.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"parliament%1:14:00::","call_ids":["senseval2.d000.s049.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"parliament%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":657,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001434,"input_uncached_usd":0.0013139999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9881112920120358}
{"item_id":"senseval2.d000.s049.t002","gold_sense_keys":["group%1:03:00::"],"candidates":[{"index":1,"sense_key":"group%1:03:00::","synset_id":"group.n.01"},{"index":2,"sense_key":"group%1:27:00::","synset_id":"group.n.02"},{"index":3,"sense_key":"group%1:09:00::","synset_id":"group.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"group%1:03:00::","call_ids":["senseval2.d000.s049.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"group%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":702,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001524,"input_uncached_usd":0.0014039999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5412986669689417}
{"item_id":"senseval2.d000.s049.t003","gold_sense_keys":["aim%2:31:00::","aim%2:31:01::"],"candidates":[{"index":1,"sense_key":"aim%2:33:00::","synset_id":"aim.v.01"},{"index":2,"sense_key":"aim%2:31:01::","synset_id":"aim.v.02"},{"index":3,"sense_key":"aim%2:32:09::","synset_id":"drive.v.11"},{"index":4,"sense_key":"aim%2:31:02::","synset_id":"calculate.v.05"},{"index":5,"sense_key":"aim%2:33:10::","synset_id":"target.v.01"},{"index":6,"sense_key":"aim%2:32:00::","synset_id":"aim.v.06"},{"index":7,"sense_key":"aim%2:31:00::","synset_id":"draw_a_bead_on.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"aim%2:31:01::","call_ids":["senseval2.d000.s049.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"aim%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":983,"cached_input_tokens":0,"output_tokens":111,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0030759999999999997,"input_uncached_usd":0.001966,"input_cached_usd":0.0,"output_usd":0.00111,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5761379590258002}
{"item_id":"senseval2.d000.s049.t004","gold_sense_keys":["improve%2:30:01::"],"candidates":[{"index":1,"sense_key":"improve%2:30:01::","synset_id":"better.v.02"},{"index":2,"sense_key":"improve%2:30:00::","synset_id":"better.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"improve%2:30:01::","call_ids":["senseval2.d000.s049.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"improve%2:30:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":660,"cached_input_tokens":0,"output_tokens":51,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00183,"input_uncached_usd":0.00132,"input_cached_usd":0.0,"output_usd":0.00051,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.525015499908477}
{"item_id":"senseval2.d000.s049.t006","gold_sense_keys":["vicar%1:18:01::"],"candidates":[{"index":1,"sense_key":"vicar%1:18:00::","synset_id":"vicar.n.01"},{"index":2,"sense_key":"vicar%1:18:02::","synset_id":"vicar.n.02"},{"index":3,"sense_key":"vicar%1:18:01::","synset_id":"vicar.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"vicar%1:18:01::","call_ids":["senseval2.d000.s049.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"vicar%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":682,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0019039999999999999,"input_uncached_usd":0.001364,"input_cached_usd":0.0,"output_usd":0.00054,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.378021709038876}
{"item_id":"senseval2.d000.s049.t007","gold_sense_keys":["say%2:32:01::"],"candidates":[{"index":1,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":6,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":9,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":10,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":11,"sense_key":"say%2:32:09::","synset_id":"say.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"say%2:32:00::","call_ids":["senseval2.d000.s049.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"say%2:32:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1198,"cached_input_tokens":0,"output_tokens":119,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0035859999999999998,"input_uncached_usd":0.0023959999999999997,"input_cached_usd":0.0,"output_usd":0.00119,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.664961334085092}
{"item_id":"senseval2.d000.s049.t008","gold_sense_keys":["president%1:18:03::"],"candidates":[{"index":1,"sense_key":"president%1:18:01::","synset_id":"president.n.01"},{"index":2,"sense_key":"president%1:18:04::","synset_id":"president_of_the_united_states.n.01"},{"index":3,"sense_key":"president%1:18:00::","synset_id":"president.n.03"},{"index":4,"sense_key":"president%1:18:03::","synset_id":"president.n.04"},{"index":5,"sense_key":"president%1:18:02::","synset_id":"president.n.05"},{"index":6,"sense_key":"president%1:04:04::","synset_id":"president_of_the_united_states.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"president%1:18:03::","call_ids":["senseval2.d000.s049.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"president%1:18:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":852,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002434,"input_uncached_usd":0.001704,"input_cached_usd":0.0,"output_usd":0.0007300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1541062499163672}
{"item_id":"senseval2.d000.s050.t000","gold_sense_keys":["hope%2:31:00::"],"candidates":[{"index":1,"sense_key":"hope%2:37:01::","synset_id":"hope.v.01"},{"index":2,"sense_key":"hope%2:37:00::","synset_id":"hope.v.02"},{"index":3,"sense_key":"hope%2:31:00::","synset_id":"hope.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"hope%2:31:00::","call_ids":["senseval2.d000.s050.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"hope%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":720,"cached_input_tokens":0,"output_tokens":80,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00224,"input_uncached_usd":0.0014399999999999999,"input_cached_usd":0.0,"output_usd":0.0008,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.458445749944076}
{"item_id":"senseval2.d000.s050.t001","gold_sense_keys":["speak%2:32:01::","speak%2:32:03::"],"candidates":[{"index":1,"sense_key":"speak%2:32:00::","synset_id":"talk.v.02"},{"index":2,"sense_key":"speak%2:32:01::","synset_id":"talk.v.01"},{"index":3,"sense_key":"speak%2:32:02::","synset_id":"speak.v.03"},{"index":4,"sense_key":"speak%2:32:03::","synset_id":"address.v.02"},{"index":5,"sense_key":"speak%2:39:00::","synset_id":"speak.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"speak%2:32:03::","call_ids":["senseval2.d000.s050.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"speak%2:32:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":815,"cached_input_tokens":0,"output_tokens":71,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00234,"input_uncached_usd":0.00163,"input_cached_usd":0.0,"output_usd":0.00071,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9582952500786632}
{"item_id":"senseval2.d000.s050.t002","gold_sense_keys":["student%1:18:00::"],"candidates":[{"index":1,"sense_key":"student%1:18:00::","synset_id":"student.n.01"},{"index":2,"sense_key":"student%1:18:01::","synset_id":"scholar.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"student%1:18:00::","call_ids":["senseval2.d000.s050.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"student%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":650,"cached_input_tokens":0,"output_tokens":50,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0018,"input_uncached_usd":0.0013,"input_cached_usd":0.0,"output_usd":0.0005,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3381783749209717}
{"item_id":"senseval2.d000.s050.t004","gold_sense_keys":["college%1:14:01::"],"candidates":[{"index":1,"sense_key":"college%1:14:00::","synset_id":"college.n.01"},{"index":2,"sense_key":"college%1:14:01::","synset_id":"college.n.02"},{"index":3,"sense_key":"college%1:06:00::","synset_id":"college.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"college%1:14:01::","call_ids":["senseval2.d000.s050.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"college%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":647,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0018640000000000002,"input_uncached_usd":0.001294,"input_cached_usd":0.0,"output_usd":0.0005700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4595294169848785}
{"item_id":"senseval2.d000.s050.t005","gold_sense_keys":["joy%1:09:00::"],"candidates":[{"index":1,"sense_key":"joy%1:12:00::","synset_id":"joy.n.01"},{"index":2,"sense_key":"joy%1:09:00::","synset_id":"joy.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"joy%1:09:00::","call_ids":["senseval2.d000.s050.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"joy%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":644,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001968,"input_uncached_usd":0.0012879999999999999,"input_cached_usd":0.0,"output_usd":0.00068,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.257359082926996}
{"item_id":"senseval2.d000.s050.t006","gold_sense_keys":["bell_ringing%1:04:00::"],"candidates":[{"index":1,"sense_key":"bell_ringing%1:11:00::","synset_id":"bell_ringing.n.01"},{"index":2,"sense_key":"bell_ringing%1:10:00::","synset_id":"electioneering.n.01"},{"index":3,"sense_key":"bell_ringing%1:04:00::","synset_id":"bell_ringing.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"bell_ringing%1:04:00::","call_ids":["senseval2.d000.s050.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"bell_ringing%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":690,"cached_input_tokens":0,"output_tokens":108,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00246,"input_uncached_usd":0.00138,"input_cached_usd":0.0,"output_usd":0.00108,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.1532016249839216}
{"item_id":"senseval2.d000.s050.t007","gold_sense_keys":["shortly%4:02:00::"],"candidates":[{"index":1,"sense_key":"shortly%4:02:01::","synset_id":"shortly.r.01"},{"index":2,"sense_key":"shortly%4:02:00::","synset_id":"soon.r.01"},{"index":3,"sense_key":"shortly%4:02:02::","synset_id":"curtly.r.01"},{"index":4,"sense_key":"shortly%4:02:04::","synset_id":"concisely.r.01"},{"index":5,"sense_key":"shortly%4:02:03::","synset_id":"shortly.r.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"shortly%4:02:00::","call_ids":["senseval2.d000.s050.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"shortly%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":839,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0017980000000000001,"input_uncached_usd":0.001678,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.514542166958563}
{"item_id":"senseval2.d000.s050.t008","gold_sense_keys":["publish%2:32:00::","publish%2:36:01::"],"candidates":[{"index":1,"sense_key":"publish%2:36:01::","synset_id":"print.v.01"},{"index":2,"sense_key":"publish%2:32:00::","synset_id":"publish.v.02"},{"index":3,"sense_key":"publish%2:36:00::","synset_id":"publish.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"publish%2:32:00::","call_ids":["senseval2.d000.s050.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"publish%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":729,"cached_input_tokens":0,"output_tokens":103,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0024879999999999998,"input_uncached_usd":0.0014579999999999999,"input_cached_usd":0.0,"output_usd":0.00103,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4444506668951362}
{"item_id":"senseval2.d000.s050.t010","gold_sense_keys":["vicar%1:18:01::"],"candidates":[{"index":1,"sense_key":"vicar%1:18:00::","synset_id":"vicar.n.01"},{"index":2,"sense_key":"vicar%1:18:02::","synset_id":"vicar.n.02"},{"index":3,"sense_key":"vicar%1:18:01::","synset_id":"vicar.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"vicar%1:18:01::","call_ids":["senseval2.d000.s050.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"vicar%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":671,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001902,"input_uncached_usd":0.0013419999999999999,"input_cached_usd":0.0,"output_usd":0.0005600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2156027080491185}
{"item_id":"senseval2.d000.s050.t011","gold_sense_keys":["country%1:14:00::","country%1:15:00::"],"candidates":[{"index":1,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":2,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":3,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":4,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":5,"sense_key":"country%1:15:02::","synset_id":"area.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"country%1:15:00::","call_ids":["senseval2.d000.s050.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"country%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":880,"cached_input_tokens":0,"output_tokens":110,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0028599999999999997,"input_uncached_usd":0.0017599999999999998,"input_cached_usd":0.0,"output_usd":0.0011,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8323078750399873}
{"item_id":"senseval2.d000.s050.t012","gold_sense_keys":["entitle%2:32:00::"],"candidates":[{"index":1,"sense_key":"entitle%2:41:00::","synset_id":"entitle.v.01"},{"index":2,"sense_key":"entitle%2:32:00::","synset_id":"entitle.v.02"},{"index":3,"sense_key":"entitle%2:41:01::","synset_id":"ennoble.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"entitle%2:32:00::","call_ids":["senseval2.d000.s050.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"entitle%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":672,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0018839999999999998,"input_uncached_usd":0.001344,"input_cached_usd":0.0,"output_usd":0.00054,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7560337919276208}
{"item_id":"senseval2.d000.s050.t013","gold_sense_keys":["bell%1:06:00::"],"candidates":[{"index":1,"sense_key":"bell%1:06:00::","synset_id":"bell.n.01"},{"index":2,"sense_key":"bell%1:06:02::","synset_id":"doorbell.n.01"},{"index":3,"sense_key":"bell%1:11:00::","synset_id":"bell.n.03"},{"index":4,"sense_key":"bell%1:28:00::","synset_id":"bell.n.04"},{"index":5,"sense_key":"bell%1:25:00::","synset_id":"bell.n.05"},{"index":6,"sense_key":"bell%1:18:02::","synset_id":"bell.n.06"},{"index":7,"sense_key":"bell%1:18:01::","synset_id":"bell.n.07"},{"index":8,"sense_key":"bell%1:18:00::","synset_id":"bell.n.08"},{"index":9,"sense_key":"bell%1:06:04::","synset_id":"chime.n.01"},{"index":10,"sense_key":"bell%1:06:03::","synset_id":"bell.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bell%1:06:00::","call_ids":["senseval2.d000.s050.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bell%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1177,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0030139999999999998,"input_uncached_usd":0.0023539999999999998,"input_cached_usd":0.0,"output_usd":0.0006600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2590941659873351}
{"item_id":"senseval2.d000.s050.t014","gold_sense_keys":["care%1:04:00::","care%1:04:01::","care%1:04:02::"],"candidates":[{"index":1,"sense_key":"care%1:04:01::","synset_id":"care.n.01"},{"index":2,"sense_key":"care%1:09:00::","synset_id":"caution.n.03"},{"index":3,"sense_key":"care%1:12:01::","synset_id":"concern.n.02"},{"index":4,"sense_key":"care%1:12:00::","synset_id":"care.n.04"},{"index":5,"sense_key":"care%1:04:02::","synset_id":"care.n.05"},{"index":6,"sense_key":"care%1:04:00::","synset_id":"care.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"care%1:04:02::","call_ids":["senseval2.d000.s050.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"care%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":918,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002496,"input_uncached_usd":0.001836,"input_cached_usd":0.0,"output_usd":0.0006600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9550422499887645}
{"item_id":"senseval2.d000.s051.t000","gold_sense_keys":["say%2:32:00::","say%2:32:15::"],"candidates":[{"index":1,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":6,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":9,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":10,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":11,"sense_key":"say%2:32:09::","synset_id":"say.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"say%2:32:00::","call_ids":["senseval2.d000.s051.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"say%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1159,"cached_input_tokens":0,"output_tokens":144,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003758,"input_uncached_usd":0.0023179999999999997,"input_cached_usd":0.0,"output_usd":0.00144,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.4231129590189084}
{"item_id":"senseval2.d000.s051.t002","gold_sense_keys":["recognize%2:31:01::"],"candidates":[{"index":1,"sense_key":"recognize%2:31:02::","synset_id":"acknowledge.v.06"},{"index":2,"sense_key":"recognize%2:31:01::","synset_id":"recognize.v.02"},{"index":3,"sense_key":"recognize%2:39:01::","synset_id":"spot.v.02"},{"index":4,"sense_key":"recognize%2:31:00::","synset_id":"recognize.v.04"},{"index":5,"sense_key":"recognize%2:41:00::","synset_id":"accredit.v.01"},{"index":6,"sense_key":"recognize%2:32:01::","synset_id":"greet.v.01"},{"index":7,"sense_key":"recognize%2:32:00::","synset_id":"acknowledge.v.04"},{"index":8,"sense_key":"recognize%2:42:00::","synset_id":"recognize.v.08"},{"index":9,"sense_key":"recognize%2:41:01::","synset_id":"recognize.v.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"recognize%2:31:01::","call_ids":["senseval2.d000.s051.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"recognize%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1098,"cached_input_tokens":0,"output_tokens":107,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003266,"input_uncached_usd":0.002196,"input_cached_usd":0.0,"output_usd":0.00107,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4672509579686448}
{"item_id":"senseval2.d000.s051.t003","gold_sense_keys":["no%4:02:01::"],"candidates":[{"index":1,"sense_key":"no%4:02:02::","synset_id":"no.r.01"},{"index":2,"sense_key":"no%4:02:01::","synset_id":"no.r.02"},{"index":3,"sense_key":"no%4:02:00::","synset_id":"no.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"no%4:02:01::","call_ids":["senseval2.d000.s051.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"no%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":659,"cached_input_tokens":0,"output_tokens":244,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003758,"input_uncached_usd":0.001318,"input_cached_usd":0.0,"output_usd":0.0024400000000000003,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.8216819170629606}
{"item_id":"senseval2.d000.s051.t005","gold_sense_keys":["high%3:00:02::"],"candidates":[{"index":1,"sense_key":"high%3:00:02::","synset_id":"high.a.01"},{"index":2,"sense_key":"high%3:00:01::","synset_id":"high.a.02"},{"index":3,"sense_key":"high%5:00:00:superior:01","synset_id":"eminent.s.01"},{"index":4,"sense_key":"high%3:00:03::","synset_id":"high.a.04"},{"index":5,"sense_key":"high%5:00:00:elated:00","synset_id":"high.s.02"},{"index":6,"sense_key":"high%5:00:00:malodorous:00","synset_id":"gamey.s.02"},{"index":7,"sense_key":"high%5:00:00:intoxicated:00","synset_id":"high.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"high%3:00:02::","call_ids":["senseval2.d000.s051.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"high%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":908,"cached_input_tokens":0,"output_tokens":110,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002916,"input_uncached_usd":0.001816,"input_cached_usd":0.0,"output_usd":0.0011,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6769310409436002}
{"item_id":"senseval2.d000.s051.t006","gold_sense_keys":["priority%1:26:00::"],"candidates":[{"index":1,"sense_key":"priority%1:26:00::","synset_id":"precedence.n.01"},{"index":2,"sense_key":"priority%1:07:00::","synset_id":"priority.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"priority%1:26:00::","call_ids":["senseval2.d000.s051.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"priority%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":644,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001408,"input_uncached_usd":0.0012879999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9998384580248967}
{"item_id":"senseval2.d000.s051.t007","gold_sense_keys":["church%1:14:00::"],"candidates":[{"index":1,"sense_key":"church%1:14:00::","synset_id":"church.n.01"},{"index":2,"sense_key":"church%1:06:00::","synset_id":"church.n.02"},{"index":3,"sense_key":"church%1:04:00::","synset_id":"church_service.n.01"},{"index":4,"sense_key":"church%1:14:01::","synset_id":"church.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"church%1:14:00::","call_ids":["senseval2.d000.s051.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"church%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":707,"cached_input_tokens":0,"output_tokens":112,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002534,"input_uncached_usd":0.001414,"input_cached_usd":0.0,"output_usd":0.0011200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9554953330662102}
{"item_id":"senseval2.d000.s051.t008","gold_sense_keys":["life%1:26:01::"],"candidates":[{"index":1,"sense_key":"life%1:26:01::","synset_id":"life.n.01"},{"index":2,"sense_key":"life%1:09:00::","synset_id":"life.n.02"},{"index":3,"sense_key":"life%1:26:02::","synset_id":"life.n.03"},{"index":4,"sense_key":"life%1:26:00::","synset_id":"animation.n.01"},{"index":5,"sense_key":"life%1:28:00::","synset_id":"life.n.05"},{"index":6,"sense_key":"life%1:28:01::","synset_id":"life.n.06"},{"index":7,"sense_key":"life%1:28:02::","synset_id":"life.n.07"},{"index":8,"sense_key":"life%1:18:00::","synset_id":"life.n.08"},{"index":9,"sense_key":"life%1:07:01::","synset_id":"liveliness.n.02"},{"index":10,"sense_key":"life%1:03:00::","synset_id":"life.n.10"},{"index":11,"sense_key":"life%1:19:00::","synset_id":"life.n.11"},{"index":12,"sense_key":"life%1:10:00::","synset_id":"biography.n.01"},{"index":13,"sense_key":"life%1:16:00::","synset_id":"life.n.13"},{"index":14,"sense_key":"life%1:28:03::","synset_id":"life_sentence.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"life%1:26:01::","call_ids":["senseval2.d000.s051.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"life%1:26:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1260,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0031199999999999995,"input_uncached_usd":0.0025199999999999997,"input_cached_usd":0.0,"output_usd":0.0006000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.534779708017595}
{"item_id":"senseval2.d000.s051.t009","gold_sense_keys":["experience%1:09:00::"],"candidates":[{"index":1,"sense_key":"experience%1:09:01::","synset_id":"experience.n.01"},{"index":2,"sense_key":"experience%1:09:00::","synset_id":"experience.n.02"},{"index":3,"sense_key":"experience%1:11:00::","synset_id":"experience.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"experience%1:09:00::","call_ids":["senseval2.d000.s051.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"experience%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":675,"cached_input_tokens":0,"output_tokens":100,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0023499999999999997,"input_uncached_usd":0.0013499999999999999,"input_cached_usd":0.0,"output_usd":0.001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7168034999631345}
{"item_id":"senseval2.d000.s052.t002","gold_sense_keys":["attack%2:30:00::"],"candidates":[{"index":1,"sense_key":"attack%2:33:00::","synset_id":"attack.v.01"},{"index":2,"sense_key":"attack%2:32:00::","synset_id":"attack.v.02"},{"index":3,"sense_key":"attack%2:33:01::","synset_id":"attack.v.03"},{"index":4,"sense_key":"attack%2:33:02::","synset_id":"assail.v.01"},{"index":5,"sense_key":"attack%2:30:00::","synset_id":"attack.v.05"},{"index":6,"sense_key":"attack%2:29:00::","synset_id":"attack.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"attack%2:30:00::","call_ids":["senseval2.d000.s052.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"attack%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":931,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001982,"input_uncached_usd":0.001862,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1930069581139833}
{"item_id":"senseval2.d000.s052.t004","gold_sense_keys":["problem%1:09:00::","problem%1:26:00::"],"candidates":[{"index":1,"sense_key":"problem%1:26:00::","synset_id":"problem.n.01"},{"index":2,"sense_key":"problem%1:10:00::","synset_id":"problem.n.02"},{"index":3,"sense_key":"problem%1:09:00::","synset_id":"trouble.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"problem%1:09:00::","call_ids":["senseval2.d000.s052.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"problem%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":671,"cached_input_tokens":0,"output_tokens":93,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0022719999999999997,"input_uncached_usd":0.0013419999999999999,"input_cached_usd":0.0,"output_usd":0.00093,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.784349332912825}
{"item_id":"senseval2.d000.s052.t006","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"},{"index":2,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"},{"index":3,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":4,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s052.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":695,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00151,"input_uncached_usd":0.00139,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.923156916978769}
{"item_id":"senseval2.d000.s053.t000","gold_sense_keys":["survey%1:04:02::"],"candidates":[{"index":1,"sense_key":"survey%1:04:02::","synset_id":"survey.n.01"},{"index":2,"sense_key":"survey%1:10:00::","synset_id":"sketch.n.03"},{"index":3,"sense_key":"survey%1:04:01::","synset_id":"view.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"survey%1:04:02::","call_ids":["senseval2.d000.s053.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"survey%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":663,"cached_input_tokens":0,"output_tokens":112,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0024460000000000003,"input_uncached_usd":0.001326,"input_cached_usd":0.0,"output_usd":0.0011200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5998078749980778}
{"item_id":"senseval2.d000.s053.t001","gold_sense_keys":["say%2:32:01::","say%2:42:00::"],"candidates":[{"index":1,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":6,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":9,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":10,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":11,"sense_key":"say%2:32:09::","synset_id":"say.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"say%2:32:01::","call_ids":["senseval2.d000.s053.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"say%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1166,"cached_input_tokens":0,"output_tokens":117,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003502,"input_uncached_usd":0.002332,"input_cached_usd":0.0,"output_usd":0.00117,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6816680000629276}
{"item_id":"senseval2.d000.s053.t003","gold_sense_keys":["bell_ringer%1:18:01::"],"candidates":[{"index":1,"sense_key":"bell_ringer%1:18:01::","synset_id":"toller.n.01"},{"index":2,"sense_key":"bell_ringer%1:18:00::","synset_id":"bell_ringer.n.02"},{"index":3,"sense_key":"bell_ringer%1:04:00::","synset_id":"bell_ringer.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bell_ringer%1:18:01::","call_ids":["senseval2.d000.s053.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bell_ringer%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":695,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00151,"input_uncached_usd":0.00139,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7114568339893594}
{"item_id":"senseval2.d000.s053.t005","gold_sense_keys":["today%1:28:00::"],"candidates":[{"index":1,"sense_key":"today%1:28:00::","synset_id":"today.n.01"},{"index":2,"sense_key":"today%1:28:01::","synset_id":"today.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"today%1:28:00::","call_ids":["senseval2.d000.s053.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"today%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":616,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001822,"input_uncached_usd":0.001232,"input_cached_usd":0.0,"output_usd":0.00059,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4223646250320598}
{"item_id":"senseval2.d000.s053.t006","gold_sense_keys":["only%4:02:02::"],"candidates":[{"index":1,"sense_key":"only%4:02:02::","synset_id":"merely.r.01"},{"index":2,"sense_key":"only%4:02:01::","synset_id":"entirely.r.02"},{"index":3,"sense_key":"only%4:02:00::","synset_id":"only.r.03"},{"index":4,"sense_key":"only%4:02:04::","synset_id":"only.r.04"},{"index":5,"sense_key":"only%4:02:06::","synset_id":"only.r.05"},{"index":6,"sense_key":"only%4:02:03::","synset_id":"only.r.06"},{"index":7,"sense_key":"only%4:02:05::","synset_id":"only.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"only%4:02:02::","call_ids":["senseval2.d000.s053.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"only%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":874,"cached_input_tokens":0,"output_tokens":103,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002778,"input_uncached_usd":0.001748,"input_cached_usd":0.0,"output_usd":0.00103,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7672722910065204}
{"item_id":"senseval2.d000.s053.t007","gold_sense_keys":["still%4:02:01::"],"candidates":[{"index":1,"sense_key":"still%4:02:01::","synset_id":"still.r.01"},{"index":2,"sense_key":"still%4:02:04::","synset_id":"however.r.01"},{"index":3,"sense_key":"still%4:02:00::","synset_id":"even.r.03"},{"index":4,"sense_key":"still%4:02:02::","synset_id":"still.r.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"still%4:02:01::","call_ids":["senseval2.d000.s053.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"still%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":807,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001734,"input_uncached_usd":0.001614,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0217242080252618}
{"item_id":"senseval2.d000.s054.t001","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"},{"index":2,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"},{"index":3,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":4,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s054.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":714,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0015480000000000001,"input_uncached_usd":0.001428,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.7743807500228286}
{"item_id":"senseval2.d000.s054.t002","gold_sense_keys":["always%4:02:04::"],"candidates":[{"index":1,"sense_key":"always%4:02:00::","synset_id":"always.r.01"},{"index":2,"sense_key":"always%4:02:04::","synset_id":"constantly.r.01"},{"index":3,"sense_key":"always%4:02:03::","synset_id":"constantly.r.02"},{"index":4,"sense_key":"always%4:02:02::","synset_id":"always.r.04"},{"index":5,"sense_key":"always%4:02:01::","synset_id":"always.r.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"always%4:02:04::","call_ids":["senseval2.d000.s054.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"always%4:02:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":797,"cached_input_tokens":0,"output_tokens":114,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0027340000000000003,"input_uncached_usd":0.001594,"input_cached_usd":0.0,"output_usd":0.0011400000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.692811332992278}
{"item_id":"senseval2.d000.s054.t003","gold_sense_keys":["live%2:42:08::"],"candidates":[{"index":1,"sense_key":"live%2:42:08::","synset_id":"populate.v.01"},{"index":2,"sense_key":"live%2:42:06::","synset_id":"live.v.02"},{"index":3,"sense_key":"live%2:42:04::","synset_id":"survive.v.01"},{"index":4,"sense_key":"live%2:42:01::","synset_id":"exist.v.02"},{"index":5,"sense_key":"live%2:42:00::","synset_id":"be.v.11"},{"index":6,"sense_key":"live%2:31:00::","synset_id":"know.v.05"},{"index":7,"sense_key":"live%2:42:07::","synset_id":"live.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"live%2:42:08::","call_ids":["senseval2.d000.s054.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"live%2:42:08::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":994,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0026279999999999997,"input_uncached_usd":0.0019879999999999997,"input_cached_usd":0.0,"output_usd":0.00064,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8828008329728618}
{"item_id":"senseval2.d000.s054.t004","gold_sense_keys":["bell%1:06:00::"],"candidates":[{"index":1,"sense_key":"bell%1:06:00::","synset_id":"bell.n.01"},{"index":2,"sense_key":"bell%1:06:02::","synset_id":"doorbell.n.01"},{"index":3,"sense_key":"bell%1:11:00::","synset_id":"bell.n.03"},{"index":4,"sense_key":"bell%1:28:00::","synset_id":"bell.n.04"},{"index":5,"sense_key":"bell%1:25:00::","synset_id":"bell.n.05"},{"index":6,"sense_key":"bell%1:18:02::","synset_id":"bell.n.06"},{"index":7,"sense_key":"bell%1:18:01::","synset_id":"bell.n.07"},{"index":8,"sense_key":"bell%1:18:00::","synset_id":"bell.n.08"},{"index":9,"sense_key":"bell%1:06:04::","synset_id":"chime.n.01"},{"index":10,"sense_key":"bell%1:06:03::","synset_id":"bell.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bell%1:06:00::","call_ids":["senseval2.d000.s054.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bell%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1158,"cached_input_tokens":0,"output_tokens":52,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002836,"input_uncached_usd":0.002316,"input_cached_usd":0.0,"output_usd":0.0005200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4127273330232129}
{"item_id":"senseval2.d000.s054.t005","gold_sense_keys":["need%2:34:00::"],"candidates":[{"index":1,"sense_key":"need%2:42:00::","synset_id":"necessitate.v.01"},{"index":2,"sense_key":"need%2:34:00::","synset_id":"want.v.02"},{"index":3,"sense_key":"need%2:34:01::","synset_id":"need.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"need%2:34:00::","call_ids":["senseval2.d000.s054.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"need%2:34:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":680,"cached_input_tokens":0,"output_tokens":295,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0043100000000000005,"input_uncached_usd":0.0013599999999999999,"input_cached_usd":0.0,"output_usd":0.0029500000000000004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.0581564171006903}
{"item_id":"senseval2.d000.s054.t006","gold_sense_keys":["ring%2:39:01::"],"candidates":[{"index":1,"sense_key":"ring%2:39:00::","synset_id":"ring.v.01"},{"index":2,"sense_key":"ring%2:39:04::","synset_id":"resound.v.01"},{"index":3,"sense_key":"ring%2:39:01::","synset_id":"ring.v.03"},{"index":4,"sense_key":"ring%2:32:00::","synset_id":"call.v.03"},{"index":5,"sense_key":"ring%2:35:03::","synset_id":"surround.v.01"},{"index":6,"sense_key":"ring%2:35:00::","synset_id":"ring.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"ring%2:39:01::","call_ids":["senseval2.d000.s054.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"ring%2:39:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":915,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0025599999999999998,"input_uncached_usd":0.0018299999999999998,"input_cached_usd":0.0,"output_usd":0.0007300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8941526249982417}
{"item_id":"senseval2.d000.s054.t007","gold_sense_keys":["small%3:00:00::"],"candidates":[{"index":1,"sense_key":"small%3:00:00::","synset_id":"small.a.01"},{"index":2,"sense_key":"small%5:00:00:limited:00","synset_id":"minor.s.03"},{"index":3,"sense_key":"small%5:00:00:young:00","synset_id":"little.s.01"},{"index":4,"sense_key":"small%5:00:00:little:03","synset_id":"small.s.03"},{"index":5,"sense_key":"small%5:00:00:inferior:01","synset_id":"humble.s.01"},{"index":6,"sense_key":"small%5:00:00:lowercase:00","synset_id":"little.s.04"},{"index":7,"sense_key":"small%5:00:00:soft:04","synset_id":"little.s.03"},{"index":8,"sense_key":"small%5:00:00:fine:00","synset_id":"small.s.07"},{"index":9,"sense_key":"small%5:00:00:moderate:00","synset_id":"modest.s.01"},{"index":10,"sense_key":"small%5:00:00:decreased:00","synset_id":"belittled.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"small%3:00:00::","call_ids":["senseval2.d000.s054.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"small%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1118,"cached_input_tokens":0,"output_tokens":118,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0034159999999999998,"input_uncached_usd":0.0022359999999999997,"input_cached_usd":0.0,"output_usd":0.00118,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8364666670095176}
{"item_id":"senseval2.d000.s055.t000","gold_sense_keys":["council%1:14:01::"],"candidates":[{"index":1,"sense_key":"council%1:14:01::","synset_id":"council.n.01"},{"index":2,"sense_key":"council%1:14:02::","synset_id":"council.n.02"},{"index":3,"sense_key":"council%1:14:00::","synset_id":"council.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"council%1:14:01::","call_ids":["senseval2.d000.s055.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"council%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":639,"cached_input_tokens":0,"output_tokens":115,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002428,"input_uncached_usd":0.0012779999999999998,"input_cached_usd":0.0,"output_usd":0.0011500000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4431742919841781}
{"item_id":"senseval2.d000.s055.t001","gold_sense_keys":["program%1:09:01::"],"candidates":[{"index":1,"sense_key":"program%1:09:00::","synset_id":"plan.n.01"},{"index":2,"sense_key":"program%1:09:01::","synset_id":"program.n.02"},{"index":3,"sense_key":"program%1:10:01::","synset_id":"broadcast.n.02"},{"index":4,"sense_key":"program%1:10:04::","synset_id":"platform.n.02"},{"index":5,"sense_key":"program%1:10:00::","synset_id":"program.n.05"},{"index":6,"sense_key":"program%1:10:05::","synset_id":"course_of_study.n.01"},{"index":7,"sense_key":"program%1:10:02::","synset_id":"program.n.07"},{"index":8,"sense_key":"program%1:04:00::","synset_id":"program.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"program%1:09:01::","call_ids":["senseval2.d000.s055.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"program%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1045,"cached_input_tokens":0,"output_tokens":96,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0030499999999999998,"input_uncached_usd":0.00209,"input_cached_usd":0.0,"output_usd":0.0009600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7204575000796467}
{"item_id":"senseval2.d000.s055.t002","gold_sense_keys":["attract%2:35:00::"],"candidates":[{"index":1,"sense_key":"attract%2:35:00::","synset_id":"attract.v.01"},{"index":2,"sense_key":"attract%2:37:00::","synset_id":"attract.v.02"},{"index":3,"sense_key":"attract%2:35:01::","synset_id":"attract.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"attract%2:35:00::","call_ids":["senseval2.d000.s055.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"attract%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":719,"cached_input_tokens":0,"output_tokens":111,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002548,"input_uncached_usd":0.001438,"input_cached_usd":0.0,"output_usd":0.00111,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6398909579729661}
{"item_id":"senseval2.d000.s055.t003","gold_sense_keys":["train%2:31:01::","train%2:32:00::","train%2:41:00::"],"candidates":[{"index":1,"sense_key":"train%2:31:01::","synset_id":"train.v.01"},{"index":2,"sense_key":"train%2:31:00::","synset_id":"train.v.02"},{"index":3,"sense_key":"train%2:41:02::","synset_id":"discipline.v.01"},{"index":4,"sense_key":"train%2:41:00::","synset_id":"prepare.v.05"},{"index":5,"sense_key":"train%2:41:01::","synset_id":"educate.v.03"},{"index":6,"sense_key":"train%2:33:00::","synset_id":"aim.v.01"},{"index":7,"sense_key":"train%2:32:00::","synset_id":"coach.v.01"},{"index":8,"sense_key":"train%2:29:00::","synset_id":"train.v.08"},{"index":9,"sense_key":"train%2:41:03::","synset_id":"train.v.09"},{"index":10,"sense_key":"train%2:38:00::","synset_id":"train.v.10"},{"index":11,"sense_key":"train%2:35:04::","synset_id":"trail.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"train%2:31:01::","call_ids":["senseval2.d000.s055.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"train%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1273,"cached_input_tokens":0,"output_tokens":130,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0038460000000000005,"input_uncached_usd":0.002546,"input_cached_usd":0.0,"output_usd":0.0013000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7833811250748113}
{"item_id":"senseval2.d000.s055.t004","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"},{"index":2,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"},{"index":3,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":4,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s055.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":692,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0015040000000000001,"input_uncached_usd":0.001384,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8466107080457732}
{"item_id":"senseval2.d000.s055.t005","gold_sense_keys":["only%4:02:02::"],"candidates":[{"index":1,"sense_key":"only%4:02:02::","synset_id":"merely.r.01"},{"index":2,"sense_key":"only%4:02:01::","synset_id":"entirely.r.02"},{"index":3,"sense_key":"only%4:02:00::","synset_id":"only.r.03"},{"index":4,"sense_key":"only%4:02:04::","synset_id":"only.r.04"},{"index":5,"sense_key":"only%4:02:06::","synset_id":"only.r.05"},{"index":6,"sense_key":"only%4:02:03::","synset_id":"only.r.06"},{"index":7,"sense_key":"only%4:02:05::","synset_id":"only.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"only%4:02:02::","call_ids":["senseval2.d000.s055.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"only%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":855,"cached_input_tokens":0,"output_tokens":52,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00223,"input_uncached_usd":0.00171,"input_cached_usd":0.0,"output_usd":0.0005200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3277409160509706}
{"item_id":"senseval2.d000.s055.t008","gold_sense_keys":["say%2:32:01::","say%2:32:13::"],"candidates":[{"index":1,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":6,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":9,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":10,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":11,"sense_key":"say%2:32:09::","synset_id":"say.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"say%2:32:00::","call_ids":["senseval2.d000.s055.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"say%2:32:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1147,"cached_input_tokens":0,"output_tokens":135,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003644,"input_uncached_usd":0.002294,"input_cached_usd":0.0,"output_usd":0.00135,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0903173339320347}
{"item_id":"senseval2.d000.s056.t000","gold_sense_keys":["right%4:02:04::"],"candidates":[{"index":1,"sense_key":"right%4:02:04::","synset_id":"right.r.01"},{"index":2,"sense_key":"right%4:02:08::","synset_id":"right.r.02"},{"index":3,"sense_key":"right%4:02:09::","synset_id":"right.r.03"},{"index":4,"sense_key":"right%4:02:03::","synset_id":"right.r.04"},{"index":5,"sense_key":"right%4:02:01::","synset_id":"properly.r.01"},{"index":6,"sense_key":"right%4:02:00::","synset_id":"right.r.06"},{"index":7,"sense_key":"right%4:02:07::","synset_id":"right.r.07"},{"index":8,"sense_key":"right%4:02:06::","synset_id":"mighty.r.01"},{"index":9,"sense_key":"right%4:02:05::","synset_id":"justly.r.02"},{"index":10,"sense_key":"right%4:02:02::","synset_id":"correctly.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"right%4:02:04::","call_ids":["senseval2.d000.s056.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"right%4:02:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":978,"cached_input_tokens":0,"output_tokens":195,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0039059999999999997,"input_uncached_usd":0.001956,"input_cached_usd":0.0,"output_usd":0.0019500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.7354720000876114}
{"item_id":"senseval2.d000.s056.t001","gold_sense_keys":["now%4:02:04::"],"candidates":[{"index":1,"sense_key":"now%4:02:05::","synset_id":"now.r.01"},{"index":2,"sense_key":"now%4:02:00::","synset_id":"nowadays.r.01"},{"index":3,"sense_key":"now%4:02:02::","synset_id":"now.r.03"},{"index":4,"sense_key":"now%4:02:04::","synset_id":"now.r.04"},{"index":5,"sense_key":"now%4:02:01::","synset_id":"immediately.r.01"},{"index":6,"sense_key":"now%4:02:07::","synset_id":"now.r.06"},{"index":7,"sense_key":"now%4:02:06::","synset_id":"now.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"now%4:02:04::","call_ids":["senseval2.d000.s056.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"now%4:02:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":906,"cached_input_tokens":0,"output_tokens":98,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002792,"input_uncached_usd":0.001812,"input_cached_usd":0.0,"output_usd":0.0009800000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6612077499739826}
{"item_id":"senseval2.d000.s056.t002","gold_sense_keys":["lucky%3:00:00::"],"candidates":[{"index":1,"sense_key":"lucky%5:00:00:fortunate:00","synset_id":"lucky.s.01"},{"index":2,"sense_key":"lucky%3:00:00::","synset_id":"lucky.a.02"},{"index":3,"sense_key":"lucky%5:00:00:propitious:00","synset_id":"golden.s.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"lucky%3:00:00::","call_ids":["senseval2.d000.s056.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"lucky%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":652,"cached_input_tokens":0,"output_tokens":106,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002364,"input_uncached_usd":0.001304,"input_cached_usd":0.0,"output_usd":0.0010600000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.937503083026968}
{"item_id":"senseval2.d000.s056.t003","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["senseval2.d000.s056.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":726,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001572,"input_uncached_usd":0.001452,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.454038416966796}
{"item_id":"senseval2.d000.s056.t005","gold_sense_keys":["new%3:00:00::","new%5:00:00:inexperienced:00","new%5:00:00:other:00"],"candidates":[{"index":1,"sense_key":"new%3:00:00::","synset_id":"new.a.01"},{"index":2,"sense_key":"new%5:00:00:original:00","synset_id":"fresh.s.03"},{"index":3,"sense_key":"new%5:00:00:inexperienced:00","synset_id":"raw.s.11"},{"index":4,"sense_key":"new%5:00:00:unprecedented:00","synset_id":"new.s.03"},{"index":5,"sense_key":"new%5:00:00:other:00","synset_id":"new.s.04"},{"index":6,"sense_key":"new%3:00:09::","synset_id":"new.a.06"},{"index":7,"sense_key":"new%5:00:02:original:00","synset_id":"newfangled.s.01"},{"index":8,"sense_key":"new%5:00:02:late:01","synset_id":"new.s.06"},{"index":9,"sense_key":"new%5:00:01:late:01","synset_id":"modern.s.04"},{"index":10,"sense_key":"new%5:00:00:early:02","synset_id":"new.s.08"},{"index":11,"sense_key":"new%5:00:00:unaccustomed:00","synset_id":"new.s.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"new%3:00:00::","call_ids":["senseval2.d000.s056.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"new%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1157,"cached_input_tokens":0,"output_tokens":124,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003554,"input_uncached_usd":0.002314,"input_cached_usd":0.0,"output_usd":0.00124,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1744891250273213}
{"item_id":"senseval2.d000.s056.t006","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"},{"index":2,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"},{"index":3,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":4,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s056.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":670,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0014600000000000001,"input_uncached_usd":0.00134,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9326752920169383}
{"item_id":"senseval2.d000.s056.t007","gold_sense_keys":["add%2:32:01::"],"candidates":[{"index":1,"sense_key":"add%2:30:00::","synset_id":"add.v.01"},{"index":2,"sense_key":"add%2:32:01::","synset_id":"add.v.02"},{"index":3,"sense_key":"add%2:40:00::","synset_id":"lend.v.01"},{"index":4,"sense_key":"add%2:31:00::","synset_id":"add.v.04"},{"index":5,"sense_key":"add%2:32:00::","synset_id":"total.v.02"},{"index":6,"sense_key":"add%2:42:00::","synset_id":"add.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"add%2:32:01::","call_ids":["senseval2.d000.s056.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"add%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":844,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001808,"input_uncached_usd":0.0016879999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1717800829792395}
{"item_id":"senseval2.d000.s057.t000","gold_sense_keys":["bright%5:00:00:auspicious:00"],"candidates":[{"index":1,"sense_key":"bright%3:00:00::","synset_id":"bright.a.01"},{"index":2,"sense_key":"bright%5:00:00:colorful:00","synset_id":"bright.s.01"},{"index":3,"sense_key":"bright%5:00:00:intelligent:00","synset_id":"bright.s.02"},{"index":4,"sense_key":"bright%5:00:00:light:06","synset_id":"bright.s.03"},{"index":5,"sense_key":"bright%5:00:00:polished:00","synset_id":"bright.s.04"},{"index":6,"sense_key":"bright%5:00:00:glorious:00","synset_id":"bright.s.05"},{"index":7,"sense_key":"bright%3:00:02::","synset_id":"undimmed.a.01"},{"index":8,"sense_key":"bright%5:00:00:reverberant:00","synset_id":"bright.s.06"},{"index":9,"sense_key":"bright%5:00:00:happy:00","synset_id":"bright.s.07"},{"index":10,"sense_key":"bright%5:00:00:auspicious:00","synset_id":"bright.s.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"bright%5:00:00:auspicious:00","call_ids":["senseval2.d000.s057.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"bright%5:00:00:auspicious:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1143,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002816,"input_uncached_usd":0.002286,"input_cached_usd":0.0,"output_usd":0.0005300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7929306249134243}
{"item_id":"senseval2.d000.s057.t001","gold_sense_keys":["sign%1:10:05::"],"candidates":[{"index":1,"sense_key":"sign%1:10:05::","synset_id":"sign.n.01"},{"index":2,"sense_key":"sign%1:10:00::","synset_id":"sign.n.02"},{"index":3,"sense_key":"sign%1:10:03::","synset_id":"signal.n.01"},{"index":4,"sense_key":"sign%1:06:01::","synset_id":"signboard.n.01"},{"index":5,"sense_key":"sign%1:15:00::","synset_id":"sign_of_the_zodiac.n.01"},{"index":6,"sense_key":"sign%1:26:00::","synset_id":"sign.n.06"},{"index":7,"sense_key":"sign%1:24:00::","synset_id":"polarity.n.02"},{"index":8,"sense_key":"sign%1:11:00::","synset_id":"augury.n.01"},{"index":9,"sense_key":"sign%1:10:01::","synset_id":"sign.n.09"},{"index":10,"sense_key":"sign%1:10:06::","synset_id":"sign.n.10"},{"index":11,"sense_key":"sign%1:10:04::","synset_id":"sign.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"sign%1:10:05::","call_ids":["senseval2.d000.s057.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"sign%1:10:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1148,"cached_input_tokens":0,"output_tokens":114,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003436,"input_uncached_usd":0.002296,"input_cached_usd":0.0,"output_usd":0.0011400000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7609925830038264}
{"item_id":"senseval2.d000.s057.t003","gold_sense_keys":["number%1:07:00::"],"candidates":[{"index":1,"sense_key":"number%1:07:00::","synset_id":"number.n.01"},{"index":2,"sense_key":"number%1:23:00::","synset_id":"number.n.02"},{"index":3,"sense_key":"number%1:10:04::","synset_id":"act.n.04"},{"index":4,"sense_key":"number%1:10:03::","synset_id":"phone_number.n.01"},{"index":5,"sense_key":"number%1:10:01::","synset_id":"numeral.n.01"},{"index":6,"sense_key":"number%1:10:00::","synset_id":"issue.n.02"},{"index":7,"sense_key":"number%1:14:00::","synset_id":"number.n.07"},{"index":8,"sense_key":"number%1:10:02::","synset_id":"number.n.08"},{"index":9,"sense_key":"number%1:07:01::","synset_id":"number.n.09"},{"index":10,"sense_key":"number%1:10:05::","synset_id":"number.n.10"},{"index":11,"sense_key":"number%1:06:00::","synset_id":"number.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"number%1:07:00::","call_ids":["senseval2.d000.s057.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"number%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1112,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0023439999999999997,"input_uncached_usd":0.002224,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0423033749684691}
{"item_id":"senseval2.d000.s057.t004","gold_sense_keys":["woman%1:18:00::"],"candidates":[{"index":1,"sense_key":"woman%1:18:00::","synset_id":"woman.n.01"},{"index":2,"sense_key":"woman%1:18:01::","synset_id":"woman.n.02"},{"index":3,"sense_key":"woman%1:18:02::","synset_id":"charwoman.n.01"},{"index":4,"sense_key":"woman%1:14:00::","synset_id":"womanhood.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"woman%1:18:00::","call_ids":["senseval2.d000.s057.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"woman%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":725,"cached_input_tokens":0,"output_tokens":83,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00228,"input_uncached_usd":0.00145,"input_cached_usd":0.0,"output_usd":0.0008300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4810900840675458}
{"item_id":"senseval2.d000.s057.t005","gold_sense_keys":["enter%2:33:00::","enter%2:38:00::"],"candidates":[{"index":1,"sense_key":"enter%2:38:00::","synset_id":"enter.v.01"},{"index":2,"sense_key":"enter%2:33:00::","synset_id":"enter.v.02"},{"index":3,"sense_key":"enter%2:41:06::","synset_id":"enroll.v.01"},{"index":4,"sense_key":"enter%2:42:00::","synset_id":"figure.v.02"},{"index":5,"sense_key":"enter%2:32:04::","synset_id":"record.v.01"},{"index":6,"sense_key":"enter%2:36:00::","synset_id":"enter.v.06"},{"index":7,"sense_key":"enter%2:41:00::","synset_id":"accede.v.02"},{"index":8,"sense_key":"enter%2:35:03::","synset_id":"insert.v.01"},{"index":9,"sense_key":"enter%2:30:10::","synset_id":"embark.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"enter%2:30:10::","call_ids":["senseval2.d000.s057.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"enter%2:30:10::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":935,"cached_input_tokens":0,"output_tokens":109,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00296,"input_uncached_usd":0.00187,"input_cached_usd":0.0,"output_usd":0.00109,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.662895916029811}
{"item_id":"senseval2.d000.s057.t006","gold_sense_keys":["once%4:02:01::"],"candidates":[{"index":1,"sense_key":"once%4:02:00::","synset_id":"once.r.01"},{"index":2,"sense_key":"once%4:02:02::","synset_id":"once.r.02"},{"index":3,"sense_key":"once%4:02:01::","synset_id":"once.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"once%4:02:01::","call_ids":["senseval2.d000.s057.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"once%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":596,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001312,"input_uncached_usd":0.001192,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":6.239790999912657}
{"item_id":"senseval2.d000.s057.t007","gold_sense_keys":["field%1:09:00::","field%1:26:00::"],"candidates":[{"index":1,"sense_key":"field%1:15:00::","synset_id":"field.n.01"},{"index":2,"sense_key":"field%1:15:04::","synset_id":"battlefield.n.01"},{"index":3,"sense_key":"field%1:15:01::","synset_id":"field.n.03"},{"index":4,"sense_key":"field%1:09:00::","synset_id":"discipline.n.01"},{"index":5,"sense_key":"field%1:19:00::","synset_id":"field.n.05"},{"index":6,"sense_key":"field%1:04:00::","synset_id":"field.n.06"},{"index":7,"sense_key":"field%1:26:00::","synset_id":"sphere.n.01"},{"index":8,"sense_key":"field%1:15:02::","synset_id":"playing_field.n.02"},{"index":9,"sense_key":"field%1:17:00::","synset_id":"plain.n.01"},{"index":10,"sense_key":"field%1:14:00::","synset_id":"field.n.10"},{"index":11,"sense_key":"field%1:15:03::","synset_id":"field.n.11"},{"index":12,"sense_key":"field%1:14:01::","synset_id":"field.n.12"},{"index":13,"sense_key":"field%1:14:02::","synset_id":"field.n.13"},{"index":14,"sense_key":"field%1:15:05::","synset_id":"field.n.14"},{"index":15,"sense_key":"field%1:14:03::","synset_id":"field.n.15"},{"index":16,"sense_key":"field%1:09:01::","synset_id":"field.n.16"},{"index":17,"sense_key":"field%1:06:00::","synset_id":"airfield.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"field%1:26:00::","call_ids":["senseval2.d000.s057.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"field%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1521,"cached_input_tokens":0,"output_tokens":163,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004672,"input_uncached_usd":0.003042,"input_cached_usd":0.0,"output_usd":0.0016300000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8040602080291137}
{"item_id":"senseval2.d000.s057.t009","gold_sense_keys":["third%1:23:00::"],"candidates":[{"index":1,"sense_key":"third%1:23:00::","synset_id":"one-third.n.01"},{"index":2,"sense_key":"third%1:04:00::","synset_id":"third_base.n.02"},{"index":3,"sense_key":"third%1:24:00::","synset_id":"third.n.03"},{"index":4,"sense_key":"third%1:10:00::","synset_id":"third.n.04"},{"index":5,"sense_key":"third%1:06:00::","synset_id":"third_gear.n.01"},{"index":6,"sense_key":"third%1:06:01::","synset_id":"third_base.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"third%1:23:00::","call_ids":["senseval2.d000.s057.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"third%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":848,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0018160000000000001,"input_uncached_usd":0.001696,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.018550124950707}
{"item_id":"senseval2.d000.s057.t010","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"},{"index":2,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"},{"index":3,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":4,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s057.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":646,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001412,"input_uncached_usd":0.001292,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2540558750042692}
{"item_id":"senseval2.d000.s057.t011","gold_sense_keys":["today%1:28:00::"],"candidates":[{"index":1,"sense_key":"today%1:28:00::","synset_id":"today.n.01"},{"index":2,"sense_key":"today%1:28:01::","synset_id":"today.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"today%1:28:00::","call_ids":["senseval2.d000.s057.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"today%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":551,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0016419999999999998,"input_uncached_usd":0.001102,"input_cached_usd":0.0,"output_usd":0.00054,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1681662499904633}
{"item_id":"senseval2.d000.s057.t012","gold_sense_keys":["woman%1:18:00::"],"candidates":[{"index":1,"sense_key":"woman%1:18:00::","synset_id":"woman.n.01"},{"index":2,"sense_key":"woman%1:18:01::","synset_id":"woman.n.02"},{"index":3,"sense_key":"woman%1:18:02::","synset_id":"charwoman.n.01"},{"index":4,"sense_key":"woman%1:14:00::","synset_id":"womanhood.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"woman%1:18:00::","call_ids":["senseval2.d000.s057.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"woman%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":725,"cached_input_tokens":0,"output_tokens":87,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00232,"input_uncached_usd":0.00145,"input_cached_usd":0.0,"output_usd":0.0008700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6313972079660743}
{"item_id":"senseval2.d000.s058.t000","gold_sense_keys":["accept%2:40:03::"],"candidates":[{"index":1,"sense_key":"accept%2:31:00::","synset_id":"accept.v.01"},{"index":2,"sense_key":"accept%2:40:00::","synset_id":"accept.v.02"},{"index":3,"sense_key":"accept%2:32:00::","synset_id":"accept.v.03"},{"index":4,"sense_key":"accept%2:31:01::","synset_id":"accept.v.04"},{"index":5,"sense_key":"accept%2:40:03::","synset_id":"accept.v.05"},{"index":6,"sense_key":"accept%2:40:01::","synset_id":"bear.v.06"},{"index":7,"sense_key":"accept%2:31:03::","synset_id":"accept.v.07"},{"index":8,"sense_key":"accept%2:42:00::","synset_id":"accept.v.08"},{"index":9,"sense_key":"accept%2:40:07::","synset_id":"accept.v.09"},{"index":10,"sense_key":"accept%2:40:12::","synset_id":"take.v.20"},{"index":11,"sense_key":"accept%2:31:08::","synset_id":"accept.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"accept%2:40:03::","call_ids":["senseval2.d000.s058.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"accept%2:40:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1198,"cached_input_tokens":0,"output_tokens":157,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003966,"input_uncached_usd":0.0023959999999999997,"input_cached_usd":0.0,"output_usd":0.0015700000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.3105769160902128}
{"item_id":"senseval2.d000.s058.t002","gold_sense_keys":["however%4:02:00::"],"candidates":[{"index":1,"sense_key":"however%4:02:00::","synset_id":"however.r.01"},{"index":2,"sense_key":"however%4:02:04::","synset_id":"however.r.02"},{"index":3,"sense_key":"however%4:02:02::","synset_id":"however.r.03"},{"index":4,"sense_key":"however%4:02:01::","synset_id":"however.r.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"however%4:02:00::","call_ids":["senseval2.d000.s058.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"however%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":805,"cached_input_tokens":0,"output_tokens":132,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0029300000000000003,"input_uncached_usd":0.0016099999999999999,"input_cached_usd":0.0,"output_usd":0.0013200000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8449332079617307}
{"item_id":"senseval2.d000.s059.t000","gold_sense_keys":["bell_ringing%1:04:00::"],"candidates":[{"index":1,"sense_key":"bell_ringing%1:11:00::","synset_id":"bell_ringing.n.01"},{"index":2,"sense_key":"bell_ringing%1:10:00::","synset_id":"electioneering.n.01"},{"index":3,"sense_key":"bell_ringing%1:04:00::","synset_id":"bell_ringing.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"bell_ringing%1:04:00::","call_ids":["senseval2.d000.s059.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"bell_ringing%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":710,"cached_input_tokens":0,"output_tokens":84,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00226,"input_uncached_usd":0.00142,"input_cached_usd":0.0,"output_usd":0.00084,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0503686669981107}
{"item_id":"senseval2.d000.s059.t001","gold_sense_keys":["group%1:03:00::"],"candidates":[{"index":1,"sense_key":"group%1:03:00::","synset_id":"group.n.01"},{"index":2,"sense_key":"group%1:27:00::","synset_id":"group.n.02"},{"index":3,"sense_key":"group%1:09:00::","synset_id":"group.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"group%1:03:00::","call_ids":["senseval2.d000.s059.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"group%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":709,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001538,"input_uncached_usd":0.001418,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0043902499601245}
{"item_id":"senseval2.d000.s059.t002","gold_sense_keys":["country%1:14:00::","country%1:15:00::"],"candidates":[{"index":1,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":2,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":3,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":4,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":5,"sense_key":"country%1:15:02::","synset_id":"area.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"country%1:15:00::","call_ids":["senseval2.d000.s059.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"country%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":899,"cached_input_tokens":0,"output_tokens":100,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002798,"input_uncached_usd":0.001798,"input_cached_usd":0.0,"output_usd":0.001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0690135830082}
{"item_id":"senseval2.d000.s059.t003","gold_sense_keys":["found%2:36:00::","found%2:41:00::"],"candidates":[{"index":1,"sense_key":"found%2:41:00::","synset_id":"establish.v.01"},{"index":2,"sense_key":"found%2:36:00::","synset_id":"establish.v.02"},{"index":3,"sense_key":"found%2:31:00::","synset_id":"establish.v.08"},{"index":4,"sense_key":"find%2:40:02::","synset_id":"find.v.01"},{"index":5,"sense_key":"detect%2:39:00::","synset_id":"detect.v.01"},{"index":6,"sense_key":"find%2:40:00::","synset_id":"find.v.03"},{"index":7,"sense_key":"determine%2:32:00::","synset_id":"determine.v.01"},{"index":8,"sense_key":"find%2:31:10::","synset_id":"find.v.05"},{"index":9,"sense_key":"witness%2:39:01::","synset_id":"witness.v.02"},{"index":10,"sense_key":"line_up%2:40:00::","synset_id":"line_up.v.02"},{"index":11,"sense_key":"discover%2:36:00::","synset_id":"discover.v.03"},{"index":12,"sense_key":"discover%2:31:00::","synset_id":"discover.v.04"},{"index":13,"sense_key":"find%2:40:03::","synset_id":"find.v.10"},{"index":14,"sense_key":"rule%2:32:00::","synset_id":"rule.v.04"},{"index":15,"sense_key":"receive%2:30:01::","synset_id":"receive.v.02"},{"index":16,"sense_key":"find%2:39:05::","synset_id":"find.v.13"},{"index":17,"sense_key":"recover%2:40:00::","synset_id":"recover.v.01"},{"index":18,"sense_key":"find%2:38:10::","synset_id":"find.v.15"},{"index":19,"sense_key":"find_oneself%2:30:00::","synset_id":"find_oneself.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"found%2:41:00::","call_ids":["senseval2.d000.s059.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"found%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1925,"cached_input_tokens":0,"output_tokens":274,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00659,"input_uncached_usd":0.0038499999999999997,"input_cached_usd":0.0,"output_usd":0.0027400000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.179776542005129}
{"item_id":"senseval2.d000.s059.t004","gold_sense_keys":["remain%2:30:00::"],"candidates":[{"index":1,"sense_key":"remain%2:30:00::","synset_id":"stay.v.01"},{"index":2,"sense_key":"remain%2:42:03::","synset_id":"stay.v.04"},{"index":3,"sense_key":"remain%2:42:05::","synset_id":"remain.v.03"},{"index":4,"sense_key":"remain%2:42:01::","synset_id":"persist.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"remain%2:30:00::","call_ids":["senseval2.d000.s059.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"remain%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":858,"cached_input_tokens":0,"output_tokens":92,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002636,"input_uncached_usd":0.0017159999999999999,"input_cached_usd":0.0,"output_usd":0.00092,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.095616374979727}
{"item_id":"senseval2.d000.s059.t005","gold_sense_keys":["fact%1:09:01::"],"candidates":[{"index":1,"sense_key":"fact%1:09:01::","synset_id":"fact.n.01"},{"index":2,"sense_key":"fact%1:10:01::","synset_id":"fact.n.02"},{"index":3,"sense_key":"fact%1:26:01::","synset_id":"fact.n.03"},{"index":4,"sense_key":"fact%1:09:02::","synset_id":"fact.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"fact%1:09:01::","call_ids":["senseval2.d000.s059.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"fact%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":789,"cached_input_tokens":0,"output_tokens":164,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0032180000000000004,"input_uncached_usd":0.001578,"input_cached_usd":0.0,"output_usd":0.0016400000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.874448167043738}
{"item_id":"senseval2.d000.s059.t006","gold_sense_keys":["particularly%4:02:00::"],"candidates":[{"index":1,"sense_key":"particularly%4:02:00::","synset_id":"particularly.r.01"},{"index":2,"sense_key":"particularly%4:02:02::","synset_id":"particularly.r.02"},{"index":3,"sense_key":"particularly%4:02:01::","synset_id":"peculiarly.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"particularly%4:02:00::","call_ids":["senseval2.d000.s059.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"particularly%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":787,"cached_input_tokens":0,"output_tokens":106,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002634,"input_uncached_usd":0.0015739999999999999,"input_cached_usd":0.0,"output_usd":0.0010600000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6354584159562364}
{"item_id":"senseval2.d000.s059.t008","gold_sense_keys":["woman%1:18:00::"],"candidates":[{"index":1,"sense_key":"woman%1:18:00::","synset_id":"woman.n.01"},{"index":2,"sense_key":"woman%1:18:01::","synset_id":"woman.n.02"},{"index":3,"sense_key":"woman%1:18:02::","synset_id":"charwoman.n.01"},{"index":4,"sense_key":"woman%1:14:00::","synset_id":"womanhood.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"woman%1:18:00::","call_ids":["senseval2.d000.s059.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"woman%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":830,"cached_input_tokens":0,"output_tokens":177,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00343,"input_uncached_usd":0.00166,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1078680000500754}
{"item_id":"senseval2.d000.s059.t009","gold_sense_keys":["group%1:03:00::"],"candidates":[{"index":1,"sense_key":"group%1:03:00::","synset_id":"group.n.01"},{"index":2,"sense_key":"group%1:27:00::","synset_id":"group.n.02"},{"index":3,"sense_key":"group%1:09:00::","synset_id":"group.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"group%1:03:00::","call_ids":["senseval2.d000.s059.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"group%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":709,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001538,"input_uncached_usd":0.001418,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9416205830639228}
{"item_id":"senseval2.d000.s059.t011","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"},{"index":2,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"},{"index":3,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":4,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s059.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":751,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001622,"input_uncached_usd":0.0015019999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5108900839695707}
{"item_id":"senseval2.d000.s059.t013","gold_sense_keys":["most%4:02:00::"],"candidates":[{"index":1,"sense_key":"most%4:02:00::","synset_id":"most.r.01"},{"index":2,"sense_key":"most%4:02:01::","synset_id":"most.r.02"},{"index":3,"sense_key":"most%4:02:02::","synset_id":"about.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"most%4:02:00::","call_ids":["senseval2.d000.s059.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"most%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":736,"cached_input_tokens":0,"output_tokens":52,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001992,"input_uncached_usd":0.001472,"input_cached_usd":0.0,"output_usd":0.0005200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.794219083036296}
{"item_id":"senseval2.d000.s059.t015","gold_sense_keys":["church%1:06:00::"],"candidates":[{"index":1,"sense_key":"church%1:14:00::","synset_id":"church.n.01"},{"index":2,"sense_key":"church%1:06:00::","synset_id":"church.n.02"},{"index":3,"sense_key":"church%1:04:00::","synset_id":"church_service.n.01"},{"index":4,"sense_key":"church%1:14:01::","synset_id":"church.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"church%1:06:00::","call_ids":["senseval2.d000.s059.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"church%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":756,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002152,"input_uncached_usd":0.0015119999999999999,"input_cached_usd":0.0,"output_usd":0.00064,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2975450420053676}
{"item_id":"senseval2.d000.s060.t001","gold_sense_keys":["woman%1:18:00::"],"candidates":[{"index":1,"sense_key":"woman%1:18:00::","synset_id":"woman.n.01"},{"index":2,"sense_key":"woman%1:18:01::","synset_id":"woman.n.02"},{"index":3,"sense_key":"woman%1:18:02::","synset_id":"charwoman.n.01"},{"index":4,"sense_key":"woman%1:14:00::","synset_id":"womanhood.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"woman%1:18:00::","call_ids":["senseval2.d000.s060.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"woman%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":862,"cached_input_tokens":0,"output_tokens":89,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002614,"input_uncached_usd":0.0017239999999999998,"input_cached_usd":0.0,"output_usd":0.0008900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.132981709088199}
{"item_id":"senseval2.d000.s060.t002","gold_sense_keys":["file%2:32:02::"],"candidates":[{"index":1,"sense_key":"file%2:32:02::","synset_id":"file.v.01"},{"index":2,"sense_key":"file%2:35:00::","synset_id":"file.v.02"},{"index":3,"sense_key":"file%2:38:00::","synset_id":"file.v.03"},{"index":4,"sense_key":"file%2:32:01::","synset_id":"charge.v.06"},{"index":5,"sense_key":"file%2:32:00::","synset_id":"file.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"file%2:32:02::","call_ids":["senseval2.d000.s060.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"file%2:32:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":816,"cached_input_tokens":0,"output_tokens":103,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0026620000000000003,"input_uncached_usd":0.001632,"input_cached_usd":0.0,"output_usd":0.00103,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7229073330527171}
{"item_id":"senseval2.d000.s060.t004","gold_sense_keys":["suit%1:04:00::"],"candidates":[{"index":1,"sense_key":"suit%1:06:00::","synset_id":"suit.n.01"},{"index":2,"sense_key":"suit%1:04:00::","synset_id":"lawsuit.n.01"},{"index":3,"sense_key":"suit%1:18:00::","synset_id":"suit.n.03"},{"index":4,"sense_key":"suit%1:10:01::","synset_id":"courtship.n.01"},{"index":5,"sense_key":"suit%1:10:00::","synset_id":"suit.n.05"},{"index":6,"sense_key":"suit%1:06:01::","synset_id":"suit.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"suit%1:04:00::","call_ids":["senseval2.d000.s060.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"suit%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1010,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00214,"input_uncached_usd":0.00202,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5326147500891238}
{"item_id":"senseval2.d000.s060.t005","gold_sense_keys":["extent%1:26:00::"],"candidates":[{"index":1,"sense_key":"extent%1:26:00::","synset_id":"extent.n.01"},{"index":2,"sense_key":"extent%1:07:00::","synset_id":"extent.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"extent%1:26:00::","call_ids":["senseval2.d000.s060.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"extent%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":690,"cached_input_tokens":0,"output_tokens":83,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00221,"input_uncached_usd":0.00138,"input_cached_usd":0.0,"output_usd":0.0008300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.811311790952459}
{"item_id":"senseval2.d000.s060.t006","gold_sense_keys":["problem%1:09:00::","problem%1:26:00::"],"candidates":[{"index":1,"sense_key":"problem%1:26:00::","synset_id":"problem.n.01"},{"index":2,"sense_key":"problem%1:10:00::","synset_id":"problem.n.02"},{"index":3,"sense_key":"problem%1:09:00::","synset_id":"trouble.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"problem%1:09:00::","call_ids":["senseval2.d000.s060.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"problem%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":758,"cached_input_tokens":0,"output_tokens":225,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0037660000000000003,"input_uncached_usd":0.001516,"input_cached_usd":0.0,"output_usd":0.0022500000000000003,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1768519999459386}
{"item_id":"senseval2.d000.s060.t007","gold_sense_keys":["surface%2:30:00::"],"candidates":[{"index":1,"sense_key":"surface%2:38:00::","synset_id":"surface.v.01"},{"index":2,"sense_key":"surface%2:35:00::","synset_id":"coat.v.01"},{"index":3,"sense_key":"surface%2:30:00::","synset_id":"come_on.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"surface%2:30:00::","call_ids":["senseval2.d000.s060.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"surface%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":765,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00165,"input_uncached_usd":0.00153,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1926368339918554}
{"item_id":"senseval2.d000.s060.t008","gold_sense_keys":["summer%1:28:00::"],"candidates":[{"index":1,"sense_key":"summer%1:28:00::","synset_id":"summer.n.01"},{"index":2,"sense_key":"summer%1:28:01::","synset_id":"summer.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"summer%1:28:00::","call_ids":["senseval2.d000.s060.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"summer%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":714,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0015480000000000001,"input_uncached_usd":0.001428,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8643181250663474}
{"item_id":"senseval2.d000.s060.t009","gold_sense_keys":["series%1:14:00::"],"candidates":[{"index":1,"sense_key":"series%1:14:00::","synset_id":"series.n.01"},{"index":2,"sense_key":"series%1:10:01::","synset_id":"serial.n.01"},{"index":3,"sense_key":"series%1:10:00::","synset_id":"series.n.03"},{"index":4,"sense_key":"series%1:11:00::","synset_id":"series.n.04"},{"index":5,"sense_key":"series%1:24:00::","synset_id":"series.n.05"},{"index":6,"sense_key":"series%1:14:01::","synset_id":"series.n.06"},{"index":7,"sense_key":"series%1:09:00::","synset_id":"series.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"series%1:14:00::","call_ids":["senseval2.d000.s060.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"series%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":993,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002106,"input_uncached_usd":0.001986,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9218964159954339}
{"item_id":"senseval2.d000.s060.t010","gold_sense_keys":["letter%1:10:00::"],"candidates":[{"index":1,"sense_key":"letter%1:10:00::","synset_id":"letter.n.01"},{"index":2,"sense_key":"letter%1:10:01::","synset_id":"letter.n.02"},{"index":3,"sense_key":"letter%1:18:00::","synset_id":"letter.n.03"},{"index":4,"sense_key":"letter%1:10:03::","synset_id":"letter.n.04"},{"index":5,"sense_key":"letter%1:10:02::","synset_id":"letter.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"letter%1:10:00::","call_ids":["senseval2.d000.s060.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"letter%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":880,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00188,"input_uncached_usd":0.0017599999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1124094580300152}
{"item_id":"senseval2.d000.s060.t012","gold_sense_keys":["newspaper%1:10:00::"],"candidates":[{"index":1,"sense_key":"newspaper%1:10:00::","synset_id":"newspaper.n.01"},{"index":2,"sense_key":"newspaper%1:14:00::","synset_id":"newspaper.n.02"},{"index":3,"sense_key":"newspaper%1:06:00::","synset_id":"newspaper.n.03"},{"index":4,"sense_key":"newspaper%1:27:00::","synset_id":"newspaper.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"newspaper%1:10:00::","call_ids":["senseval2.d000.s060.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"newspaper%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":867,"cached_input_tokens":0,"output_tokens":113,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0028640000000000002,"input_uncached_usd":0.0017339999999999999,"input_cached_usd":0.0,"output_usd":0.0011300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5638275000965223}
{"item_id":"senseval2.d000.s060.t013","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"},{"index":2,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"},{"index":3,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":4,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s060.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":782,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001684,"input_uncached_usd":0.0015639999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1118407920002937}
{"item_id":"senseval2.d000.s061.t000","gold_sense_keys":["writer%1:18:01::"],"candidates":[{"index":1,"sense_key":"writer%1:18:00::","synset_id":"writer.n.01"},{"index":2,"sense_key":"writer%1:18:01::","synset_id":"writer.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"writer%1:18:01::","call_ids":["senseval2.d000.s061.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"writer%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":721,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002002,"input_uncached_usd":0.001442,"input_cached_usd":0.0,"output_usd":0.0005600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2893441669875756}
{"item_id":"senseval2.d000.s061.t001","gold_sense_keys":["sign%2:32:02::"],"candidates":[{"index":1,"sense_key":"sign%2:32:02::","synset_id":"sign.v.01"},{"index":2,"sense_key":"sign%2:41:00::","synset_id":"sign.v.02"},{"index":3,"sense_key":"sign%2:32:00::","synset_id":"sign.v.03"},{"index":4,"sense_key":"sign%2:41:01::","synset_id":"sign.v.04"},{"index":5,"sense_key":"sign%2:32:01::","synset_id":"sign.v.05"},{"index":6,"sense_key":"sign%2:35:02::","synset_id":"sign.v.06"},{"index":7,"sense_key":"sign%2:32:04::","synset_id":"sign.v.07"},{"index":8,"sense_key":"sign%2:32:03::","synset_id":"bless.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"sign%2:32:02::","call_ids":["senseval2.d000.s061.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"sign%2:32:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1161,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0024419999999999997,"input_uncached_usd":0.002322,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2408133329590783}
{"item_id":"senseval2.d000.s061.t002","gold_sense_keys":["letter%1:10:00::"],"candidates":[{"index":1,"sense_key":"letter%1:10:00::","synset_id":"letter.n.01"},{"index":2,"sense_key":"letter%1:10:01::","synset_id":"letter.n.02"},{"index":3,"sense_key":"letter%1:18:00::","synset_id":"letter.n.03"},{"index":4,"sense_key":"letter%1:10:03::","synset_id":"letter.n.04"},{"index":5,"sense_key":"letter%1:10:02::","synset_id":"letter.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"letter%1:10:00::","call_ids":["senseval2.d000.s061.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"letter%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":937,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001994,"input_uncached_usd":0.001874,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1395720000145957}
{"item_id":"senseval2.d000.s061.t005","gold_sense_keys":["male%1:18:00::"],"candidates":[{"index":1,"sense_key":"male%1:05:00::","synset_id":"male.n.01"},{"index":2,"sense_key":"male%1:18:00::","synset_id":"male.n.02"},{"index":3,"sense_key":"male%1:15:00::","synset_id":"male.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"male%1:18:00::","call_ids":["senseval2.d000.s061.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"male%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":769,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001658,"input_uncached_usd":0.0015379999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0452594169182703}
{"item_id":"senseval2.d000.s061.t006","gold_sense_keys":["remark%2:32:01::"],"candidates":[{"index":1,"sense_key":"remark%2:32:00::","synset_id":"note.v.01"},{"index":2,"sense_key":"remark%2:32:01::","synset_id":"comment.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"remark%2:32:01::","call_ids":["senseval2.d000.s061.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"remark%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":768,"cached_input_tokens":0,"output_tokens":103,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002566,"input_uncached_usd":0.001536,"input_cached_usd":0.0,"output_usd":0.00103,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.81940595805645}
{"item_id":"senseval2.d000.s061.t007","gold_sense_keys":["frequency%1:28:00::"],"candidates":[{"index":1,"sense_key":"frequency%1:28:00::","synset_id":"frequency.n.01"},{"index":2,"sense_key":"frequency%1:24:00::","synset_id":"frequency.n.02"},{"index":3,"sense_key":"frequency%1:23:00::","synset_id":"frequency.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"frequency%1:28:00::","call_ids":["senseval2.d000.s061.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"frequency%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":830,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0017800000000000001,"input_uncached_usd":0.00166,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.000627417000942}
{"item_id":"senseval2.d000.s061.t008","gold_sense_keys":["woman%1:18:00::"],"candidates":[{"index":1,"sense_key":"woman%1:18:00::","synset_id":"woman.n.01"},{"index":2,"sense_key":"woman%1:18:01::","synset_id":"woman.n.02"},{"index":3,"sense_key":"woman%1:18:02::","synset_id":"charwoman.n.01"},{"index":4,"sense_key":"woman%1:14:00::","synset_id":"womanhood.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"woman%1:18:00::","call_ids":["senseval2.d000.s061.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"woman%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":918,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002486,"input_uncached_usd":0.001836,"input_cached_usd":0.0,"output_usd":0.0006500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5045729579869658}
{"item_id":"senseval2.d000.s061.t010","gold_sense_keys":["suggest%2:32:00::"],"candidates":[{"index":1,"sense_key":"suggest%2:32:00::","synset_id":"propose.v.01"},{"index":2,"sense_key":"suggest%2:32:04::","synset_id":"hint.v.01"},{"index":3,"sense_key":"suggest%2:32:02::","synset_id":"suggest.v.03"},{"index":4,"sense_key":"suggest%2:32:03::","synset_id":"indicate.v.05"},{"index":5,"sense_key":"suggest%2:32:01::","synset_id":"suggest.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"suggest%2:32:00::","call_ids":["senseval2.d000.s061.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"suggest%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":928,"cached_input_tokens":0,"output_tokens":74,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0025960000000000002,"input_uncached_usd":0.001856,"input_cached_usd":0.0,"output_usd":0.0007400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2665358340600505}
{"item_id":"senseval2.d000.s061.t012","gold_sense_keys":["back%4:02:05::"],"candidates":[{"index":1,"sense_key":"back%4:02:04::","synset_id":"back.r.01"},{"index":2,"sense_key":"back%4:02:00::","synset_id":"back.r.02"},{"index":3,"sense_key":"back%4:02:05::","synset_id":"back.r.03"},{"index":4,"sense_key":"back%4:02:01::","synset_id":"back.r.04"},{"index":5,"sense_key":"back%4:02:06::","synset_id":"back.r.05"},{"index":6,"sense_key":"back%4:02:03::","synset_id":"back.r.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"back%4:02:05::","call_ids":["senseval2.d000.s061.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"back%4:02:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":940,"cached_input_tokens":0,"output_tokens":87,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00275,"input_uncached_usd":0.00188,"input_cached_usd":0.0,"output_usd":0.0008700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2381033750716597}
{"item_id":"senseval2.d000.s061.t013","gold_sense_keys":["traditional%3:00:00::"],"candidates":[{"index":1,"sense_key":"traditional%3:00:00::","synset_id":"traditional.a.01"},{"index":2,"sense_key":"traditional%5:00:00:orthodox:00","synset_id":"traditional.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"traditional%3:00:00::","call_ids":["senseval2.d000.s061.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"traditional%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":753,"cached_input_tokens":0,"output_tokens":90,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002406,"input_uncached_usd":0.001506,"input_cached_usd":0.0,"output_usd":0.0009000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.901216333033517}
{"item_id":"senseval2.d000.s061.t014","gold_sense_keys":["role%1:04:00::","role%1:04:01::"],"candidates":[{"index":1,"sense_key":"role%1:04:00::","synset_id":"function.n.03"},{"index":2,"sense_key":"role%1:09:00::","synset_id":"character.n.04"},{"index":3,"sense_key":"role%1:07:00::","synset_id":"function.n.02"},{"index":4,"sense_key":"role%1:04:01::","synset_id":"role.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"role%1:04:01::","call_ids":["senseval2.d000.s061.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"role%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":892,"cached_input_tokens":0,"output_tokens":86,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002644,"input_uncached_usd":0.001784,"input_cached_usd":0.0,"output_usd":0.0008600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9844345420133322}
{"item_id":"senseval2.d000.s061.t015","gold_sense_keys":["make%2:36:07::"],"candidates":[{"index":1,"sense_key":"make%2:41:00::","synset_id":"make.v.01"},{"index":2,"sense_key":"make%2:30:00::","synset_id":"make.v.02"},{"index":3,"sense_key":"make%2:36:00::","synset_id":"make.v.03"},{"index":4,"sense_key":"make%2:32:00::","synset_id":"induce.v.02"},{"index":5,"sense_key":"make%2:36:08::","synset_id":"cause.v.01"},{"index":6,"sense_key":"make%2:36:01::","synset_id":"produce.v.02"},{"index":7,"sense_key":"make%2:31:00::","synset_id":"draw.v.04"},{"index":8,"sense_key":"make%2:36:15::","synset_id":"make.v.08"},{"index":9,"sense_key":"make%2:36:09::","synset_id":"create.v.05"},{"index":10,"sense_key":"make%2:40:01::","synset_id":"gain.v.08"},{"index":11,"sense_key":"make%2:36:05::","synset_id":"do.v.08"},{"index":12,"sense_key":"make%2:42:05::","synset_id":"form.v.02"},{"index":13,"sense_key":"make%2:38:02::","synset_id":"reach.v.07"},{"index":14,"sense_key":"make%2:42:00::","synset_id":"make.v.14"},{"index":15,"sense_key":"make%2:36:11::","synset_id":"make.v.15"},{"index":16,"sense_key":"make%2:36:12::","synset_id":"make.v.16"},{"index":17,"sense_key":"make%2:36:13::","synset_id":"construct.v.01"},{"index":18,"sense_key":"make%2:30:15::","synset_id":"make.v.18"},{"index":19,"sense_key":"make%2:29:08::","synset_id":"make.v.19"},{"index":20,"sense_key":"make%2:41:03::","synset_id":"name.v.03"},{"index":21,"sense_key":"make%2:40:02::","synset_id":"have.v.17"},{"index":22,"sense_key":"make%2:38:00::","synset_id":"reach.v.01"},{"index":23,"sense_key":"make%2:31:13::","synset_id":"lay_down.v.01"},{"index":24,"sense_key":"make%2:41:13::","synset_id":"make.v.24"},{"index":25,"sense_key":"make%2:41:01::","synset_id":"make.v.25"},{"index":26,"sense_key":"make%2:36:04::","synset_id":"hold.v.03"},{"index":27,"sense_key":"make%2:30:02::","synset_id":"make.v.27"},{"index":28,"sense_key":"make%2:38:05::","synset_id":"take.v.27"},{"index":29,"sense_key":"make%2:29:00::","synset_id":"stool.v.04"},{"index":30,"sense_key":"make%2:42:09::","synset_id":"make.v.30"},{"index":31,"sense_key":"make%2:42:13::","synset_id":"make.v.31"},{"index":32,"sense_key":"make%2:42:12::","synset_id":"make.v.32"},{"index":33,"sense_key":"make%2:42:11::","synset_id":"make.v.33"},{"index":34,"sense_key":"make%2:42:15::","synset_id":"make.v.34"},{"index":35,"sense_key":"make%2:39:12::","synset_id":"make.v.35"},{"index":36,"sense_key":"make%2:38:10::","synset_id":"make.v.36"},{"index":37,"sense_key":"make%2:38:15::","synset_id":"make.v.37"},{"index":38,"sense_key":"make%2:36:10::","synset_id":"make.v.38"},{"index":39,"sense_key":"make%2:36:07::","synset_id":"cook.v.02"},{"index":40,"sense_key":"make%2:35:13::","synset_id":"seduce.v.01"},{"index":41,"sense_key":"make%2:32:14::","synset_id":"make.v.41"},{"index":42,"sense_key":"make%2:32:09::","synset_id":"make.v.42"},{"index":43,"sense_key":"make%2:31:10::","synset_id":"make.v.43"},{"index":44,"sense_key":"make%2:31:12::","synset_id":"make.v.44"},{"index":45,"sense_key":"make%2:30:10::","synset_id":"make.v.45"},{"index":46,"sense_key":"make%2:30:14::","synset_id":"make.v.46"},{"index":47,"sense_key":"make%2:30:12::","synset_id":"make.v.47"},{"index":48,"sense_key":"make%2:29:06::","synset_id":"make.v.48"},{"index":49,"sense_key":"make%2:29:01::","synset_id":"make.v.49"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":39,"chosen_sense_key":"make%2:36:07::","call_ids":["senseval2.d000.s061.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":39,"predicted_sense_key":"make%2:36:07::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":3302,"cached_input_tokens":0,"output_tokens":129,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.007894,"input_uncached_usd":0.006604,"input_cached_usd":0.0,"output_usd":0.0012900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.275317332940176}
{"item_id":"senseval2.d000.s061.t016","gold_sense_keys":["tea%1:13:00::"],"candidates":[{"index":1,"sense_key":"tea%1:13:00::","synset_id":"tea.n.01"},{"index":2,"sense_key":"tea%1:13:01::","synset_id":"tea.n.02"},{"index":3,"sense_key":"tea%1:20:00::","synset_id":"tea.n.03"},{"index":4,"sense_key":"tea%1:14:00::","synset_id":"tea.n.04"},{"index":5,"sense_key":"tea%1:13:02::","synset_id":"tea.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"tea%1:13:00::","call_ids":["senseval2.d000.s061.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"tea%1:13:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1003,"cached_input_tokens":0,"output_tokens":42,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002426,"input_uncached_usd":0.002006,"input_cached_usd":0.0,"output_usd":0.00042,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6262204999802634}
{"item_id":"senseval2.d000.s061.t017","gold_sense_keys":["meeting%1:14:00::","meeting%1:14:01::"],"candidates":[{"index":1,"sense_key":"meeting%1:14:00::","synset_id":"meeting.n.01"},{"index":2,"sense_key":"meeting%1:14:01::","synset_id":"meeting.n.02"},{"index":3,"sense_key":"meeting%1:11:00::","synset_id":"meeting.n.03"},{"index":4,"sense_key":"meeting%1:04:00::","synset_id":"meeting.n.04"},{"index":5,"sense_key":"meeting%1:04:02::","synset_id":"merging.n.01"},{"index":6,"sense_key":"meeting%1:15:00::","synset_id":"confluence.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"meeting%1:14:00::","call_ids":["senseval2.d000.s061.t017__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"meeting%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1059,"cached_input_tokens":0,"output_tokens":129,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0034080000000000004,"input_uncached_usd":0.002118,"input_cached_usd":0.0,"output_usd":0.0012900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7759877500357106}
{"item_id":"senseval2.d000.s062.t000","gold_sense_keys":["torrent%1:23:00::"],"candidates":[{"index":1,"sense_key":"torrent%1:19:01::","synset_id":"downpour.n.01"},{"index":2,"sense_key":"torrent%1:11:00::","synset_id":"torrent.n.02"},{"index":3,"sense_key":"torrent%1:23:00::","synset_id":"flood.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"torrent%1:23:00::","call_ids":["senseval2.d000.s062.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"torrent%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":844,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001808,"input_uncached_usd":0.0016879999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5870081660104915}
{"item_id":"senseval2.d000.s062.t001","gold_sense_keys":["reply%1:10:01::"],"candidates":[{"index":1,"sense_key":"reply%1:10:01::","synset_id":"answer.n.01"},{"index":2,"sense_key":"reply%1:10:00::","synset_id":"reply.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"reply%1:10:01::","call_ids":["senseval2.d000.s062.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"reply%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":790,"cached_input_tokens":0,"output_tokens":128,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00286,"input_uncached_usd":0.00158,"input_cached_usd":0.0,"output_usd":0.00128,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8833602500380948}
{"item_id":"senseval2.d000.s062.t002","gold_sense_keys":["follow%2:42:00::","follow%2:42:01::","follow%2:42:05::"],"candidates":[{"index":1,"sense_key":"follow%2:38:00::","synset_id":"follow.v.01"},{"index":2,"sense_key":"follow%2:42:00::","synset_id":"postdate.v.01"},{"index":3,"sense_key":"follow%2:42:05::","synset_id":"follow.v.03"},{"index":4,"sense_key":"follow%2:38:01::","synset_id":"follow.v.04"},{"index":5,"sense_key":"follow%2:41:00::","synset_id":"comply.v.01"},{"index":6,"sense_key":"follow%2:42:01::","synset_id":"follow.v.06"},{"index":7,"sense_key":"follow%2:30:00::","synset_id":"follow.v.07"},{"index":8,"sense_key":"follow%2:42:02::","synset_id":"follow.v.08"},{"index":9,"sense_key":"follow%2:40:00::","synset_id":"adopt.v.01"},{"index":10,"sense_key":"follow%2:42:03::","synset_id":"follow.v.10"},{"index":11,"sense_key":"follow%2:36:00::","synset_id":"take_after.v.02"},{"index":12,"sense_key":"follow%2:31:00::","synset_id":"trace.v.01"},{"index":13,"sense_key":"follow%2:41:02::","synset_id":"watch.v.02"},{"index":14,"sense_key":"follow%2:41:01::","synset_id":"succeed.v.02"},{"index":15,"sense_key":"follow%2:36:01::","synset_id":"play_along.v.02"},{"index":16,"sense_key":"follow%2:30:01::","synset_id":"keep_up.v.04"},{"index":17,"sense_key":"follow%2:42:04::","synset_id":"come.v.05"},{"index":18,"sense_key":"follow%2:41:10::","synset_id":"follow.v.18"},{"index":19,"sense_key":"follow%2:41:11::","synset_id":"follow.v.19"},{"index":20,"sense_key":"follow%2:41:09::","synset_id":"be.v.08"},{"index":21,"sense_key":"follow%2:39:13::","synset_id":"surveil.v.01"},{"index":22,"sense_key":"follow%2:38:13::","synset_id":"pursue.v.02"},{"index":23,"sense_key":"follow%2:31:14::","synset_id":"follow.v.23"},{"index":24,"sense_key":"follow%2:30:12::","synset_id":"stick_to.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"follow%2:42:01::","call_ids":["senseval2.d000.s062.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"follow%2:42:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2175,"cached_input_tokens":0,"output_tokens":128,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00563,"input_uncached_usd":0.00435,"input_cached_usd":0.0,"output_usd":0.00128,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7452007919782773}
{"item_id":"senseval2.d000.s062.t003","gold_sense_keys":["woman%1:18:00::"],"candidates":[{"index":1,"sense_key":"woman%1:18:00::","synset_id":"woman.n.01"},{"index":2,"sense_key":"woman%1:18:01::","synset_id":"woman.n.02"},{"index":3,"sense_key":"woman%1:18:02::","synset_id":"charwoman.n.01"},{"index":4,"sense_key":"woman%1:14:00::","synset_id":"womanhood.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"woman%1:18:00::","call_ids":["senseval2.d000.s062.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"woman%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":930,"cached_input_tokens":0,"output_tokens":81,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00267,"input_uncached_usd":0.0018599999999999999,"input_cached_usd":0.0,"output_usd":0.0008100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3647571670589969}
{"item_id":"senseval2.d000.s062.t004","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"},{"index":2,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"},{"index":3,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":4,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s062.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":850,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00182,"input_uncached_usd":0.0017,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1946620419621468}
{"item_id":"senseval2.d000.s062.t005","gold_sense_keys":["observe%2:32:00::"],"candidates":[{"index":1,"sense_key":"observe%2:39:02::","synset_id":"detect.v.01"},{"index":2,"sense_key":"observe%2:32:00::","synset_id":"note.v.01"},{"index":3,"sense_key":"observe%2:39:01::","synset_id":"note.v.03"},{"index":4,"sense_key":"observe%2:39:00::","synset_id":"observe.v.04"},{"index":5,"sense_key":"observe%2:41:01::","synset_id":"respect.v.02"},{"index":6,"sense_key":"observe%2:41:02::","synset_id":"observe.v.06"},{"index":7,"sense_key":"observe%2:41:00::","synset_id":"watch.v.02"},{"index":8,"sense_key":"observe%2:31:00::","synset_id":"observe.v.08"},{"index":9,"sense_key":"observe%2:41:04::","synset_id":"observe.v.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"observe%2:32:00::","call_ids":["senseval2.d000.s062.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"observe%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1236,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0025919999999999997,"input_uncached_usd":0.002472,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1388833749806508}
{"item_id":"senseval2.d000.s062.t006","gold_sense_keys":["average%5:00:00:common:01"],"candidates":[{"index":1,"sense_key":"average%5:00:02:normal:01","synset_id":"average.s.01"},{"index":2,"sense_key":"average%5:00:00:common:01","synset_id":"average.s.02"},{"index":3,"sense_key":"average%5:00:00:ordinary:00","synset_id":"average.s.03"},{"index":4,"sense_key":"average%5:00:00:moderate:00","synset_id":"average.s.04"},{"index":5,"sense_key":"average%5:00:03:normal:01","synset_id":"modal.s.01"},{"index":6,"sense_key":"average%5:00:01:normal:01","synset_id":"median.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"average%5:00:00:common:01","call_ids":["senseval2.d000.s062.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"average%5:00:00:common:01","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1116,"cached_input_tokens":0,"output_tokens":144,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003672,"input_uncached_usd":0.002232,"input_cached_usd":0.0,"output_usd":0.00144,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.986770791001618}
{"item_id":"senseval2.d000.s062.t007","gold_sense_keys":["male%3:00:00::"],"candidates":[{"index":1,"sense_key":"male%3:00:00::","synset_id":"male.a.01"},{"index":2,"sense_key":"male%5:00:02:masculine:01","synset_id":"male.s.01"},{"index":3,"sense_key":"male%5:00:01:male:00","synset_id":"male.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"male%3:00:00::","call_ids":["senseval2.d000.s062.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"male%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":838,"cached_input_tokens":0,"output_tokens":291,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004586,"input_uncached_usd":0.001676,"input_cached_usd":0.0,"output_usd":0.0029100000000000003,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.2319065410410985}
{"item_id":"senseval2.d000.s062.t008","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"},{"index":2,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"},{"index":3,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":4,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s062.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":850,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00182,"input_uncached_usd":0.0017,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9952474160818383}
{"item_id":"senseval2.d000.s062.t013","gold_sense_keys":["badly%4:02:00::","badly%4:02:12::"],"candidates":[{"index":1,"sense_key":"badly%4:02:02::","synset_id":"badly.r.01"},{"index":2,"sense_key":"badly%4:02:00::","synset_id":"ill.r.01"},{"index":3,"sense_key":"badly%4:02:06::","synset_id":"badly.r.03"},{"index":4,"sense_key":"badly%4:02:05::","synset_id":"badly.r.04"},{"index":5,"sense_key":"badly%4:02:04::","synset_id":"badly.r.05"},{"index":6,"sense_key":"badly%4:02:03::","synset_id":"badly.r.06"},{"index":7,"sense_key":"badly%4:02:12::","synset_id":"badly.r.07"},{"index":8,"sense_key":"badly%4:02:10::","synset_id":"badly.r.08"},{"index":9,"sense_key":"badly%4:02:07::","synset_id":"ill.r.02"},{"index":10,"sense_key":"badly%4:02:08::","synset_id":"badly.r.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"badly%4:02:00::","call_ids":["senseval2.d000.s062.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"badly%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1354,"cached_input_tokens":0,"output_tokens":158,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004288,"input_uncached_usd":0.002708,"input_cached_usd":0.0,"output_usd":0.00158,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1113382499897853}
{"item_id":"senseval2.d000.s062.t014","gold_sense_keys":["dressed%5:00:00:clothed:00","dressed%5:00:02:clothed:00"],"candidates":[{"index":1,"sense_key":"dressed%5:00:00:clothed:00","synset_id":"appareled.s.01"},{"index":2,"sense_key":"dressed%5:00:00:treated:02","synset_id":"dressed.s.02"},{"index":3,"sense_key":"dressed%5:00:00:finished:02","synset_id":"dressed.s.03"},{"index":4,"sense_key":"dressed%5:00:02:clothed:00","synset_id":"dressed.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"dressed%5:00:00:clothed:00","call_ids":["senseval2.d000.s062.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"dressed%5:00:00:clothed:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":921,"cached_input_tokens":0,"output_tokens":90,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002742,"input_uncached_usd":0.0018419999999999999,"input_cached_usd":0.0,"output_usd":0.0009000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.477460041991435}
{"item_id":"senseval2.d000.s062.t017","gold_sense_keys":["large%3:00:00::"],"candidates":[{"index":1,"sense_key":"large%3:00:00::","synset_id":"large.a.01"},{"index":2,"sense_key":"large%5:00:00:significant:00","synset_id":"large.s.01"},{"index":3,"sense_key":"large%5:00:00:rhetorical:00","synset_id":"bombastic.s.01"},{"index":4,"sense_key":"large%5:00:00:generous:02","synset_id":"big.s.10"},{"index":5,"sense_key":"large%5:00:00:conspicuous:00","synset_id":"big.s.04"},{"index":6,"sense_key":"large%5:00:00:comprehensive:00","synset_id":"large.s.05"},{"index":7,"sense_key":"large%5:00:00:pregnant:00","synset_id":"big.s.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"large%3:00:00::","call_ids":["senseval2.d000.s062.t017__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"large%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1118,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0023559999999999996,"input_uncached_usd":0.0022359999999999997,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.018945666961372}
{"item_id":"senseval2.d000.s062.t019","gold_sense_keys":["unwashed%5:00:00:dirty:01"],"candidates":[{"index":1,"sense_key":"unwashed%5:00:00:lowborn:00","synset_id":"common.s.04"},{"index":2,"sense_key":"unwashed%5:00:00:dirty:01","synset_id":"unwashed.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"unwashed%5:00:00:dirty:01","call_ids":["senseval2.d000.s062.t019__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"unwashed%5:00:00:dirty:01","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":795,"cached_input_tokens":0,"output_tokens":104,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00263,"input_uncached_usd":0.0015899999999999998,"input_cached_usd":0.0,"output_usd":0.0010400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6210882499581203}
{"item_id":"senseval2.d000.s063.t000","gold_sense_keys":["woman%1:18:00::"],"candidates":[{"index":1,"sense_key":"woman%1:18:00::","synset_id":"woman.n.01"},{"index":2,"sense_key":"woman%1:18:01::","synset_id":"woman.n.02"},{"index":3,"sense_key":"woman%1:18:02::","synset_id":"charwoman.n.01"},{"index":4,"sense_key":"woman%1:14:00::","synset_id":"womanhood.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"woman%1:18:00::","call_ids":["senseval2.d000.s063.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"woman%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":905,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00247,"input_uncached_usd":0.00181,"input_cached_usd":0.0,"output_usd":0.0006600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3841000830288976}
{"item_id":"senseval2.d000.s063.t001","gold_sense_keys":["write%2:32:01::"],"candidates":[{"index":1,"sense_key":"write%2:36:00::","synset_id":"write.v.01"},{"index":2,"sense_key":"write%2:32:00::","synset_id":"write.v.02"},{"index":3,"sense_key":"write%2:36:02::","synset_id":"publish.v.03"},{"index":4,"sense_key":"write%2:32:08::","synset_id":"write.v.04"},{"index":5,"sense_key":"write%2:32:01::","synset_id":"write.v.05"},{"index":6,"sense_key":"write%2:36:04::","synset_id":"compose.v.02"},{"index":7,"sense_key":"write%2:36:01::","synset_id":"write.v.07"},{"index":8,"sense_key":"write%2:32:03::","synset_id":"write.v.08"},{"index":9,"sense_key":"write%2:36:03::","synset_id":"spell.v.03"},{"index":10,"sense_key":"write%2:36:10::","synset_id":"write.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"write%2:32:01::","call_ids":["senseval2.d000.s063.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"write%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1180,"cached_input_tokens":0,"output_tokens":150,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0038599999999999997,"input_uncached_usd":0.0023599999999999997,"input_cached_usd":0.0,"output_usd":0.0015,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8717496250756085}
{"item_id":"senseval2.d000.s063.t002","gold_sense_keys":["say%2:32:00::"],"candidates":[{"index":1,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":6,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":9,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":10,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":11,"sense_key":"say%2:32:09::","synset_id":"say.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"say%2:32:00::","call_ids":["senseval2.d000.s063.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"say%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1281,"cached_input_tokens":0,"output_tokens":189,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004452,"input_uncached_usd":0.002562,"input_cached_usd":0.0,"output_usd":0.0018900000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.235116875031963}
{"item_id":"senseval2.d000.s063.t003","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["senseval2.d000.s063.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":882,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001884,"input_uncached_usd":0.001764,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7021907500457019}
{"item_id":"senseval2.d000.s063.t004","gold_sense_keys":["never%4:02:00::"],"candidates":[{"index":1,"sense_key":"never%4:02:00::","synset_id":"never.r.01"},{"index":2,"sense_key":"never%4:02:01::","synset_id":"never.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"never%4:02:00::","call_ids":["senseval2.d000.s063.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"never%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":738,"cached_input_tokens":0,"output_tokens":76,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002236,"input_uncached_usd":0.0014759999999999999,"input_cached_usd":0.0,"output_usd":0.00076,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4381066670175642}
{"item_id":"senseval2.d000.s063.t005","gold_sense_keys":["know%2:31:04::"],"candidates":[{"index":1,"sense_key":"know%2:31:01::","synset_id":"know.v.01"},{"index":2,"sense_key":"know%2:31:03::","synset_id":"know.v.02"},{"index":3,"sense_key":"know%2:31:02::","synset_id":"know.v.03"},{"index":4,"sense_key":"know%2:31:00::","synset_id":"know.v.04"},{"index":5,"sense_key":"know%2:31:04::","synset_id":"know.v.05"},{"index":6,"sense_key":"know%2:31:06::","synset_id":"acknowledge.v.06"},{"index":7,"sense_key":"know%2:31:15::","synset_id":"know.v.07"},{"index":8,"sense_key":"know%2:35:00::","synset_id":"sleep_together.v.01"},{"index":9,"sense_key":"know%2:31:11::","synset_id":"know.v.09"},{"index":10,"sense_key":"know%2:31:12::","synset_id":"know.v.10"},{"index":11,"sense_key":"know%2:31:14::","synset_id":"know.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"know%2:31:04::","call_ids":["senseval2.d000.s063.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"know%2:31:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1344,"cached_input_tokens":0,"output_tokens":106,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003748,"input_uncached_usd":0.002688,"input_cached_usd":0.0,"output_usd":0.0010600000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4234679589280859}
{"item_id":"senseval2.d000.s063.t006","gold_sense_keys":["lady%1:18:02::"],"candidates":[{"index":1,"sense_key":"lady%1:18:02::","synset_id":"lady.n.01"},{"index":2,"sense_key":"lady%1:18:01::","synset_id":"dame.n.02"},{"index":3,"sense_key":"lady%1:18:00::","synset_id":"lady.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"lady%1:18:02::","call_ids":["senseval2.d000.s063.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"lady%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":795,"cached_input_tokens":0,"output_tokens":177,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00336,"input_uncached_usd":0.0015899999999999998,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1422251670155674}
{"item_id":"senseval2.d000.s063.t008","gold_sense_keys":["belfry%1:06:00::"],"candidates":[{"index":1,"sense_key":"belfry%1:06:01::","synset_id":"campanile.n.01"},{"index":2,"sense_key":"belfry%1:06:00::","synset_id":"belfry.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"belfry%1:06:00::","call_ids":["senseval2.d000.s063.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"belfry%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":717,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002104,"input_uncached_usd":0.001434,"input_cached_usd":0.0,"output_usd":0.00067,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3899889590684325}
{"item_id":"senseval2.d000.s064.t000","gold_sense_keys":["see%2:39:00::","see%2:39:02::"],"candidates":[{"index":1,"sense_key":"see%2:39:00::","synset_id":"see.v.01"},{"index":2,"sense_key":"see%2:31:01::","synset_id":"understand.v.02"},{"index":3,"sense_key":"see%2:39:02::","synset_id":"witness.v.02"},{"index":4,"sense_key":"see%2:36:00::","synset_id":"visualize.v.01"},{"index":5,"sense_key":"see%2:31:00::","synset_id":"see.v.05"},{"index":6,"sense_key":"see%2:31:02::","synset_id":"learn.v.02"},{"index":7,"sense_key":"see%2:39:03::","synset_id":"watch.v.03"},{"index":8,"sense_key":"see%2:38:00::","synset_id":"meet.v.01"},{"index":9,"sense_key":"see%2:32:00::","synset_id":"determine.v.08"},{"index":10,"sense_key":"see%2:31:03::","synset_id":"see.v.10"},{"index":11,"sense_key":"see%2:41:11::","synset_id":"see.v.11"},{"index":12,"sense_key":"see%2:41:12::","synset_id":"see.v.12"},{"index":13,"sense_key":"see%2:41:10::","synset_id":"visit.v.01"},{"index":14,"sense_key":"see%2:41:14::","synset_id":"attend.v.02"},{"index":15,"sense_key":"see%2:35:09::","synset_id":"see.v.15"},{"index":16,"sense_key":"see%2:41:00::","synset_id":"go_steady.v.01"},{"index":17,"sense_key":"see%2:39:05::","synset_id":"see.v.17"},{"index":18,"sense_key":"see%2:31:08::","synset_id":"see.v.18"},{"index":19,"sense_key":"see%2:39:11::","synset_id":"see.v.19"},{"index":20,"sense_key":"see%2:39:13::","synset_id":"examine.v.02"},{"index":21,"sense_key":"see%2:39:12::","synset_id":"experience.v.01"},{"index":22,"sense_key":"see%2:38:03::","synset_id":"see.v.22"},{"index":23,"sense_key":"see%2:33:00::","synset_id":"see.v.23"},{"index":24,"sense_key":"see%2:31:13::","synset_id":"interpret.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"see%2:39:02::","call_ids":["senseval2.d000.s064.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"see%2:39:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2165,"cached_input_tokens":0,"output_tokens":122,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.005549999999999999,"input_uncached_usd":0.00433,"input_cached_usd":0.0,"output_usd":0.0012200000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7523123340215534}
{"item_id":"senseval2.d000.s064.t001","gold_sense_keys":["man%1:18:00::"],"candidates":[{"index":1,"sense_key":"man%1:18:00::","synset_id":"man.n.01"},{"index":2,"sense_key":"man%1:18:03::","synset_id":"serviceman.n.01"},{"index":3,"sense_key":"man%1:18:04::","synset_id":"man.n.03"},{"index":4,"sense_key":"man%1:05:01::","synset_id":"homo.n.02"},{"index":5,"sense_key":"man%1:18:08::","synset_id":"man.n.05"},{"index":6,"sense_key":"man%1:18:06::","synset_id":"man.n.06"},{"index":7,"sense_key":"man%1:18:05::","synset_id":"valet.n.01"},{"index":8,"sense_key":"man%1:18:07::","synset_id":"man.n.08"},{"index":9,"sense_key":"man%1:15:00::","synset_id":"man.n.09"},{"index":10,"sense_key":"man%1:06:00::","synset_id":"man.n.10"},{"index":11,"sense_key":"man%1:05:00::","synset_id":"world.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"man%1:18:00::","call_ids":["senseval2.d000.s064.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"man%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1350,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00326,"input_uncached_usd":0.0026999999999999997,"input_cached_usd":0.0,"output_usd":0.0005600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3499620420625433}
{"item_id":"senseval2.d000.s064.t002","gold_sense_keys":["die%2:30:00::"],"candidates":[{"index":1,"sense_key":"die%2:30:00::","synset_id":"die.v.01"},{"index":2,"sense_key":"die%2:39:00::","synset_id":"die.v.02"},{"index":3,"sense_key":"die%2:37:00::","synset_id":"die.v.03"},{"index":4,"sense_key":"die%2:30:04::","synset_id":"fail.v.04"},{"index":5,"sense_key":"die%2:37:03::","synset_id":"die.v.05"},{"index":6,"sense_key":"die%2:37:01::","synset_id":"die.v.06"},{"index":7,"sense_key":"die%2:35:00::","synset_id":"die.v.07"},{"index":8,"sense_key":"die%2:33:00::","synset_id":"die.v.08"},{"index":9,"sense_key":"die%2:30:05::","synset_id":"die.v.09"},{"index":10,"sense_key":"die%2:30:01::","synset_id":"die.v.10"},{"index":11,"sense_key":"die%2:30:02::","synset_id":"die.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"die%2:30:00::","call_ids":["senseval2.d000.s064.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"die%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1253,"cached_input_tokens":0,"output_tokens":89,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003396,"input_uncached_usd":0.002506,"input_cached_usd":0.0,"output_usd":0.0008900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5229613330448046}
{"item_id":"senseval2.d001.s000.t000","gold_sense_keys":["medical%3:01:00::"],"candidates":[{"index":1,"sense_key":"medical%3:01:00::","synset_id":"medical.a.01"},{"index":2,"sense_key":"medical%3:00:00::","synset_id":"medical.a.02"},{"index":3,"sense_key":"medical%3:01:02::","synset_id":"aesculapian.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"medical%3:01:00::","call_ids":["senseval2.d001.s000.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"medical%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":481,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001082,"input_uncached_usd":0.000962,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0563739580102265}
{"item_id":"senseval2.d001.s000.t002","gold_sense_keys":["start%2:30:00::"],"candidates":[{"index":1,"sense_key":"start%2:30:00::","synset_id":"get_down.v.07"},{"index":2,"sense_key":"start%2:30:01::","synset_id":"begin.v.03"},{"index":3,"sense_key":"start%2:38:02::","synset_id":"depart.v.03"},{"index":4,"sense_key":"start%2:42:00::","synset_id":"begin.v.02"},{"index":5,"sense_key":"start%2:36:00::","synset_id":"originate.v.02"},{"index":6,"sense_key":"start%2:36:01::","synset_id":"start.v.06"},{"index":7,"sense_key":"start%2:38:04::","synset_id":"startle.v.02"},{"index":8,"sense_key":"start%2:38:01::","synset_id":"start.v.08"},{"index":9,"sense_key":"start%2:38:00::","synset_id":"start.v.09"},{"index":10,"sense_key":"start%2:41:00::","synset_id":"start.v.10"},{"index":11,"sense_key":"start%2:33:00::","synset_id":"start.v.11"},{"index":12,"sense_key":"start%2:42:03::","synset_id":"begin.v.07"},{"index":13,"sense_key":"start%2:41:04::","synset_id":"begin.v.08"},{"index":14,"sense_key":"start%2:38:03::","synset_id":"start.v.14"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"start%2:30:00::","call_ids":["senseval2.d001.s000.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"start%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1198,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0025159999999999996,"input_uncached_usd":0.0023959999999999997,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":6.6305553750135005}
{"item_id":"senseval2.d001.s000.t003","gold_sense_keys":["handful%1:23:01::"],"candidates":[{"index":1,"sense_key":"handful%1:23:01::","synset_id":"handful.n.01"},{"index":2,"sense_key":"handful%1:23:00::","synset_id":"handful.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"handful%1:23:01::","call_ids":["senseval2.d001.s000.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"handful%1:23:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":407,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0009339999999999999,"input_uncached_usd":0.0008139999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1497193749528378}
{"item_id":"senseval2.d001.s000.t005","gold_sense_keys":["damaged%3:00:00::"],"candidates":[{"index":1,"sense_key":"damaged%3:00:00::","synset_id":"damaged.a.01"},{"index":2,"sense_key":"damaged%5:00:00:disreputable:00","synset_id":"discredited.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"damaged%3:00:00::","call_ids":["senseval2.d001.s000.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"damaged%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":449,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001018,"input_uncached_usd":0.0008979999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0996519590262324}
{"item_id":"senseval2.d001.s000.t006","gold_sense_keys":["unleash%2:35:02::"],"candidates":[{"index":1,"sense_key":"unleash%2:35:00::","synset_id":"unleash.v.01"},{"index":2,"sense_key":"unleash%2:35:01::","synset_id":"unleash.v.02"},{"index":3,"sense_key":"unleash%2:35:02::","synset_id":"unleash.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"unleash%2:35:02::","call_ids":["senseval2.d001.s000.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"unleash%2:35:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":465,"cached_input_tokens":0,"output_tokens":154,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00247,"input_uncached_usd":0.0009299999999999999,"input_cached_usd":0.0,"output_usd":0.0015400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.96931237494573}
{"item_id":"senseval2.d001.s000.t007","gold_sense_keys":["chaotic%5:00:00:wild:02"],"candidates":[{"index":1,"sense_key":"chaotic%5:00:00:disorganized:00","synset_id":"chaotic.s.01"},{"index":2,"sense_key":"chaotic%5:00:00:wild:02","synset_id":"chaotic.s.02"},{"index":3,"sense_key":"chaotic%3:01:00::","synset_id":"chaotic.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"chaotic%5:00:00:wild:02","call_ids":["senseval2.d001.s000.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"chaotic%5:00:00:wild:02","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":466,"cached_input_tokens":0,"output_tokens":139,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0023220000000000003,"input_uncached_usd":0.000932,"input_cached_usd":0.0,"output_usd":0.0013900000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1961831250227988}
{"item_id":"senseval2.d001.s000.t008","gold_sense_keys":["growth%1:22:02::"],"candidates":[{"index":1,"sense_key":"growth%1:22:00::","synset_id":"growth.n.01"},{"index":2,"sense_key":"growth%1:22:01::","synset_id":"growth.n.02"},{"index":3,"sense_key":"growth%1:22:02::","synset_id":"increase.n.03"},{"index":4,"sense_key":"growth%1:14:00::","synset_id":"growth.n.04"},{"index":5,"sense_key":"growth%1:11:01::","synset_id":"emergence.n.01"},{"index":6,"sense_key":"growth%1:26:00::","synset_id":"growth.n.06"},{"index":7,"sense_key":"growth%1:17:00::","synset_id":"growth.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"growth%1:22:02::","call_ids":["senseval2.d001.s000.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"growth%1:22:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":754,"cached_input_tokens":0,"output_tokens":122,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0027280000000000004,"input_uncached_usd":0.001508,"input_cached_usd":0.0,"output_usd":0.0012200000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.058528000023216}
{"item_id":"senseval2.d001.s000.t009","gold_sense_keys":["cell%1:03:00::"],"candidates":[{"index":1,"sense_key":"cell%1:06:03::","synset_id":"cell.n.01"},{"index":2,"sense_key":"cell%1:03:00::","synset_id":"cell.n.02"},{"index":3,"sense_key":"cell%1:06:00::","synset_id":"cell.n.03"},{"index":4,"sense_key":"cell%1:14:00::","synset_id":"cell.n.04"},{"index":5,"sense_key":"cell%1:06:04::","synset_id":"cellular_telephone.n.01"},{"index":6,"sense_key":"cell%1:06:02::","synset_id":"cell.n.06"},{"index":7,"sense_key":"cell%1:06:01::","synset_id":"cell.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cell%1:03:00::","call_ids":["senseval2.d001.s000.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cell%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":723,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0015660000000000001,"input_uncached_usd":0.001446,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.464392250054516}
{"item_id":"senseval2.d001.s000.t010","gold_sense_keys":["characterize%2:42:01::"],"candidates":[{"index":1,"sense_key":"characterize%2:32:00::","synset_id":"qualify.v.06"},{"index":2,"sense_key":"characterize%2:42:01::","synset_id":"characterize.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"characterize%2:42:01::","call_ids":["senseval2.d001.s000.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"characterize%2:42:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":466,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001052,"input_uncached_usd":0.000932,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1778376660076901}
{"item_id":"senseval2.d001.s000.t011","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s000.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":631,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001382,"input_uncached_usd":0.0012619999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0117356670089066}
{"item_id":"senseval2.d001.s001.t001","gold_sense_keys":["say%2:32:01::","say%2:32:13::"],"candidates":[{"index":1,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":6,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":9,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":10,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":11,"sense_key":"say%2:32:09::","synset_id":"say.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"say%2:32:01::","call_ids":["senseval2.d001.s001.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"say%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1008,"cached_input_tokens":0,"output_tokens":101,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003026,"input_uncached_usd":0.002016,"input_cached_usd":0.0,"output_usd":0.00101,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5430078329518437}
{"item_id":"senseval2.d001.s001.t002","gold_sense_keys":["discovery%1:04:00::"],"candidates":[{"index":1,"sense_key":"discovery%1:04:00::","synset_id":"discovery.n.01"},{"index":2,"sense_key":"discovery%1:10:00::","synset_id":"discovery.n.02"},{"index":3,"sense_key":"discovery%1:09:00::","synset_id":"discovery.n.03"},{"index":4,"sense_key":"discovery%1:10:01::","synset_id":"discovery.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"discovery%1:04:00::","call_ids":["senseval2.d001.s001.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"discovery%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":540,"cached_input_tokens":0,"output_tokens":106,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0021400000000000004,"input_uncached_usd":0.00108,"input_cached_usd":0.0,"output_usd":0.0010600000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2865520420018584}
{"item_id":"senseval2.d001.s001.t004","gold_sense_keys":["recent%5:00:00:past:00"],"candidates":[{"index":1,"sense_key":"recent%5:00:00:new:00","synset_id":"recent.s.01"},{"index":2,"sense_key":"recent%5:00:00:past:00","synset_id":"late.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"recent%5:00:00:past:00","call_ids":["senseval2.d001.s001.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"recent%5:00:00:past:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":455,"cached_input_tokens":0,"output_tokens":104,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0019500000000000001,"input_uncached_usd":0.00091,"input_cached_usd":0.0,"output_usd":0.0010400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5415666250046343}
{"item_id":"senseval2.d001.s001.t005","gold_sense_keys":["month%1:28:01::"],"candidates":[{"index":1,"sense_key":"month%1:28:01::","synset_id":"calendar_month.n.01"},{"index":2,"sense_key":"month%1:28:00::","synset_id":"month.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"month%1:28:00::","call_ids":["senseval2.d001.s001.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"month%1:28:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":455,"cached_input_tokens":0,"output_tokens":96,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0018700000000000001,"input_uncached_usd":0.00091,"input_cached_usd":0.0,"output_usd":0.0009600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4596511670388281}
{"item_id":"senseval2.d001.s001.t007","gold_sense_keys":["new%3:00:00::","new%5:00:00:original:00","new%5:00:00:other:00","new%5:00:00:unprecedented:00"],"candidates":[{"index":1,"sense_key":"new%3:00:00::","synset_id":"new.a.01"},{"index":2,"sense_key":"new%5:00:00:original:00","synset_id":"fresh.s.03"},{"index":3,"sense_key":"new%5:00:00:inexperienced:00","synset_id":"raw.s.11"},{"index":4,"sense_key":"new%5:00:00:unprecedented:00","synset_id":"new.s.03"},{"index":5,"sense_key":"new%5:00:00:other:00","synset_id":"new.s.04"},{"index":6,"sense_key":"new%3:00:09::","synset_id":"new.a.06"},{"index":7,"sense_key":"new%5:00:02:original:00","synset_id":"newfangled.s.01"},{"index":8,"sense_key":"new%5:00:02:late:01","synset_id":"new.s.06"},{"index":9,"sense_key":"new%5:00:01:late:01","synset_id":"modern.s.04"},{"index":10,"sense_key":"new%5:00:00:early:02","synset_id":"new.s.08"},{"index":11,"sense_key":"new%5:00:00:unaccustomed:00","synset_id":"new.s.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"new%5:00:00:original:00","call_ids":["senseval2.d001.s001.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"new%5:00:00:original:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1040,"cached_input_tokens":0,"output_tokens":99,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00307,"input_uncached_usd":0.00208,"input_cached_usd":0.0,"output_usd":0.00099,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.398790208972059}
{"item_id":"senseval2.d001.s001.t009","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s001.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":668,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001456,"input_uncached_usd":0.001336,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.067165832966566}
{"item_id":"senseval2.d001.s001.t010","gold_sense_keys":["develop%2:30:00::"],"candidates":[{"index":1,"sense_key":"develop%2:36:01::","synset_id":"develop.v.01"},{"index":2,"sense_key":"develop%2:36:00::","synset_id":"evolve.v.01"},{"index":3,"sense_key":"develop%2:30:06::","synset_id":"develop.v.03"},{"index":4,"sense_key":"develop%2:29:00::","synset_id":"grow.v.08"},{"index":5,"sense_key":"develop%2:42:00::","synset_id":"originate.v.01"},{"index":6,"sense_key":"develop%2:30:04::","synset_id":"build_up.v.05"},{"index":7,"sense_key":"develop%2:32:00::","synset_id":"explicate.v.02"},{"index":8,"sense_key":"develop%2:31:00::","synset_id":"train.v.01"},{"index":9,"sense_key":"develop%2:30:05::","synset_id":"develop.v.09"},{"index":10,"sense_key":"develop%2:30:00::","synset_id":"develop.v.10"},{"index":11,"sense_key":"develop%2:30:09::","synset_id":"modernize.v.02"},{"index":12,"sense_key":"develop%2:30:01::","synset_id":"develop.v.12"},{"index":13,"sense_key":"develop%2:36:09::","synset_id":"develop.v.13"},{"index":14,"sense_key":"develop%2:30:10::","synset_id":"develop.v.14"},{"index":15,"sense_key":"develop%2:39:00::","synset_id":"develop.v.15"},{"index":16,"sense_key":"develop%2:35:04::","synset_id":"develop.v.16"},{"index":17,"sense_key":"develop%2:33:03::","synset_id":"develop.v.17"},{"index":18,"sense_key":"develop%2:33:02::","synset_id":"develop.v.18"},{"index":19,"sense_key":"develop%2:30:13::","synset_id":"develop.v.19"},{"index":20,"sense_key":"develop%2:30:03::","synset_id":"break.v.53"},{"index":21,"sense_key":"develop%2:30:12::","synset_id":"develop.v.21"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"develop%2:30:00::","call_ids":["senseval2.d001.s001.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"develop%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1730,"cached_input_tokens":0,"output_tokens":140,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00486,"input_uncached_usd":0.00346,"input_cached_usd":0.0,"output_usd":0.0014000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.426219334010966}
{"item_id":"senseval2.d001.s002.t001","gold_sense_keys":["understanding%1:09:01::"],"candidates":[{"index":1,"sense_key":"understanding%1:09:01::","synset_id":"understanding.n.01"},{"index":2,"sense_key":"understanding%1:10:00::","synset_id":"agreement.n.01"},{"index":3,"sense_key":"understanding%1:09:03::","synset_id":"sympathy.n.01"},{"index":4,"sense_key":"understanding%1:09:00::","synset_id":"reason.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"understanding%1:09:01::","call_ids":["senseval2.d001.s002.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"understanding%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":680,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00148,"input_uncached_usd":0.0013599999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.4188985829241574}
{"item_id":"senseval2.d001.s002.t003","gold_sense_keys":["expect%2:31:00::"],"candidates":[{"index":1,"sense_key":"expect%2:31:00::","synset_id":"expect.v.01"},{"index":2,"sense_key":"expect%2:32:00::","synset_id":"ask.v.04"},{"index":3,"sense_key":"expect%2:31:01::","synset_id":"expect.v.03"},{"index":4,"sense_key":"expect%2:31:05::","synset_id":"expect.v.04"},{"index":5,"sense_key":"expect%2:29:02::","synset_id":"expect.v.05"},{"index":6,"sense_key":"expect%2:29:00::","synset_id":"have_a_bun_in_the_oven.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"expect%2:31:00::","call_ids":["senseval2.d001.s002.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"expect%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":735,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00206,"input_uncached_usd":0.00147,"input_cached_usd":0.0,"output_usd":0.00059,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.411952708964236}
{"item_id":"senseval2.d001.s002.t004","gold_sense_keys":["produce%2:36:03::"],"candidates":[{"index":1,"sense_key":"produce%2:36:02::","synset_id":"produce.v.01"},{"index":2,"sense_key":"produce%2:36:00::","synset_id":"produce.v.02"},{"index":3,"sense_key":"produce%2:36:03::","synset_id":"produce.v.03"},{"index":4,"sense_key":"produce%2:39:01::","synset_id":"produce.v.04"},{"index":5,"sense_key":"produce%2:36:05::","synset_id":"grow.v.07"},{"index":6,"sense_key":"produce%2:39:00::","synset_id":"produce.v.06"},{"index":7,"sense_key":"produce%2:29:00::","synset_id":"grow.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"produce%2:36:03::","call_ids":["senseval2.d001.s002.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"produce%2:36:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":858,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001836,"input_uncached_usd":0.0017159999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7594564999453723}
{"item_id":"senseval2.d001.s002.t005","gold_sense_keys":["array%1:10:00::"],"candidates":[{"index":1,"sense_key":"array%1:14:00::","synset_id":"array.n.01"},{"index":2,"sense_key":"array%1:10:00::","synset_id":"array.n.02"},{"index":3,"sense_key":"array%1:06:01::","synset_id":"array.n.03"},{"index":4,"sense_key":"array%1:06:00::","synset_id":"array.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"array%1:10:00::","call_ids":["senseval2.d001.s002.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"array%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":542,"cached_input_tokens":0,"output_tokens":136,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002444,"input_uncached_usd":0.001084,"input_cached_usd":0.0,"output_usd":0.00136,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8362896660109982}
{"item_id":"senseval2.d001.s002.t006","gold_sense_keys":["new%3:00:00::","new%5:00:00:original:00","new%5:00:00:other:00","new%5:00:00:unprecedented:00"],"candidates":[{"index":1,"sense_key":"new%3:00:00::","synset_id":"new.a.01"},{"index":2,"sense_key":"new%5:00:00:original:00","synset_id":"fresh.s.03"},{"index":3,"sense_key":"new%5:00:00:inexperienced:00","synset_id":"raw.s.11"},{"index":4,"sense_key":"new%5:00:00:unprecedented:00","synset_id":"new.s.03"},{"index":5,"sense_key":"new%5:00:00:other:00","synset_id":"new.s.04"},{"index":6,"sense_key":"new%3:00:09::","synset_id":"new.a.06"},{"index":7,"sense_key":"new%5:00:02:original:00","synset_id":"newfangled.s.01"},{"index":8,"sense_key":"new%5:00:02:late:01","synset_id":"new.s.06"},{"index":9,"sense_key":"new%5:00:01:late:01","synset_id":"modern.s.04"},{"index":10,"sense_key":"new%5:00:00:early:02","synset_id":"new.s.08"},{"index":11,"sense_key":"new%5:00:00:unaccustomed:00","synset_id":"new.s.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"new%5:00:00:original:00","call_ids":["senseval2.d001.s002.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"new%5:00:00:original:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1046,"cached_input_tokens":0,"output_tokens":103,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003122,"input_uncached_usd":0.002092,"input_cached_usd":0.0,"output_usd":0.00103,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7193522499874234}
{"item_id":"senseval2.d001.s002.t007","gold_sense_keys":["strategy%1:09:00::"],"candidates":[{"index":1,"sense_key":"strategy%1:09:00::","synset_id":"scheme.n.01"},{"index":2,"sense_key":"strategy%1:09:01::","synset_id":"strategy.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"strategy%1:09:00::","call_ids":["senseval2.d001.s002.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"strategy%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":450,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00102,"input_uncached_usd":0.0009,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0859245840692893}
{"item_id":"senseval2.d001.s002.t008","gold_sense_keys":["future%3:00:00::","future%5:00:00:prospective:00"],"candidates":[{"index":1,"sense_key":"future%3:00:00::","synset_id":"future.a.01"},{"index":2,"sense_key":"future%5:00:00:prospective:00","synset_id":"future.s.01"},{"index":3,"sense_key":"future%5:00:00:incoming:00","synset_id":"future.s.02"},{"index":4,"sense_key":"future%3:01:00::","synset_id":"future.a.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"future%3:00:00::","call_ids":["senseval2.d001.s002.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"future%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":563,"cached_input_tokens":0,"output_tokens":251,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0036360000000000003,"input_uncached_usd":0.001126,"input_cached_usd":0.0,"output_usd":0.00251,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.0921925000147894}
{"item_id":"senseval2.d001.s002.t009","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s002.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":674,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0014680000000000001,"input_uncached_usd":0.001348,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1022606249898672}
{"item_id":"senseval2.d001.s002.t010","gold_sense_keys":["treatment%1:04:00::"],"candidates":[{"index":1,"sense_key":"treatment%1:04:00::","synset_id":"treatment.n.01"},{"index":2,"sense_key":"treatment%1:04:01::","synset_id":"treatment.n.02"},{"index":3,"sense_key":"treatment%1:07:00::","synset_id":"treatment.n.03"},{"index":4,"sense_key":"treatment%1:10:00::","synset_id":"discussion.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"treatment%1:04:00::","call_ids":["senseval2.d001.s002.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"treatment%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":630,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00138,"input_uncached_usd":0.0012599999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.175736290984787}
{"item_id":"senseval2.d001.s003.t000","gold_sense_keys":["future%1:28:00::"],"candidates":[{"index":1,"sense_key":"future%1:28:00::","synset_id":"future.n.01"},{"index":2,"sense_key":"future%1:10:00::","synset_id":"future.n.02"},{"index":3,"sense_key":"future%1:06:00::","synset_id":"future.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"future%1:28:00::","call_ids":["senseval2.d001.s003.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"future%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":545,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0012100000000000001,"input_uncached_usd":0.00109,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1439888749737293}
{"item_id":"senseval2.d001.s004.t002","gold_sense_keys":["develop%2:36:01::"],"candidates":[{"index":1,"sense_key":"develop%2:36:01::","synset_id":"develop.v.01"},{"index":2,"sense_key":"develop%2:36:00::","synset_id":"evolve.v.01"},{"index":3,"sense_key":"develop%2:30:06::","synset_id":"develop.v.03"},{"index":4,"sense_key":"develop%2:29:00::","synset_id":"grow.v.08"},{"index":5,"sense_key":"develop%2:42:00::","synset_id":"originate.v.01"},{"index":6,"sense_key":"develop%2:30:04::","synset_id":"build_up.v.05"},{"index":7,"sense_key":"develop%2:32:00::","synset_id":"explicate.v.02"},{"index":8,"sense_key":"develop%2:31:00::","synset_id":"train.v.01"},{"index":9,"sense_key":"develop%2:30:05::","synset_id":"develop.v.09"},{"index":10,"sense_key":"develop%2:30:00::","synset_id":"develop.v.10"},{"index":11,"sense_key":"develop%2:30:09::","synset_id":"modernize.v.02"},{"index":12,"sense_key":"develop%2:30:01::","synset_id":"develop.v.12"},{"index":13,"sense_key":"develop%2:36:09::","synset_id":"develop.v.13"},{"index":14,"sense_key":"develop%2:30:10::","synset_id":"develop.v.14"},{"index":15,"sense_key":"develop%2:39:00::","synset_id":"develop.v.15"},{"index":16,"sense_key":"develop%2:35:04::","synset_id":"develop.v.16"},{"index":17,"sense_key":"develop%2:33:03::","synset_id":"develop.v.17"},{"index":18,"sense_key":"develop%2:33:02::","synset_id":"develop.v.18"},{"index":19,"sense_key":"develop%2:30:13::","synset_id":"develop.v.19"},{"index":20,"sense_key":"develop%2:30:03::","synset_id":"break.v.53"},{"index":21,"sense_key":"develop%2:30:12::","synset_id":"develop.v.21"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"develop%2:36:01::","call_ids":["senseval2.d001.s004.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"develop%2:36:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1851,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004472,"input_uncached_usd":0.003702,"input_cached_usd":0.0,"output_usd":0.0007700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8270558749791235}
{"item_id":"senseval2.d001.s004.t004","gold_sense_keys":["base%2:31:00::"],"candidates":[{"index":1,"sense_key":"base%2:31:00::","synset_id":"establish.v.08"},{"index":2,"sense_key":"base%2:42:04::","synset_id":"base.v.02"},{"index":3,"sense_key":"base%2:34:00::","synset_id":"free-base.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"base%2:31:00::","call_ids":["senseval2.d001.s004.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"base%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":620,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00136,"input_uncached_usd":0.00124,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":6.633485084050335}
{"item_id":"senseval2.d001.s004.t006","gold_sense_keys":["identify%2:31:01::"],"candidates":[{"index":1,"sense_key":"identify%2:31:00::","synset_id":"identify.v.01"},{"index":2,"sense_key":"identify%2:32:00::","synset_id":"name.v.02"},{"index":3,"sense_key":"identify%2:31:04::","synset_id":"identify.v.03"},{"index":4,"sense_key":"identify%2:31:03::","synset_id":"identify.v.04"},{"index":5,"sense_key":"identify%2:31:01::","synset_id":"identify.v.05"},{"index":6,"sense_key":"identify%2:31:02::","synset_id":"identify.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"identify%2:31:01::","call_ids":["senseval2.d001.s004.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"identify%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":834,"cached_input_tokens":0,"output_tokens":113,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002798,"input_uncached_usd":0.001668,"input_cached_usd":0.0,"output_usd":0.0011300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.342307333019562}
{"item_id":"senseval2.d001.s004.t008","gold_sense_keys":["first%3:00:00::"],"candidates":[{"index":1,"sense_key":"first%3:00:00::","synset_id":"first.a.01"},{"index":2,"sense_key":"first%5:00:00:ordinal:00","synset_id":"first.s.01"},{"index":3,"sense_key":"first%5:00:02:opening:00","synset_id":"inaugural.s.01"},{"index":4,"sense_key":"first%5:00:01:opening:00","synset_id":"beginning.s.01"},{"index":5,"sense_key":"first%5:00:00:best:00","synset_id":"first.s.04"},{"index":6,"sense_key":"first%3:00:03::","synset_id":"first.a.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"first%3:00:00::","call_ids":["senseval2.d001.s004.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"first%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":833,"cached_input_tokens":0,"output_tokens":156,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003226,"input_uncached_usd":0.001666,"input_cached_usd":0.0,"output_usd":0.0015600000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.861553500057198}
{"item_id":"senseval2.d001.s004.t009","gold_sense_keys":["time%1:11:00::"],"candidates":[{"index":1,"sense_key":"time%1:11:00::","synset_id":"time.n.01"},{"index":2,"sense_key":"time%1:28:05::","synset_id":"time.n.02"},{"index":3,"sense_key":"time%1:28:00::","synset_id":"time.n.03"},{"index":4,"sense_key":"time%1:28:06::","synset_id":"time.n.04"},{"index":5,"sense_key":"time%1:03:00::","synset_id":"time.n.05"},{"index":6,"sense_key":"time%1:11:01::","synset_id":"time.n.06"},{"index":7,"sense_key":"time%1:28:03::","synset_id":"clock_time.n.01"},{"index":8,"sense_key":"time%1:28:04::","synset_id":"fourth_dimension.n.01"},{"index":9,"sense_key":"time%1:07:01::","synset_id":"meter.n.04"},{"index":10,"sense_key":"time%1:28:01::","synset_id":"prison_term.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"time%1:11:00::","call_ids":["senseval2.d001.s004.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"time%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1041,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002692,"input_uncached_usd":0.002082,"input_cached_usd":0.0,"output_usd":0.0006100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.464618374942802}
{"item_id":"senseval2.d001.s004.t011","gold_sense_keys":["otherwise%4:02:01::"],"candidates":[{"index":1,"sense_key":"otherwise%4:02:01::","synset_id":"otherwise.r.01"},{"index":2,"sense_key":"otherwise%4:02:03::","synset_id":"differently.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"otherwise%4:02:01::","call_ids":["senseval2.d001.s004.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"otherwise%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":591,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001302,"input_uncached_usd":0.0011819999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3934772500069812}
{"item_id":"senseval2.d001.s004.t012","gold_sense_keys":["healthy%3:00:00::"],"candidates":[{"index":1,"sense_key":"healthy%3:00:00::","synset_id":"healthy.a.01"},{"index":2,"sense_key":"healthy%5:00:00:sound:00","synset_id":"healthy.s.01"},{"index":3,"sense_key":"healthy%5:00:00:wholesome:00","synset_id":"healthy.s.02"},{"index":4,"sense_key":"healthy%5:00:00:reasonable:00","synset_id":"healthy.s.03"},{"index":5,"sense_key":"healthy%5:00:00:considerable:00","synset_id":"goodly.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"healthy%3:00:00::","call_ids":["senseval2.d001.s004.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"healthy%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":844,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001808,"input_uncached_usd":0.0016879999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1634549159789458}
{"item_id":"senseval2.d001.s004.t013","gold_sense_keys":["individual%1:03:00::"],"candidates":[{"index":1,"sense_key":"individual%1:03:00::","synset_id":"person.n.01"},{"index":2,"sense_key":"individual%1:18:00::","synset_id":"individual.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"individual%1:03:00::","call_ids":["senseval2.d001.s004.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"individual%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":557,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001234,"input_uncached_usd":0.001114,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4969065840123221}
{"item_id":"senseval2.d001.s004.t014","gold_sense_keys":["likely%3:00:00::"],"candidates":[{"index":1,"sense_key":"likely%3:00:00::","synset_id":"likely.a.01"},{"index":2,"sense_key":"likely%3:00:04::","synset_id":"probable.a.01"},{"index":3,"sense_key":"likely%5:00:00:prospective:00","synset_id":"likely.s.01"},{"index":4,"sense_key":"likely%5:00:00:credible:00","synset_id":"likely.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"likely%3:00:00::","call_ids":["senseval2.d001.s004.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"likely%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":695,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00216,"input_uncached_usd":0.00139,"input_cached_usd":0.0,"output_usd":0.0007700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5778402499854565}
{"item_id":"senseval2.d001.s004.t015","gold_sense_keys":["get%2:29:00::"],"candidates":[{"index":1,"sense_key":"get%2:40:00::","synset_id":"get.v.01"},{"index":2,"sense_key":"get%2:30:00::","synset_id":"become.v.01"},{"index":3,"sense_key":"get%2:30:02::","synset_id":"get.v.03"},{"index":4,"sense_key":"get%2:30:03::","synset_id":"receive.v.02"},{"index":5,"sense_key":"get%2:38:00::","synset_id":"arrive.v.01"},{"index":6,"sense_key":"get%2:35:03::","synset_id":"bring.v.04"},{"index":7,"sense_key":"get%2:39:00::","synset_id":"experience.v.03"},{"index":8,"sense_key":"get%2:33:00::","synset_id":"pay_back.v.02"},{"index":9,"sense_key":"get%2:40:01::","synset_id":"have.v.17"},{"index":10,"sense_key":"get%2:32:00::","synset_id":"induce.v.02"},{"index":11,"sense_key":"get%2:35:09::","synset_id":"get.v.11"},{"index":12,"sense_key":"get%2:29:01::","synset_id":"grow.v.08"},{"index":13,"sense_key":"get%2:29:00::","synset_id":"contract.v.04"},{"index":14,"sense_key":"get%2:32:03::","synset_id":"get.v.14"},{"index":15,"sense_key":"get%2:30:01::","synset_id":"make.v.02"},{"index":16,"sense_key":"get%2:32:08::","synset_id":"drive.v.11"},{"index":17,"sense_key":"get%2:31:09::","synset_id":"catch.v.18"},{"index":18,"sense_key":"get%2:35:00::","synset_id":"catch.v.07"},{"index":19,"sense_key":"get%2:35:10::","synset_id":"get.v.19"},{"index":20,"sense_key":"get%2:31:03::","synset_id":"get.v.20"},{"index":21,"sense_key":"get%2:40:14::","synset_id":"get.v.21"},{"index":22,"sense_key":"get%2:40:13::","synset_id":"get.v.22"},{"index":23,"sense_key":"get%2:39:13::","synset_id":"catch.v.21"},{"index":24,"sense_key":"get%2:39:09::","synset_id":"catch.v.22"},{"index":25,"sense_key":"get%2:39:14::","synset_id":"get.v.25"},{"index":26,"sense_key":"get%2:38:15::","synset_id":"scram.v.01"},{"index":27,"sense_key":"get%2:38:12::","synset_id":"get.v.27"},{"index":28,"sense_key":"get%2:37:12::","synset_id":"get.v.28"},{"index":29,"sense_key":"get%2:37:11::","synset_id":"get.v.29"},{"index":30,"sense_key":"get%2:36:14::","synset_id":"catch.v.24"},{"index":31,"sense_key":"get%2:36:00::","synset_id":"draw.v.15"},{"index":32,"sense_key":"get%2:35:12::","synset_id":"get.v.32"},{"index":33,"sense_key":"get%2:31:10::","synset_id":"perplex.v.01"},{"index":34,"sense_key":"get%2:30:12::","synset_id":"get_down.v.07"},{"index":35,"sense_key":"get%2:29:11::","synset_id":"suffer.v.02"},{"index":36,"sense_key":"get%2:29:08::","synset_id":"beget.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":13,"chosen_sense_key":"get%2:29:00::","call_ids":["senseval2.d001.s004.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":13,"predicted_sense_key":"get%2:29:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2813,"cached_input_tokens":0,"output_tokens":80,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.006426,"input_uncached_usd":0.005626,"input_cached_usd":0.0,"output_usd":0.0008,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3189937920542434}
{"item_id":"senseval2.d001.s004.t016","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s004.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":789,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001698,"input_uncached_usd":0.001578,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5678134170593694}
{"item_id":"senseval2.d001.s005.t000","gold_sense_keys":["set%1:14:00::"],"candidates":[{"index":1,"sense_key":"set%1:14:00::","synset_id":"set.n.01"},{"index":2,"sense_key":"set%1:14:02::","synset_id":"set.n.02"},{"index":3,"sense_key":"set%1:04:00::","synset_id":"set.n.03"},{"index":4,"sense_key":"set%1:06:00::","synset_id":"stage_set.n.01"},{"index":5,"sense_key":"set%1:14:01::","synset_id":"set.n.05"},{"index":6,"sense_key":"set%1:09:01::","synset_id":"bent.n.01"},{"index":7,"sense_key":"set%1:04:01::","synset_id":"set.n.07"},{"index":8,"sense_key":"set%1:28:00::","synset_id":"set.n.08"},{"index":9,"sense_key":"set%1:22:00::","synset_id":"hardening.n.02"},{"index":10,"sense_key":"set%1:18:00::","synset_id":"set.n.10"},{"index":11,"sense_key":"set%1:11:00::","synset_id":"set.n.11"},{"index":12,"sense_key":"set%1:09:00::","synset_id":"set.n.12"},{"index":13,"sense_key":"set%1:06:01::","synset_id":"set.n.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"set%1:14:00::","call_ids":["senseval2.d001.s005.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"set%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1318,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0027559999999999998,"input_uncached_usd":0.002636,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4792511250125244}
{"item_id":"senseval2.d001.s005.t001","gold_sense_keys":["discovery%1:04:00::","discovery%1:09:00::"],"candidates":[{"index":1,"sense_key":"discovery%1:04:00::","synset_id":"discovery.n.01"},{"index":2,"sense_key":"discovery%1:10:00::","synset_id":"discovery.n.02"},{"index":3,"sense_key":"discovery%1:09:00::","synset_id":"discovery.n.03"},{"index":4,"sense_key":"discovery%1:10:01::","synset_id":"discovery.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"discovery%1:10:00::","call_ids":["senseval2.d001.s005.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"discovery%1:10:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":692,"cached_input_tokens":0,"output_tokens":132,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0027040000000000002,"input_uncached_usd":0.001384,"input_cached_usd":0.0,"output_usd":0.0013200000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.99462362495251}
{"item_id":"senseval2.d001.s005.t002","gold_sense_keys":["say%2:32:00::","say%2:32:15::"],"candidates":[{"index":1,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":6,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":9,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":10,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":11,"sense_key":"say%2:32:09::","synset_id":"say.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"say%2:32:00::","call_ids":["senseval2.d001.s005.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"say%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1159,"cached_input_tokens":0,"output_tokens":187,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004188,"input_uncached_usd":0.0023179999999999997,"input_cached_usd":0.0,"output_usd":0.0018700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.235974165960215}
{"item_id":"senseval2.d001.s005.t005","gold_sense_keys":["find%2:31:09::","find%2:32:00::","find%2:36:00::","find%2:39:02::"],"candidates":[{"index":1,"sense_key":"find%2:40:02::","synset_id":"find.v.01"},{"index":2,"sense_key":"find%2:39:02::","synset_id":"detect.v.01"},{"index":3,"sense_key":"find%2:40:00::","synset_id":"find.v.03"},{"index":4,"sense_key":"find%2:32:00::","synset_id":"determine.v.01"},{"index":5,"sense_key":"find%2:31:10::","synset_id":"find.v.05"},{"index":6,"sense_key":"find%2:39:01::","synset_id":"witness.v.02"},{"index":7,"sense_key":"find%2:40:01::","synset_id":"line_up.v.02"},{"index":8,"sense_key":"find%2:36:00::","synset_id":"discover.v.03"},{"index":9,"sense_key":"find%2:31:09::","synset_id":"discover.v.04"},{"index":10,"sense_key":"find%2:40:03::","synset_id":"find.v.10"},{"index":11,"sense_key":"find%2:32:01::","synset_id":"rule.v.04"},{"index":12,"sense_key":"find%2:30:00::","synset_id":"receive.v.02"},{"index":13,"sense_key":"find%2:39:05::","synset_id":"find.v.13"},{"index":14,"sense_key":"find%2:40:15::","synset_id":"recover.v.01"},{"index":15,"sense_key":"find%2:38:10::","synset_id":"find.v.15"},{"index":16,"sense_key":"find%2:30:13::","synset_id":"find_oneself.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"find%2:36:00::","call_ids":["senseval2.d001.s005.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"find%2:36:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1651,"cached_input_tokens":0,"output_tokens":216,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.005462,"input_uncached_usd":0.003302,"input_cached_usd":0.0,"output_usd":0.00216,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.868233707966283}
{"item_id":"senseval2.d001.s006.t000","gold_sense_keys":["only%4:02:05::"],"candidates":[{"index":1,"sense_key":"only%4:02:02::","synset_id":"merely.r.01"},{"index":2,"sense_key":"only%4:02:01::","synset_id":"entirely.r.02"},{"index":3,"sense_key":"only%4:02:00::","synset_id":"only.r.03"},{"index":4,"sense_key":"only%4:02:04::","synset_id":"only.r.04"},{"index":5,"sense_key":"only%4:02:06::","synset_id":"only.r.05"},{"index":6,"sense_key":"only%4:02:03::","synset_id":"only.r.06"},{"index":7,"sense_key":"only%4:02:05::","synset_id":"only.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"only%4:02:05::","call_ids":["senseval2.d001.s006.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"only%4:02:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":862,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002294,"input_uncached_usd":0.0017239999999999998,"input_cached_usd":0.0,"output_usd":0.0005700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3165647499263287}
{"item_id":"senseval2.d001.s006.t001","gold_sense_keys":["decade%1:28:00::"],"candidates":[{"index":1,"sense_key":"decade%1:28:00::","synset_id":"decade.n.01"},{"index":2,"sense_key":"decade%1:23:00::","synset_id":"ten.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"decade%1:28:00::","call_ids":["senseval2.d001.s006.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"decade%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":602,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001324,"input_uncached_usd":0.001204,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.996689916937612}
{"item_id":"senseval2.d001.s006.t003","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s006.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":813,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001746,"input_uncached_usd":0.0016259999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.077107375022024}
{"item_id":"senseval2.d001.s006.t004","gold_sense_keys":["know%2:31:01::"],"candidates":[{"index":1,"sense_key":"know%2:31:01::","synset_id":"know.v.01"},{"index":2,"sense_key":"know%2:31:03::","synset_id":"know.v.02"},{"index":3,"sense_key":"know%2:31:02::","synset_id":"know.v.03"},{"index":4,"sense_key":"know%2:31:00::","synset_id":"know.v.04"},{"index":5,"sense_key":"know%2:31:04::","synset_id":"know.v.05"},{"index":6,"sense_key":"know%2:31:06::","synset_id":"acknowledge.v.06"},{"index":7,"sense_key":"know%2:31:15::","synset_id":"know.v.07"},{"index":8,"sense_key":"know%2:35:00::","synset_id":"sleep_together.v.01"},{"index":9,"sense_key":"know%2:31:11::","synset_id":"know.v.09"},{"index":10,"sense_key":"know%2:31:12::","synset_id":"know.v.10"},{"index":11,"sense_key":"know%2:31:14::","synset_id":"know.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"know%2:31:01::","call_ids":["senseval2.d001.s006.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"know%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1217,"cached_input_tokens":0,"output_tokens":117,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003604,"input_uncached_usd":0.002434,"input_cached_usd":0.0,"output_usd":0.00117,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.591691083041951}
{"item_id":"senseval2.d001.s006.t006","gold_sense_keys":["molecular%3:01:00::"],"candidates":[{"index":1,"sense_key":"molecular%3:01:00::","synset_id":"molecular.a.01"},{"index":2,"sense_key":"molecular%3:00:00::","synset_id":"molecular.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"molecular%3:01:00::","call_ids":["senseval2.d001.s006.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"molecular%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":623,"cached_input_tokens":0,"output_tokens":99,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002236,"input_uncached_usd":0.001246,"input_cached_usd":0.0,"output_usd":0.00099,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7655461661051959}
{"item_id":"senseval2.d001.s006.t007","gold_sense_keys":["level%1:09:00::"],"candidates":[{"index":1,"sense_key":"level%1:07:00::","synset_id":"degree.n.01"},{"index":2,"sense_key":"level%1:26:01::","synset_id":"grade.n.02"},{"index":3,"sense_key":"level%1:26:00::","synset_id":"degree.n.02"},{"index":4,"sense_key":"level%1:07:01::","synset_id":"level.n.04"},{"index":5,"sense_key":"level%1:06:00::","synset_id":"level.n.05"},{"index":6,"sense_key":"level%1:06:03::","synset_id":"horizontal_surface.n.01"},{"index":7,"sense_key":"level%1:09:00::","synset_id":"level.n.07"},{"index":8,"sense_key":"level%1:06:01::","synset_id":"floor.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"level%1:09:00::","call_ids":["senseval2.d001.s006.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"level%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1017,"cached_input_tokens":0,"output_tokens":134,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003374,"input_uncached_usd":0.002034,"input_cached_usd":0.0,"output_usd":0.00134,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6469922920223325}
{"item_id":"senseval2.d001.s007.t000","gold_sense_keys":["today%1:28:00::"],"candidates":[{"index":1,"sense_key":"today%1:28:00::","synset_id":"today.n.01"},{"index":2,"sense_key":"today%1:28:01::","synset_id":"today.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"today%1:28:00::","call_ids":["senseval2.d001.s007.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"today%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":601,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001752,"input_uncached_usd":0.001202,"input_cached_usd":0.0,"output_usd":0.00055,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3201928329654038}
{"item_id":"senseval2.d001.s007.t001","gold_sense_keys":["know%2:31:01::","know%2:31:02::"],"candidates":[{"index":1,"sense_key":"know%2:31:01::","synset_id":"know.v.01"},{"index":2,"sense_key":"know%2:31:03::","synset_id":"know.v.02"},{"index":3,"sense_key":"know%2:31:02::","synset_id":"know.v.03"},{"index":4,"sense_key":"know%2:31:00::","synset_id":"know.v.04"},{"index":5,"sense_key":"know%2:31:04::","synset_id":"know.v.05"},{"index":6,"sense_key":"know%2:31:06::","synset_id":"acknowledge.v.06"},{"index":7,"sense_key":"know%2:31:15::","synset_id":"know.v.07"},{"index":8,"sense_key":"know%2:35:00::","synset_id":"sleep_together.v.01"},{"index":9,"sense_key":"know%2:31:11::","synset_id":"know.v.09"},{"index":10,"sense_key":"know%2:31:12::","synset_id":"know.v.10"},{"index":11,"sense_key":"know%2:31:14::","synset_id":"know.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"know%2:31:01::","call_ids":["senseval2.d001.s007.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"know%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1213,"cached_input_tokens":0,"output_tokens":142,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0038459999999999996,"input_uncached_usd":0.0024259999999999998,"input_cached_usd":0.0,"output_usd":0.00142,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8729599999496713}
{"item_id":"senseval2.d001.s007.t002","gold_sense_keys":["accumulation%1:22:00::"],"candidates":[{"index":1,"sense_key":"accumulation%1:22:00::","synset_id":"accretion.n.01"},{"index":2,"sense_key":"accumulation%1:14:00::","synset_id":"collection.n.01"},{"index":3,"sense_key":"accumulation%1:04:00::","synset_id":"accumulation.n.03"},{"index":4,"sense_key":"accumulation%1:21:00::","synset_id":"accumulation.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"accumulation%1:22:00::","call_ids":["senseval2.d001.s007.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"accumulation%1:22:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":693,"cached_input_tokens":0,"output_tokens":146,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002846,"input_uncached_usd":0.0013859999999999999,"input_cached_usd":0.0,"output_usd":0.0014600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.3979712910950184}
{"item_id":"senseval2.d001.s007.t003","gold_sense_keys":["altered%3:00:00::"],"candidates":[{"index":1,"sense_key":"altered%3:00:00::","synset_id":"altered.a.01"},{"index":2,"sense_key":"altered%5:00:00:castrated:00","synset_id":"altered.s.01"},{"index":3,"sense_key":"altered%5:00:00:modified:00","synset_id":"adapted.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"altered%3:00:00::","call_ids":["senseval2.d001.s007.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"altered%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":703,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001526,"input_uncached_usd":0.001406,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0131974159739912}
{"item_id":"senseval2.d001.s007.t005","gold_sense_keys":["initiate%2:36:01::"],"candidates":[{"index":1,"sense_key":"initiate%2:36:01::","synset_id":"originate.v.02"},{"index":2,"sense_key":"initiate%2:36:00::","synset_id":"initiate.v.02"},{"index":3,"sense_key":"initiate%2:41:01::","synset_id":"initiate.v.03"},{"index":4,"sense_key":"initiate%2:32:00::","synset_id":"broach.v.01"},{"index":5,"sense_key":"initiate%2:30:00::","synset_id":"lead_up.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"initiate%2:36:01::","call_ids":["senseval2.d001.s007.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"initiate%2:36:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":795,"cached_input_tokens":0,"output_tokens":137,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00296,"input_uncached_usd":0.0015899999999999998,"input_cached_usd":0.0,"output_usd":0.0013700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6950150830671191}
{"item_id":"senseval2.d001.s007.t006","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s007.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":810,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00174,"input_uncached_usd":0.00162,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0386629580752924}
{"item_id":"senseval2.d001.s007.t007","gold_sense_keys":["then%4:02:00::","then%4:02:02::"],"candidates":[{"index":1,"sense_key":"then%4:02:00::","synset_id":"then.r.01"},{"index":2,"sense_key":"then%4:02:02::","synset_id":"then.r.02"},{"index":3,"sense_key":"then%4:02:01::","synset_id":"then.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"then%4:02:00::","call_ids":["senseval2.d001.s007.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"then%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":655,"cached_input_tokens":0,"output_tokens":70,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00201,"input_uncached_usd":0.00131,"input_cached_usd":0.0,"output_usd":0.0007000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6031095000216737}
{"item_id":"senseval2.d001.s007.t008","gold_sense_keys":["propel%2:35:00::"],"candidates":[{"index":1,"sense_key":"propel%2:35:00::","synset_id":"propel.v.01"},{"index":2,"sense_key":"propel%2:36:00::","synset_id":"motivate.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"propel%2:35:00::","call_ids":["senseval2.d001.s007.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"propel%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":610,"cached_input_tokens":0,"output_tokens":118,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0024000000000000002,"input_uncached_usd":0.00122,"input_cached_usd":0.0,"output_usd":0.00118,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.75127716595307}
{"item_id":"senseval2.d001.s007.t009","gold_sense_keys":["deadly%5:00:01:fatal:00"],"candidates":[{"index":1,"sense_key":"deadly%5:00:01:fatal:00","synset_id":"deadly.s.01"},{"index":2,"sense_key":"deadly%5:00:02:fatal:00","synset_id":"deadly.s.02"},{"index":3,"sense_key":"deadly%5:00:00:toxic:00","synset_id":"deadly.s.03"},{"index":4,"sense_key":"deadly%5:00:00:unpardonable:00","synset_id":"deadly.s.04"},{"index":5,"sense_key":"deadly%5:00:00:noxious:00","synset_id":"baneful.s.01"},{"index":6,"sense_key":"deadly%5:00:00:virulent:00","synset_id":"deadly.s.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"deadly%5:00:01:fatal:00","call_ids":["senseval2.d001.s007.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"deadly%5:00:01:fatal:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":868,"cached_input_tokens":0,"output_tokens":119,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0029259999999999998,"input_uncached_usd":0.001736,"input_cached_usd":0.0,"output_usd":0.00119,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.991445375024341}
{"item_id":"senseval2.d001.s007.t010","gold_sense_keys":["state%1:03:00::"],"candidates":[{"index":1,"sense_key":"state%1:15:01::","synset_id":"state.n.01"},{"index":2,"sense_key":"state%1:03:00::","synset_id":"state.n.02"},{"index":3,"sense_key":"state%1:14:01::","synset_id":"state.n.03"},{"index":4,"sense_key":"state%1:14:00::","synset_id":"state.n.04"},{"index":5,"sense_key":"state%1:26:02::","synset_id":"state_of_matter.n.01"},{"index":6,"sense_key":"state%1:26:01::","synset_id":"state.n.06"},{"index":7,"sense_key":"state%1:15:00::","synset_id":"country.n.02"},{"index":8,"sense_key":"state%1:14:02::","synset_id":"department_of_state.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"state%1:03:00::","call_ids":["senseval2.d001.s007.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"state%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1023,"cached_input_tokens":0,"output_tokens":89,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0029360000000000002,"input_uncached_usd":0.002046,"input_cached_usd":0.0,"output_usd":0.0008900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0160721249412745}
{"item_id":"senseval2.d001.s008.t001","gold_sense_keys":["call%2:32:00::"],"candidates":[{"index":1,"sense_key":"call%2:32:02::","synset_id":"name.v.01"},{"index":2,"sense_key":"call%2:32:00::","synset_id":"call.v.02"},{"index":3,"sense_key":"call%2:32:01::","synset_id":"call.v.03"},{"index":4,"sense_key":"call%2:32:09::","synset_id":"shout.v.02"},{"index":5,"sense_key":"call%2:32:05::","synset_id":"call.v.05"},{"index":6,"sense_key":"call%2:41:00::","synset_id":"visit.v.03"},{"index":7,"sense_key":"call%2:41:04::","synset_id":"call.v.07"},{"index":8,"sense_key":"call%2:32:13::","synset_id":"call.v.08"},{"index":9,"sense_key":"call%2:32:04::","synset_id":"call.v.09"},{"index":10,"sense_key":"call%2:32:03::","synset_id":"call.v.10"},{"index":11,"sense_key":"call%2:42:09::","synset_id":"call.v.11"},{"index":12,"sense_key":"call%2:41:14::","synset_id":"address.v.06"},{"index":13,"sense_key":"call%2:41:12::","synset_id":"call.v.13"},{"index":14,"sense_key":"call%2:40:09::","synset_id":"call.v.14"},{"index":15,"sense_key":"call%2:40:05::","synset_id":"bid.v.04"},{"index":16,"sense_key":"call%2:38:00::","synset_id":"call.v.16"},{"index":17,"sense_key":"call%2:32:06::","synset_id":"call.v.17"},{"index":18,"sense_key":"call%2:32:10::","synset_id":"predict.v.01"},{"index":19,"sense_key":"call%2:32:12::","synset_id":"call.v.19"},{"index":20,"sense_key":"call%2:32:07::","synset_id":"call.v.20"},{"index":21,"sense_key":"call%2:32:08::","synset_id":"call.v.21"},{"index":22,"sense_key":"call%2:32:11::","synset_id":"call.v.22"},{"index":23,"sense_key":"call%2:32:15::","synset_id":"call.v.23"},{"index":24,"sense_key":"call%2:32:14::","synset_id":"call.v.24"},{"index":25,"sense_key":"call%2:31:13::","synset_id":"call.v.25"},{"index":26,"sense_key":"call%2:31:00::","synset_id":"call.v.26"},{"index":27,"sense_key":"call%2:31:05::","synset_id":"call.v.27"},{"index":28,"sense_key":"call%2:29:05::","synset_id":"call.v.28"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"call%2:32:02::","call_ids":["senseval2.d001.s008.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"call%2:32:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":2101,"cached_input_tokens":0,"output_tokens":147,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.005672,"input_uncached_usd":0.004202,"input_cached_usd":0.0,"output_usd":0.0014700000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7820368750253692}
{"item_id":"senseval2.d001.s008.t002","gold_sense_keys":["new%3:00:00::","new%5:00:00:original:00","new%5:00:00:other:00","new%5:00:00:unprecedented:00"],"candidates":[{"index":1,"sense_key":"new%3:00:00::","synset_id":"new.a.01"},{"index":2,"sense_key":"new%5:00:00:original:00","synset_id":"fresh.s.03"},{"index":3,"sense_key":"new%5:00:00:inexperienced:00","synset_id":"raw.s.11"},{"index":4,"sense_key":"new%5:00:00:unprecedented:00","synset_id":"new.s.03"},{"index":5,"sense_key":"new%5:00:00:other:00","synset_id":"new.s.04"},{"index":6,"sense_key":"new%3:00:09::","synset_id":"new.a.06"},{"index":7,"sense_key":"new%5:00:02:original:00","synset_id":"newfangled.s.01"},{"index":8,"sense_key":"new%5:00:02:late:01","synset_id":"new.s.06"},{"index":9,"sense_key":"new%5:00:01:late:01","synset_id":"modern.s.04"},{"index":10,"sense_key":"new%5:00:00:early:02","synset_id":"new.s.08"},{"index":11,"sense_key":"new%5:00:00:unaccustomed:00","synset_id":"new.s.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"new%3:00:00::","call_ids":["senseval2.d001.s008.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"new%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1169,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0029679999999999997,"input_uncached_usd":0.002338,"input_cached_usd":0.0,"output_usd":0.00063,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.265957250026986}
{"item_id":"senseval2.d001.s008.t003","gold_sense_keys":["class%1:14:00::"],"candidates":[{"index":1,"sense_key":"class%1:14:00::","synset_id":"class.n.01"},{"index":2,"sense_key":"class%1:14:03::","synset_id":"class.n.02"},{"index":3,"sense_key":"class%1:14:01::","synset_id":"class.n.03"},{"index":4,"sense_key":"class%1:04:00::","synset_id":"course.n.01"},{"index":5,"sense_key":"class%1:14:05::","synset_id":"class.n.05"},{"index":6,"sense_key":"class%1:14:04::","synset_id":"class.n.06"},{"index":7,"sense_key":"class%1:14:02::","synset_id":"class.n.07"},{"index":8,"sense_key":"class%1:07:00::","synset_id":"class.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"class%1:14:00::","call_ids":["senseval2.d001.s008.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"class%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":950,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00202,"input_uncached_usd":0.0019,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1238103329669684}
{"item_id":"senseval2.d001.s008.t005","gold_sense_keys":["simply%4:02:01::"],"candidates":[{"index":1,"sense_key":"simply%4:02:00::","synset_id":"merely.r.01"},{"index":2,"sense_key":"simply%4:02:03::","synset_id":"just.r.04"},{"index":3,"sense_key":"simply%4:02:02::","synset_id":"simply.r.03"},{"index":4,"sense_key":"simply%4:02:01::","synset_id":"plainly.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"simply%4:02:00::","call_ids":["senseval2.d001.s008.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"simply%4:02:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":707,"cached_input_tokens":0,"output_tokens":76,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0021739999999999997,"input_uncached_usd":0.001414,"input_cached_usd":0.0,"output_usd":0.00076,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3420238749822602}
{"item_id":"senseval2.d001.s009.t000","gold_sense_keys":["function%2:35:00::"],"candidates":[{"index":1,"sense_key":"function%2:35:00::","synset_id":"function.v.01"},{"index":2,"sense_key":"function%2:42:12::","synset_id":"serve.v.01"},{"index":3,"sense_key":"function%2:33:00::","synset_id":"officiate.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"function%2:35:00::","call_ids":["senseval2.d001.s009.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"function%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":737,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002054,"input_uncached_usd":0.0014739999999999998,"input_cached_usd":0.0,"output_usd":0.00058,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5154559159418568}
{"item_id":"senseval2.d001.s009.t004","gold_sense_keys":["hold%2:41:15::"],"candidates":[{"index":1,"sense_key":"hold%2:42:00::","synset_id":"keep.v.01"},{"index":2,"sense_key":"hold%2:35:00::","synset_id":"hold.v.02"},{"index":3,"sense_key":"hold%2:36:00::","synset_id":"hold.v.03"},{"index":4,"sense_key":"hold%2:40:00::","synset_id":"have.v.01"},{"index":5,"sense_key":"hold%2:31:01::","synset_id":"deem.v.01"},{"index":6,"sense_key":"hold%2:37:00::","synset_id":"harbor.v.01"},{"index":7,"sense_key":"hold%2:35:03::","synset_id":"restrain.v.03"},{"index":8,"sense_key":"hold%2:40:02::","synset_id":"retain.v.03"},{"index":9,"sense_key":"hold%2:40:04::","synset_id":"bear.v.11"},{"index":10,"sense_key":"hold%2:35:01::","synset_id":"hold.v.10"},{"index":11,"sense_key":"hold%2:42:13::","synset_id":"hold.v.11"},{"index":12,"sense_key":"hold%2:42:05::","synset_id":"accommodate.v.04"},{"index":13,"sense_key":"hold%2:42:01::","synset_id":"hold.v.13"},{"index":14,"sense_key":"hold%2:35:02::","synset_id":"hold.v.14"},{"index":15,"sense_key":"hold%2:42:02::","synset_id":"prevail.v.02"},{"index":16,"sense_key":"hold%2:31:08::","synset_id":"hold.v.16"},{"index":17,"sense_key":"hold%2:42:07::","synset_id":"hold.v.17"},{"index":18,"sense_key":"hold%2:42:14::","synset_id":"contain.v.05"},{"index":19,"sense_key":"hold%2:41:00::","synset_id":"reserve.v.04"},{"index":20,"sense_key":"hold%2:33:05::","synset_id":"defend.v.03"},{"index":21,"sense_key":"hold%2:32:13::","synset_id":"oblige.v.02"},{"index":22,"sense_key":"hold%2:31:04::","synset_id":"hold.v.22"},{"index":23,"sense_key":"hold%2:31:10::","synset_id":"hold.v.23"},{"index":24,"sense_key":"hold%2:42:08::","synset_id":"defy.v.01"},{"index":25,"sense_key":"hold%2:42:04::","synset_id":"apply.v.02"},{"index":26,"sense_key":"hold%2:42:12::","synset_id":"hold.v.26"},{"index":27,"sense_key":"hold%2:41:15::","synset_id":"control.v.02"},{"index":28,"sense_key":"hold%2:41:13::","synset_id":"hold.v.28"},{"index":29,"sense_key":"hold%2:41:14::","synset_id":"hold.v.29"},{"index":30,"sense_key":"hold%2:38:05::","synset_id":"halt.v.01"},{"index":31,"sense_key":"hold%2:35:09::","synset_id":"hold.v.31"},{"index":32,"sense_key":"hold%2:34:00::","synset_id":"carry.v.33"},{"index":33,"sense_key":"hold%2:33:00::","synset_id":"hold.v.33"},{"index":34,"sense_key":"hold%2:32:11::","synset_id":"declare.v.04"},{"index":35,"sense_key":"hold%2:32:15::","synset_id":"agree.v.01"},{"index":36,"sense_key":"hold%2:29:04::","synset_id":"hold.v.36"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":27,"chosen_sense_key":"hold%2:41:15::","call_ids":["senseval2.d001.s009.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":27,"predicted_sense_key":"hold%2:41:15::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2881,"cached_input_tokens":0,"output_tokens":78,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.006542,"input_uncached_usd":0.005762,"input_cached_usd":0.0,"output_usd":0.0007800000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.127243874943815}
{"item_id":"senseval2.d001.s009.t005","gold_sense_keys":["cell%1:03:00::"],"candidates":[{"index":1,"sense_key":"cell%1:06:03::","synset_id":"cell.n.01"},{"index":2,"sense_key":"cell%1:03:00::","synset_id":"cell.n.02"},{"index":3,"sense_key":"cell%1:06:00::","synset_id":"cell.n.03"},{"index":4,"sense_key":"cell%1:14:00::","synset_id":"cell.n.04"},{"index":5,"sense_key":"cell%1:06:04::","synset_id":"cellular_telephone.n.01"},{"index":6,"sense_key":"cell%1:06:02::","synset_id":"cell.n.06"},{"index":7,"sense_key":"cell%1:06:01::","synset_id":"cell.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cell%1:03:00::","call_ids":["senseval2.d001.s009.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cell%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":950,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00202,"input_uncached_usd":0.0019,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4615677919937298}
{"item_id":"senseval2.d001.s009.t006","gold_sense_keys":["growth%1:22:02::"],"candidates":[{"index":1,"sense_key":"growth%1:22:00::","synset_id":"growth.n.01"},{"index":2,"sense_key":"growth%1:22:01::","synset_id":"growth.n.02"},{"index":3,"sense_key":"growth%1:22:02::","synset_id":"increase.n.03"},{"index":4,"sense_key":"growth%1:14:00::","synset_id":"growth.n.04"},{"index":5,"sense_key":"growth%1:11:01::","synset_id":"emergence.n.01"},{"index":6,"sense_key":"growth%1:26:00::","synset_id":"growth.n.06"},{"index":7,"sense_key":"growth%1:17:00::","synset_id":"growth.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"growth%1:22:02::","call_ids":["senseval2.d001.s009.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"growth%1:22:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":980,"cached_input_tokens":0,"output_tokens":141,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0033699999999999997,"input_uncached_usd":0.00196,"input_cached_usd":0.0,"output_usd":0.00141,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0487414160743356}
{"item_id":"senseval2.d001.s010.t001","gold_sense_keys":["damage%2:30:00::"],"candidates":[{"index":1,"sense_key":"damage%2:30:00::","synset_id":"damage.v.01"},{"index":2,"sense_key":"damage%2:30:01::","synset_id":"damage.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"damage%2:30:00::","call_ids":["senseval2.d001.s010.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"damage%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":626,"cached_input_tokens":0,"output_tokens":128,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002532,"input_uncached_usd":0.0012519999999999999,"input_cached_usd":0.0,"output_usd":0.00128,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.4528040420264006}
{"item_id":"senseval2.d001.s010.t003","gold_sense_keys":["radiation%1:19:00::"],"candidates":[{"index":1,"sense_key":"radiation%1:19:00::","synset_id":"radiation.n.01"},{"index":2,"sense_key":"radiation%1:04:00::","synset_id":"radiation.n.02"},{"index":3,"sense_key":"radiation%1:26:00::","synset_id":"radiation_sickness.n.01"},{"index":4,"sense_key":"radiation%1:22:00::","synset_id":"radiation.n.04"},{"index":5,"sense_key":"radiation%1:11:00::","synset_id":"radiation.n.05"},{"index":6,"sense_key":"radiation%1:08:00::","synset_id":"radiation.n.06"},{"index":7,"sense_key":"radiation%1:04:01::","synset_id":"radiotherapy.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"radiation%1:19:00::","call_ids":["senseval2.d001.s010.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"radiation%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1014,"cached_input_tokens":0,"output_tokens":76,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0027879999999999997,"input_uncached_usd":0.002028,"input_cached_usd":0.0,"output_usd":0.00076,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.418311332934536}
{"item_id":"senseval2.d001.s010.t011","gold_sense_keys":["work%2:35:02::","work%2:41:03::"],"candidates":[{"index":1,"sense_key":"work%2:41:02::","synset_id":"work.v.01"},{"index":2,"sense_key":"work%2:41:00::","synset_id":"work.v.02"},{"index":3,"sense_key":"work%2:41:03::","synset_id":"work.v.03"},{"index":4,"sense_key":"work%2:35:02::","synset_id":"function.v.01"},{"index":5,"sense_key":"work%2:36:00::","synset_id":"work.v.05"},{"index":6,"sense_key":"work%2:29:00::","synset_id":"exercise.v.03"},{"index":7,"sense_key":"work%2:38:00::","synset_id":"make.v.36"},{"index":8,"sense_key":"work%2:41:04::","synset_id":"work.v.08"},{"index":9,"sense_key":"work%2:41:05::","synset_id":"work.v.09"},{"index":10,"sense_key":"work%2:38:02::","synset_id":"work.v.10"},{"index":11,"sense_key":"work%2:36:01::","synset_id":"bring.v.03"},{"index":12,"sense_key":"work%2:41:01::","synset_id":"work.v.12"},{"index":13,"sense_key":"work%2:36:09::","synset_id":"cultivate.v.02"},{"index":14,"sense_key":"work%2:42:10::","synset_id":"work.v.14"},{"index":15,"sense_key":"work%2:41:10::","synset_id":"influence.v.01"},{"index":16,"sense_key":"work%2:41:11::","synset_id":"work.v.16"},{"index":17,"sense_key":"work%2:41:12::","synset_id":"work.v.17"},{"index":18,"sense_key":"work%2:39:10::","synset_id":"work.v.18"},{"index":19,"sense_key":"work%2:37:10::","synset_id":"work.v.19"},{"index":20,"sense_key":"work%2:36:12::","synset_id":"shape.v.02"},{"index":21,"sense_key":"work%2:35:14::","synset_id":"work.v.21"},{"index":22,"sense_key":"work%2:35:00::","synset_id":"knead.v.01"},{"index":23,"sense_key":"work%2:34:10::","synset_id":"exploit.v.01"},{"index":24,"sense_key":"work%2:31:13::","synset_id":"solve.v.01"},{"index":25,"sense_key":"work%2:30:14::","synset_id":"ferment.v.03"},{"index":26,"sense_key":"work%2:30:13::","synset_id":"sour.v.01"},{"index":27,"sense_key":"work%2:30:10::","synset_id":"work.v.27"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"work%2:35:02::","call_ids":["senseval2.d001.s010.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"work%2:35:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2159,"cached_input_tokens":0,"output_tokens":105,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0053679999999999995,"input_uncached_usd":0.004318,"input_cached_usd":0.0,"output_usd":0.0010500000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1205842499621212}
{"item_id":"senseval2.d001.s010.t012","gold_sense_keys":["cell%1:03:00::"],"candidates":[{"index":1,"sense_key":"cell%1:06:03::","synset_id":"cell.n.01"},{"index":2,"sense_key":"cell%1:03:00::","synset_id":"cell.n.02"},{"index":3,"sense_key":"cell%1:06:00::","synset_id":"cell.n.03"},{"index":4,"sense_key":"cell%1:14:00::","synset_id":"cell.n.04"},{"index":5,"sense_key":"cell%1:06:04::","synset_id":"cellular_telephone.n.01"},{"index":6,"sense_key":"cell%1:06:02::","synset_id":"cell.n.06"},{"index":7,"sense_key":"cell%1:06:01::","synset_id":"cell.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cell%1:03:00::","call_ids":["senseval2.d001.s010.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cell%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":930,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00198,"input_uncached_usd":0.0018599999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":4.506314416998066}
{"item_id":"senseval2.d001.s010.t014","gold_sense_keys":["turn%2:30:03::","turn%2:42:00::"],"candidates":[{"index":1,"sense_key":"turn%2:38:00::","synset_id":"turn.v.01"},{"index":2,"sense_key":"turn%2:30:04::","synset_id":"change_state.v.01"},{"index":3,"sense_key":"turn%2:42:00::","synset_id":"become.v.02"},{"index":4,"sense_key":"turn%2:38:02::","synset_id":"turn.v.04"},{"index":5,"sense_key":"turn%2:30:00::","synset_id":"change_by_reversal.v.01"},{"index":6,"sense_key":"turn%2:38:03::","synset_id":"turn.v.06"},{"index":7,"sense_key":"turn%2:30:03::","synset_id":"turn.v.07"},{"index":8,"sense_key":"turn%2:35:08::","synset_id":"turn.v.08"},{"index":9,"sense_key":"turn%2:38:04::","synset_id":"turn.v.09"},{"index":10,"sense_key":"turn%2:38:01::","synset_id":"turn.v.10"},{"index":11,"sense_key":"turn%2:38:06::","synset_id":"turn.v.11"},{"index":12,"sense_key":"turn%2:36:00::","synset_id":"plow.v.01"},{"index":13,"sense_key":"turn%2:35:06::","synset_id":"turn.v.13"},{"index":14,"sense_key":"turn%2:30:09::","synset_id":"turn.v.14"},{"index":15,"sense_key":"turn%2:29:00::","synset_id":"twist.v.10"},{"index":16,"sense_key":"turn%2:42:13::","synset_id":"turn.v.16"},{"index":17,"sense_key":"turn%2:41:10::","synset_id":"turn.v.17"},{"index":18,"sense_key":"turn%2:40:13::","synset_id":"turn.v.18"},{"index":19,"sense_key":"turn%2:38:13::","synset_id":"turn.v.19"},{"index":20,"sense_key":"turn%2:38:14::","synset_id":"turn.v.20"},{"index":21,"sense_key":"turn%2:35:11::","synset_id":"flex.v.05"},{"index":22,"sense_key":"turn%2:35:10::","synset_id":"turn.v.22"},{"index":23,"sense_key":"turn%2:33:13::","synset_id":"turn.v.23"},{"index":24,"sense_key":"turn%2:32:14::","synset_id":"call_on.v.01"},{"index":25,"sense_key":"turn%2:30:01::","synset_id":"sour.v.01"},{"index":26,"sense_key":"turn%2:30:14::","synset_id":"turn.v.26"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"turn%2:30:03::","call_ids":["senseval2.d001.s010.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"turn%2:30:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2082,"cached_input_tokens":0,"output_tokens":139,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.005554,"input_uncached_usd":0.004164,"input_cached_usd":0.0,"output_usd":0.0013900000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0682729999534786}
{"item_id":"senseval2.d001.s011.t001","gold_sense_keys":["identify%2:31:01::"],"candidates":[{"index":1,"sense_key":"identify%2:31:00::","synset_id":"identify.v.01"},{"index":2,"sense_key":"identify%2:32:00::","synset_id":"name.v.02"},{"index":3,"sense_key":"identify%2:31:04::","synset_id":"identify.v.03"},{"index":4,"sense_key":"identify%2:31:03::","synset_id":"identify.v.04"},{"index":5,"sense_key":"identify%2:31:01::","synset_id":"identify.v.05"},{"index":6,"sense_key":"identify%2:31:02::","synset_id":"identify.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"identify%2:31:01::","call_ids":["senseval2.d001.s011.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"identify%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":881,"cached_input_tokens":0,"output_tokens":149,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003252,"input_uncached_usd":0.001762,"input_cached_usd":0.0,"output_usd":0.0014900000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1305433340603486}
{"item_id":"senseval2.d001.s011.t003","gold_sense_keys":["differ%2:42:00::"],"candidates":[{"index":1,"sense_key":"differ%2:42:00::","synset_id":"differ.v.01"},{"index":2,"sense_key":"differ%2:32:00::","synset_id":"disagree.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"differ%2:42:00::","call_ids":["senseval2.d001.s011.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"differ%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":628,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001376,"input_uncached_usd":0.001256,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.517741332994774}
{"item_id":"senseval2.d001.s011.t004","gold_sense_keys":["family%1:14:04::"],"candidates":[{"index":1,"sense_key":"family%1:14:02::","synset_id":"family.n.01"},{"index":2,"sense_key":"family%1:14:00::","synset_id":"family.n.02"},{"index":3,"sense_key":"family%1:14:04::","synset_id":"class.n.01"},{"index":4,"sense_key":"family%1:14:01::","synset_id":"family.n.04"},{"index":5,"sense_key":"family%1:18:00::","synset_id":"kin.n.01"},{"index":6,"sense_key":"family%1:14:03::","synset_id":"family.n.06"},{"index":7,"sense_key":"family%1:14:06::","synset_id":"syndicate.n.01"},{"index":8,"sense_key":"family%1:14:05::","synset_id":"family.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"family%1:14:04::","call_ids":["senseval2.d001.s011.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"family%1:14:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1036,"cached_input_tokens":0,"output_tokens":115,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003222,"input_uncached_usd":0.002072,"input_cached_usd":0.0,"output_usd":0.0011500000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.160116083920002}
{"item_id":"senseval2.d001.s011.t006","gold_sense_keys":["discover%2:31:00::","discover%2:31:03::","discover%2:36:00::","discover%2:39:03::"],"candidates":[{"index":1,"sense_key":"discover%2:39:03::","synset_id":"detect.v.01"},{"index":2,"sense_key":"discover%2:31:01::","synset_id":"learn.v.02"},{"index":3,"sense_key":"discover%2:36:00::","synset_id":"discover.v.03"},{"index":4,"sense_key":"discover%2:31:00::","synset_id":"discover.v.04"},{"index":5,"sense_key":"discover%2:40:00::","synset_id":"fall_upon.v.01"},{"index":6,"sense_key":"discover%2:32:04::","synset_id":"unwrap.v.02"},{"index":7,"sense_key":"discover%2:39:00::","synset_id":"discover.v.07"},{"index":8,"sense_key":"discover%2:31:03::","synset_id":"identify.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"discover%2:36:00::","call_ids":["senseval2.d001.s011.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"discover%2:36:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1133,"cached_input_tokens":0,"output_tokens":166,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003926,"input_uncached_usd":0.002266,"input_cached_usd":0.0,"output_usd":0.0016600000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.7160072079859674}
{"item_id":"senseval2.d001.s011.t007","gold_sense_keys":["early%3:00:00::"],"candidates":[{"index":1,"sense_key":"early%3:00:00::","synset_id":"early.a.01"},{"index":2,"sense_key":"early%3:00:02::","synset_id":"early.a.02"},{"index":3,"sense_key":"early%5:00:00:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"early%5:00:00:young:00","synset_id":"early.s.02"},{"index":5,"sense_key":"early%3:00:01::","synset_id":"early.a.05"},{"index":6,"sense_key":"early%5:00:01:future:00","synset_id":"early.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"early%3:00:00::","call_ids":["senseval2.d001.s011.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"early%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":862,"cached_input_tokens":0,"output_tokens":42,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002144,"input_uncached_usd":0.0017239999999999998,"input_cached_usd":0.0,"output_usd":0.00042,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.112974957912229}
{"item_id":"senseval2.d001.s011.t008","gold_sense_keys":["call%2:32:02::"],"candidates":[{"index":1,"sense_key":"call%2:32:02::","synset_id":"name.v.01"},{"index":2,"sense_key":"call%2:32:00::","synset_id":"call.v.02"},{"index":3,"sense_key":"call%2:32:01::","synset_id":"call.v.03"},{"index":4,"sense_key":"call%2:32:09::","synset_id":"shout.v.02"},{"index":5,"sense_key":"call%2:32:05::","synset_id":"call.v.05"},{"index":6,"sense_key":"call%2:41:00::","synset_id":"visit.v.03"},{"index":7,"sense_key":"call%2:41:04::","synset_id":"call.v.07"},{"index":8,"sense_key":"call%2:32:13::","synset_id":"call.v.08"},{"index":9,"sense_key":"call%2:32:04::","synset_id":"call.v.09"},{"index":10,"sense_key":"call%2:32:03::","synset_id":"call.v.10"},{"index":11,"sense_key":"call%2:42:09::","synset_id":"call.v.11"},{"index":12,"sense_key":"call%2:41:14::","synset_id":"address.v.06"},{"index":13,"sense_key":"call%2:41:12::","synset_id":"call.v.13"},{"index":14,"sense_key":"call%2:40:09::","synset_id":"call.v.14"},{"index":15,"sense_key":"call%2:40:05::","synset_id":"bid.v.04"},{"index":16,"sense_key":"call%2:38:00::","synset_id":"call.v.16"},{"index":17,"sense_key":"call%2:32:06::","synset_id":"call.v.17"},{"index":18,"sense_key":"call%2:32:10::","synset_id":"predict.v.01"},{"index":19,"sense_key":"call%2:32:12::","synset_id":"call.v.19"},{"index":20,"sense_key":"call%2:32:07::","synset_id":"call.v.20"},{"index":21,"sense_key":"call%2:32:08::","synset_id":"call.v.21"},{"index":22,"sense_key":"call%2:32:11::","synset_id":"call.v.22"},{"index":23,"sense_key":"call%2:32:15::","synset_id":"call.v.23"},{"index":24,"sense_key":"call%2:32:14::","synset_id":"call.v.24"},{"index":25,"sense_key":"call%2:31:13::","synset_id":"call.v.25"},{"index":26,"sense_key":"call%2:31:00::","synset_id":"call.v.26"},{"index":27,"sense_key":"call%2:31:05::","synset_id":"call.v.27"},{"index":28,"sense_key":"call%2:29:05::","synset_id":"call.v.28"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"call%2:32:02::","call_ids":["senseval2.d001.s011.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"call%2:32:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2140,"cached_input_tokens":0,"output_tokens":98,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00526,"input_uncached_usd":0.00428,"input_cached_usd":0.0,"output_usd":0.0009800000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4981257910840213}
{"item_id":"senseval2.d001.s012.t001","gold_sense_keys":["present%3:00:02::"],"candidates":[{"index":1,"sense_key":"present%3:00:01::","synset_id":"present.a.01"},{"index":2,"sense_key":"present%3:00:02::","synset_id":"present.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"present%3:00:02::","call_ids":["senseval2.d001.s012.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"present%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":661,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0020020000000000003,"input_uncached_usd":0.001322,"input_cached_usd":0.0,"output_usd":0.00068,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8401776250684634}
{"item_id":"senseval2.d001.s012.t002","gold_sense_keys":["cell%1:03:00::"],"candidates":[{"index":1,"sense_key":"cell%1:06:03::","synset_id":"cell.n.01"},{"index":2,"sense_key":"cell%1:03:00::","synset_id":"cell.n.02"},{"index":3,"sense_key":"cell%1:06:00::","synset_id":"cell.n.03"},{"index":4,"sense_key":"cell%1:14:00::","synset_id":"cell.n.04"},{"index":5,"sense_key":"cell%1:06:04::","synset_id":"cellular_telephone.n.01"},{"index":6,"sense_key":"cell%1:06:02::","synset_id":"cell.n.06"},{"index":7,"sense_key":"cell%1:06:01::","synset_id":"cell.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cell%1:03:00::","call_ids":["senseval2.d001.s012.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cell%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":961,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002042,"input_uncached_usd":0.0019219999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9960369999753311}
{"item_id":"senseval2.d001.s012.t003","gold_sense_keys":["become%2:30:00::"],"candidates":[{"index":1,"sense_key":"become%2:30:00::","synset_id":"become.v.01"},{"index":2,"sense_key":"become%2:42:01::","synset_id":"become.v.02"},{"index":3,"sense_key":"become%2:42:00::","synset_id":"become.v.03"},{"index":4,"sense_key":"become%2:42:03::","synset_id":"become.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"become%2:30:00::","call_ids":["senseval2.d001.s012.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"become%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":779,"cached_input_tokens":0,"output_tokens":128,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0028380000000000002,"input_uncached_usd":0.001558,"input_cached_usd":0.0,"output_usd":0.00128,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.3341621250147}
{"item_id":"senseval2.d001.s012.t006","gold_sense_keys":["find%2:39:02::"],"candidates":[{"index":1,"sense_key":"find%2:40:02::","synset_id":"find.v.01"},{"index":2,"sense_key":"find%2:39:02::","synset_id":"detect.v.01"},{"index":3,"sense_key":"find%2:40:00::","synset_id":"find.v.03"},{"index":4,"sense_key":"find%2:32:00::","synset_id":"determine.v.01"},{"index":5,"sense_key":"find%2:31:10::","synset_id":"find.v.05"},{"index":6,"sense_key":"find%2:39:01::","synset_id":"witness.v.02"},{"index":7,"sense_key":"find%2:40:01::","synset_id":"line_up.v.02"},{"index":8,"sense_key":"find%2:36:00::","synset_id":"discover.v.03"},{"index":9,"sense_key":"find%2:31:09::","synset_id":"discover.v.04"},{"index":10,"sense_key":"find%2:40:03::","synset_id":"find.v.10"},{"index":11,"sense_key":"find%2:32:01::","synset_id":"rule.v.04"},{"index":12,"sense_key":"find%2:30:00::","synset_id":"receive.v.02"},{"index":13,"sense_key":"find%2:39:05::","synset_id":"find.v.13"},{"index":14,"sense_key":"find%2:40:15::","synset_id":"recover.v.01"},{"index":15,"sense_key":"find%2:38:10::","synset_id":"find.v.15"},{"index":16,"sense_key":"find%2:30:13::","synset_id":"find_oneself.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"find%2:39:02::","call_ids":["senseval2.d001.s012.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"find%2:39:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1701,"cached_input_tokens":0,"output_tokens":94,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0043419999999999995,"input_uncached_usd":0.0034019999999999996,"input_cached_usd":0.0,"output_usd":0.0009400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4617578749312088}
{"item_id":"senseval2.d001.s012.t007","gold_sense_keys":["normal%3:00:01::"],"candidates":[{"index":1,"sense_key":"normal%3:00:01::","synset_id":"normal.a.01"},{"index":2,"sense_key":"normal%3:00:02::","synset_id":"normal.a.02"},{"index":3,"sense_key":"normal%3:00:03::","synset_id":"normal.a.03"},{"index":4,"sense_key":"normal%5:00:00:perpendicular:00","synset_id":"normal.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"normal%3:00:01::","call_ids":["senseval2.d001.s012.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"normal%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":758,"cached_input_tokens":0,"output_tokens":101,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002526,"input_uncached_usd":0.001516,"input_cached_usd":0.0,"output_usd":0.00101,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7979478749912232}
{"item_id":"senseval2.d001.s012.t008","gold_sense_keys":["cancerous%5:00:00:malignant:02"],"candidates":[{"index":1,"sense_key":"cancerous%5:00:00:malignant:02","synset_id":"cancerous.s.01"},{"index":2,"sense_key":"cancerous%5:00:00:malign:00","synset_id":"cancerous.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancerous%5:00:00:malignant:02","call_ids":["senseval2.d001.s012.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancerous%5:00:00:malignant:02","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":681,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001482,"input_uncached_usd":0.001362,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.127827625023201}
{"item_id":"senseval2.d001.s012.t009","gold_sense_keys":["cell%1:03:00::"],"candidates":[{"index":1,"sense_key":"cell%1:06:03::","synset_id":"cell.n.01"},{"index":2,"sense_key":"cell%1:03:00::","synset_id":"cell.n.02"},{"index":3,"sense_key":"cell%1:06:00::","synset_id":"cell.n.03"},{"index":4,"sense_key":"cell%1:14:00::","synset_id":"cell.n.04"},{"index":5,"sense_key":"cell%1:06:04::","synset_id":"cellular_telephone.n.01"},{"index":6,"sense_key":"cell%1:06:02::","synset_id":"cell.n.06"},{"index":7,"sense_key":"cell%1:06:01::","synset_id":"cell.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cell%1:03:00::","call_ids":["senseval2.d001.s012.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cell%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":961,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002042,"input_uncached_usd":0.0019219999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9483822080073878}
{"item_id":"senseval2.d001.s012.t010","gold_sense_keys":["suggest%2:32:02::"],"candidates":[{"index":1,"sense_key":"suggest%2:32:00::","synset_id":"propose.v.01"},{"index":2,"sense_key":"suggest%2:32:04::","synset_id":"hint.v.01"},{"index":3,"sense_key":"suggest%2:32:02::","synset_id":"suggest.v.03"},{"index":4,"sense_key":"suggest%2:32:03::","synset_id":"indicate.v.05"},{"index":5,"sense_key":"suggest%2:32:01::","synset_id":"suggest.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"suggest%2:32:02::","call_ids":["senseval2.d001.s012.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"suggest%2:32:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":844,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001808,"input_uncached_usd":0.0016879999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1695702919969335}
{"item_id":"senseval2.d001.s012.t012","gold_sense_keys":["cause%2:36:00::"],"candidates":[{"index":1,"sense_key":"cause%2:36:00::","synset_id":"cause.v.01"},{"index":2,"sense_key":"cause%2:32:00::","synset_id":"induce.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cause%2:36:00::","call_ids":["senseval2.d001.s012.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cause%2:36:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":704,"cached_input_tokens":0,"output_tokens":100,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002408,"input_uncached_usd":0.001408,"input_cached_usd":0.0,"output_usd":0.001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.877286000060849}
{"item_id":"senseval2.d001.s012.t013","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s012.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":869,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001858,"input_uncached_usd":0.001738,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0187502920161933}
{"item_id":"senseval2.d001.s013.t000","gold_sense_keys":["recent%5:00:00:past:00"],"candidates":[{"index":1,"sense_key":"recent%5:00:00:new:00","synset_id":"recent.s.01"},{"index":2,"sense_key":"recent%5:00:00:past:00","synset_id":"late.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"recent%5:00:00:past:00","call_ids":["senseval2.d001.s013.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"recent%5:00:00:past:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":645,"cached_input_tokens":0,"output_tokens":71,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002,"input_uncached_usd":0.00129,"input_cached_usd":0.0,"output_usd":0.00071,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3661954579874873}
{"item_id":"senseval2.d001.s013.t001","gold_sense_keys":["month%1:28:01::"],"candidates":[{"index":1,"sense_key":"month%1:28:01::","synset_id":"calendar_month.n.01"},{"index":2,"sense_key":"month%1:28:00::","synset_id":"month.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"month%1:28:00::","call_ids":["senseval2.d001.s013.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"month%1:28:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":645,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0019,"input_uncached_usd":0.00129,"input_cached_usd":0.0,"output_usd":0.0006100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7921310409437865}
{"item_id":"senseval2.d001.s013.t003","gold_sense_keys":["come%2:30:03::"],"candidates":[{"index":1,"sense_key":"come%2:38:00::","synset_id":"come.v.01"},{"index":2,"sense_key":"come%2:38:04::","synset_id":"arrive.v.01"},{"index":3,"sense_key":"come%2:30:01::","synset_id":"come.v.03"},{"index":4,"sense_key":"come%2:30:03::","synset_id":"come.v.04"},{"index":5,"sense_key":"come%2:42:04::","synset_id":"come.v.05"},{"index":6,"sense_key":"come%2:30:02::","synset_id":"come.v.06"},{"index":7,"sense_key":"come%2:42:13::","synset_id":"issue_forth.v.01"},{"index":8,"sense_key":"come%2:42:09::","synset_id":"hail.v.02"},{"index":9,"sense_key":"come%2:42:03::","synset_id":"come.v.09"},{"index":10,"sense_key":"come%2:42:05::","synset_id":"come.v.10"},{"index":11,"sense_key":"come%2:38:03::","synset_id":"come.v.11"},{"index":12,"sense_key":"come%2:42:00::","synset_id":"fall.v.04"},{"index":13,"sense_key":"come%2:30:13::","synset_id":"come.v.13"},{"index":14,"sense_key":"come%2:42:12::","synset_id":"total.v.01"},{"index":15,"sense_key":"come%2:42:11::","synset_id":"come.v.15"},{"index":16,"sense_key":"come%2:38:02::","synset_id":"come.v.16"},{"index":17,"sense_key":"come%2:31:14::","synset_id":"occur.v.02"},{"index":18,"sense_key":"come%2:42:15::","synset_id":"derive.v.05"},{"index":19,"sense_key":"come%2:42:02::","synset_id":"do.v.04"},{"index":20,"sense_key":"come%2:39:00::","synset_id":"come.v.20"},{"index":21,"sense_key":"come%2:31:13::","synset_id":"come.v.21"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"come%2:30:03::","call_ids":["senseval2.d001.s013.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"come%2:30:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1714,"cached_input_tokens":0,"output_tokens":78,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004208,"input_uncached_usd":0.003428,"input_cached_usd":0.0,"output_usd":0.0007800000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9423427080037072}
{"item_id":"senseval2.d001.s013.t006","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s013.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":858,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001836,"input_uncached_usd":0.0017159999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1801650000270456}
{"item_id":"senseval2.d001.s013.t011","gold_sense_keys":["turn%2:42:13::"],"candidates":[{"index":1,"sense_key":"turn%2:38:00::","synset_id":"turn.v.01"},{"index":2,"sense_key":"turn%2:30:04::","synset_id":"change_state.v.01"},{"index":3,"sense_key":"turn%2:42:00::","synset_id":"become.v.02"},{"index":4,"sense_key":"turn%2:38:02::","synset_id":"turn.v.04"},{"index":5,"sense_key":"turn%2:30:00::","synset_id":"change_by_reversal.v.01"},{"index":6,"sense_key":"turn%2:38:03::","synset_id":"turn.v.06"},{"index":7,"sense_key":"turn%2:30:03::","synset_id":"turn.v.07"},{"index":8,"sense_key":"turn%2:35:08::","synset_id":"turn.v.08"},{"index":9,"sense_key":"turn%2:38:04::","synset_id":"turn.v.09"},{"index":10,"sense_key":"turn%2:38:01::","synset_id":"turn.v.10"},{"index":11,"sense_key":"turn%2:38:06::","synset_id":"turn.v.11"},{"index":12,"sense_key":"turn%2:36:00::","synset_id":"plow.v.01"},{"index":13,"sense_key":"turn%2:35:06::","synset_id":"turn.v.13"},{"index":14,"sense_key":"turn%2:30:09::","synset_id":"turn.v.14"},{"index":15,"sense_key":"turn%2:29:00::","synset_id":"twist.v.10"},{"index":16,"sense_key":"turn%2:42:13::","synset_id":"turn.v.16"},{"index":17,"sense_key":"turn%2:41:10::","synset_id":"turn.v.17"},{"index":18,"sense_key":"turn%2:40:13::","synset_id":"turn.v.18"},{"index":19,"sense_key":"turn%2:38:13::","synset_id":"turn.v.19"},{"index":20,"sense_key":"turn%2:38:14::","synset_id":"turn.v.20"},{"index":21,"sense_key":"turn%2:35:11::","synset_id":"flex.v.05"},{"index":22,"sense_key":"turn%2:35:10::","synset_id":"turn.v.22"},{"index":23,"sense_key":"turn%2:33:13::","synset_id":"turn.v.23"},{"index":24,"sense_key":"turn%2:32:14::","synset_id":"call_on.v.01"},{"index":25,"sense_key":"turn%2:30:01::","synset_id":"sour.v.01"},{"index":26,"sense_key":"turn%2:30:14::","synset_id":"turn.v.26"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":16,"chosen_sense_key":"turn%2:42:13::","call_ids":["senseval2.d001.s013.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":16,"predicted_sense_key":"turn%2:42:13::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2102,"cached_input_tokens":0,"output_tokens":149,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.005693999999999999,"input_uncached_usd":0.004203999999999999,"input_cached_usd":0.0,"output_usd":0.0014900000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0067890000063926}
{"item_id":"senseval2.d001.s013.t013","gold_sense_keys":["cell%1:03:00::"],"candidates":[{"index":1,"sense_key":"cell%1:06:03::","synset_id":"cell.n.01"},{"index":2,"sense_key":"cell%1:03:00::","synset_id":"cell.n.02"},{"index":3,"sense_key":"cell%1:06:00::","synset_id":"cell.n.03"},{"index":4,"sense_key":"cell%1:14:00::","synset_id":"cell.n.04"},{"index":5,"sense_key":"cell%1:06:04::","synset_id":"cellular_telephone.n.01"},{"index":6,"sense_key":"cell%1:06:02::","synset_id":"cell.n.06"},{"index":7,"sense_key":"cell%1:06:01::","synset_id":"cell.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cell%1:03:00::","call_ids":["senseval2.d001.s013.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cell%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":950,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00202,"input_uncached_usd":0.0019,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0878265830688179}
{"item_id":"senseval2.d001.s013.t015","gold_sense_keys":["only%4:02:03::"],"candidates":[{"index":1,"sense_key":"only%4:02:02::","synset_id":"merely.r.01"},{"index":2,"sense_key":"only%4:02:01::","synset_id":"entirely.r.02"},{"index":3,"sense_key":"only%4:02:00::","synset_id":"only.r.03"},{"index":4,"sense_key":"only%4:02:04::","synset_id":"only.r.04"},{"index":5,"sense_key":"only%4:02:06::","synset_id":"only.r.05"},{"index":6,"sense_key":"only%4:02:03::","synset_id":"only.r.06"},{"index":7,"sense_key":"only%4:02:05::","synset_id":"only.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"only%4:02:03::","call_ids":["senseval2.d001.s013.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"only%4:02:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":906,"cached_input_tokens":0,"output_tokens":108,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002892,"input_uncached_usd":0.001812,"input_cached_usd":0.0,"output_usd":0.00108,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6691770000616089}
{"item_id":"senseval2.d001.s013.t017","gold_sense_keys":["damage%2:30:00::"],"candidates":[{"index":1,"sense_key":"damage%2:30:00::","synset_id":"damage.v.01"},{"index":2,"sense_key":"damage%2:30:01::","synset_id":"damage.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"damage%2:30:00::","call_ids":["senseval2.d001.s013.t017__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"damage%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":646,"cached_input_tokens":0,"output_tokens":123,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002522,"input_uncached_usd":0.001292,"input_cached_usd":0.0,"output_usd":0.0012300000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6946283339057118}
{"item_id":"senseval2.d001.s014.t002","gold_sense_keys":["inherit%2:40:01::"],"candidates":[{"index":1,"sense_key":"inherit%2:40:00::","synset_id":"inherit.v.01"},{"index":2,"sense_key":"inherit%2:40:02::","synset_id":"inherit.v.02"},{"index":3,"sense_key":"inherit%2:40:01::","synset_id":"inherit.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"inherit%2:40:01::","call_ids":["senseval2.d001.s014.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"inherit%2:40:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":700,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00152,"input_uncached_usd":0.0014,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.382300333934836}
{"item_id":"senseval2.d001.s014.t003","gold_sense_keys":["copy%1:06:00::"],"candidates":[{"index":1,"sense_key":"copy%1:10:00::","synset_id":"transcript.n.02"},{"index":2,"sense_key":"copy%1:06:00::","synset_id":"copy.n.02"},{"index":3,"sense_key":"copy%1:10:01::","synset_id":"copy.n.03"},{"index":4,"sense_key":"copy%1:10:02::","synset_id":"copy.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"copy%1:06:00::","call_ids":["senseval2.d001.s014.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"copy%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":759,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0016380000000000001,"input_uncached_usd":0.001518,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8820992909604684}
{"item_id":"senseval2.d001.s014.t004","gold_sense_keys":["parent%1:18:00::"],"candidates":[{"index":1,"sense_key":"parent%1:18:00::","synset_id":"parent.n.01"},{"index":2,"sense_key":"parent%1:03:00::","synset_id":"parent.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"parent%1:03:00::","call_ids":["senseval2.d001.s014.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"parent%1:03:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":651,"cached_input_tokens":0,"output_tokens":150,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002802,"input_uncached_usd":0.001302,"input_cached_usd":0.0,"output_usd":0.0015,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2411860829452053}
{"item_id":"senseval2.d001.s015.t000","gold_sense_keys":["copy%1:06:00::"],"candidates":[{"index":1,"sense_key":"copy%1:10:00::","synset_id":"transcript.n.02"},{"index":2,"sense_key":"copy%1:06:00::","synset_id":"copy.n.02"},{"index":3,"sense_key":"copy%1:10:01::","synset_id":"copy.n.03"},{"index":4,"sense_key":"copy%1:10:02::","synset_id":"copy.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"copy%1:06:00::","call_ids":["senseval2.d001.s015.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"copy%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":798,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002256,"input_uncached_usd":0.001596,"input_cached_usd":0.0,"output_usd":0.0006600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.409885665983893}
{"item_id":"senseval2.d001.s015.t003","gold_sense_keys":["need%2:42:00::"],"candidates":[{"index":1,"sense_key":"need%2:42:00::","synset_id":"necessitate.v.01"},{"index":2,"sense_key":"need%2:34:00::","synset_id":"want.v.02"},{"index":3,"sense_key":"need%2:34:01::","synset_id":"need.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"need%2:42:00::","call_ids":["senseval2.d001.s015.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"need%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":754,"cached_input_tokens":0,"output_tokens":204,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003548,"input_uncached_usd":0.001508,"input_cached_usd":0.0,"output_usd":0.00204,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2739259579684585}
{"item_id":"senseval2.d001.s015.t004","gold_sense_keys":["control%2:41:01::"],"candidates":[{"index":1,"sense_key":"control%2:41:00::","synset_id":"control.v.01"},{"index":2,"sense_key":"control%2:41:01::","synset_id":"control.v.02"},{"index":3,"sense_key":"control%2:35:00::","synset_id":"operate.v.03"},{"index":4,"sense_key":"control%2:37:06::","synset_id":"manipulate.v.05"},{"index":5,"sense_key":"control%2:41:02::","synset_id":"control.v.05"},{"index":6,"sense_key":"control%2:31:11::","synset_id":"control.v.06"},{"index":7,"sense_key":"control%2:31:01::","synset_id":"see.v.10"},{"index":8,"sense_key":"control%2:31:00::","synset_id":"master.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"control%2:41:01::","call_ids":["senseval2.d001.s015.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"control%2:41:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1146,"cached_input_tokens":0,"output_tokens":148,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003772,"input_uncached_usd":0.0022919999999999998,"input_cached_usd":0.0,"output_usd":0.0014800000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.531148958951235}
{"item_id":"senseval2.d001.s015.t005","gold_sense_keys":["cell%1:03:00::"],"candidates":[{"index":1,"sense_key":"cell%1:06:03::","synset_id":"cell.n.01"},{"index":2,"sense_key":"cell%1:03:00::","synset_id":"cell.n.02"},{"index":3,"sense_key":"cell%1:06:00::","synset_id":"cell.n.03"},{"index":4,"sense_key":"cell%1:14:00::","synset_id":"cell.n.04"},{"index":5,"sense_key":"cell%1:06:04::","synset_id":"cellular_telephone.n.01"},{"index":6,"sense_key":"cell%1:06:02::","synset_id":"cell.n.06"},{"index":7,"sense_key":"cell%1:06:01::","synset_id":"cell.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cell%1:03:00::","call_ids":["senseval2.d001.s015.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cell%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":994,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0021079999999999996,"input_uncached_usd":0.0019879999999999997,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.079346709069796}
{"item_id":"senseval2.d001.s015.t006","gold_sense_keys":["growth%1:22:02::"],"candidates":[{"index":1,"sense_key":"growth%1:22:00::","synset_id":"growth.n.01"},{"index":2,"sense_key":"growth%1:22:01::","synset_id":"growth.n.02"},{"index":3,"sense_key":"growth%1:22:02::","synset_id":"increase.n.03"},{"index":4,"sense_key":"growth%1:14:00::","synset_id":"growth.n.04"},{"index":5,"sense_key":"growth%1:11:01::","synset_id":"emergence.n.01"},{"index":6,"sense_key":"growth%1:26:00::","synset_id":"growth.n.06"},{"index":7,"sense_key":"growth%1:17:00::","synset_id":"growth.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"growth%1:22:02::","call_ids":["senseval2.d001.s015.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"growth%1:22:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1025,"cached_input_tokens":0,"output_tokens":116,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0032099999999999997,"input_uncached_usd":0.0020499999999999997,"input_cached_usd":0.0,"output_usd":0.00116,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.3183206249959767}
{"item_id":"senseval2.d001.s015.t007","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s015.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":902,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001924,"input_uncached_usd":0.0018039999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4840765409171581}
{"item_id":"senseval2.d001.s015.t008","gold_sense_keys":["arise%2:42:00::"],"candidates":[{"index":1,"sense_key":"arise%2:42:00::","synset_id":"originate.v.01"},{"index":2,"sense_key":"arise%2:42:01::","synset_id":"arise.v.02"},{"index":3,"sense_key":"arise%2:38:00::","synset_id":"arise.v.03"},{"index":4,"sense_key":"arise%2:30:00::","synset_id":"arise.v.04"},{"index":5,"sense_key":"arise%2:38:04::","synset_id":"rise.v.01"},{"index":6,"sense_key":"arise%2:41:00::","synset_id":"rebel.v.01"},{"index":7,"sense_key":"arise%2:29:00::","synset_id":"get_up.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"arise%2:42:00::","call_ids":["senseval2.d001.s015.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"arise%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1031,"cached_input_tokens":0,"output_tokens":124,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0033020000000000002,"input_uncached_usd":0.002062,"input_cached_usd":0.0,"output_usd":0.00124,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.2422765410738066}
{"item_id":"senseval2.d001.s015.t009","gold_sense_keys":["copy%1:06:00::"],"candidates":[{"index":1,"sense_key":"copy%1:10:00::","synset_id":"transcript.n.02"},{"index":2,"sense_key":"copy%1:06:00::","synset_id":"copy.n.02"},{"index":3,"sense_key":"copy%1:10:01::","synset_id":"copy.n.03"},{"index":4,"sense_key":"copy%1:10:02::","synset_id":"copy.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"copy%1:06:00::","call_ids":["senseval2.d001.s015.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"copy%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":798,"cached_input_tokens":0,"output_tokens":85,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0024460000000000003,"input_uncached_usd":0.001596,"input_cached_usd":0.0,"output_usd":0.0008500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.362037832965143}
{"item_id":"senseval2.d001.s015.t010","gold_sense_keys":["impaired%3:00:00::"],"candidates":[{"index":1,"sense_key":"impaired%3:00:00::","synset_id":"impaired.a.01"},{"index":2,"sense_key":"impaired%5:00:00:unfit:01","synset_id":"afflicted.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"impaired%3:00:00::","call_ids":["senseval2.d001.s015.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"impaired%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":681,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002082,"input_uncached_usd":0.001362,"input_cached_usd":0.0,"output_usd":0.00072,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3615718750515953}
{"item_id":"senseval2.d001.s016.t000","gold_sense_keys":["person%1:03:00::"],"candidates":[{"index":1,"sense_key":"person%1:03:00::","synset_id":"person.n.01"},{"index":2,"sense_key":"person%1:08:00::","synset_id":"person.n.02"},{"index":3,"sense_key":"person%1:10:00::","synset_id":"person.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"person%1:03:00::","call_ids":["senseval2.d001.s016.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"person%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":768,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0016560000000000001,"input_uncached_usd":0.001536,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1596982500050217}
{"item_id":"senseval2.d001.s016.t001","gold_sense_keys":["bear%2:29:01::"],"candidates":[{"index":1,"sense_key":"bear%2:42:01::","synset_id":"bear.v.01"},{"index":2,"sense_key":"bear%2:29:01::","synset_id":"give_birth.v.01"},{"index":3,"sense_key":"bear%2:31:00::","synset_id":"digest.v.03"},{"index":4,"sense_key":"bear%2:35:00::","synset_id":"bear.v.04"},{"index":5,"sense_key":"bear%2:36:00::","synset_id":"bear.v.05"},{"index":6,"sense_key":"bear%2:40:02::","synset_id":"bear.v.06"},{"index":7,"sense_key":"bear%2:42:14::","synset_id":"hold.v.11"},{"index":8,"sense_key":"bear%2:40:01::","synset_id":"yield.v.10"},{"index":9,"sense_key":"bear%2:29:02::","synset_id":"wear.v.02"},{"index":10,"sense_key":"bear%2:41:00::","synset_id":"behave.v.02"},{"index":11,"sense_key":"bear%2:40:00::","synset_id":"bear.v.11"},{"index":12,"sense_key":"bear%2:35:01::","synset_id":"hold.v.14"},{"index":13,"sense_key":"bear%2:29:00::","synset_id":"have_a_bun_in_the_oven.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bear%2:29:01::","call_ids":["senseval2.d001.s016.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bear%2:29:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1438,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003416,"input_uncached_usd":0.002876,"input_cached_usd":0.0,"output_usd":0.00054,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3160819169133902}
{"item_id":"senseval2.d001.s016.t003","gold_sense_keys":["copy%1:06:00::"],"candidates":[{"index":1,"sense_key":"copy%1:10:00::","synset_id":"transcript.n.02"},{"index":2,"sense_key":"copy%1:06:00::","synset_id":"copy.n.02"},{"index":3,"sense_key":"copy%1:10:01::","synset_id":"copy.n.03"},{"index":4,"sense_key":"copy%1:10:02::","synset_id":"copy.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"copy%1:06:00::","call_ids":["senseval2.d001.s016.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"copy%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":788,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001696,"input_uncached_usd":0.001576,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0559791669948027}
{"item_id":"senseval2.d001.s016.t005","gold_sense_keys":["copy%1:06:00::"],"candidates":[{"index":1,"sense_key":"copy%1:10:00::","synset_id":"transcript.n.02"},{"index":2,"sense_key":"copy%1:06:00::","synset_id":"copy.n.02"},{"index":3,"sense_key":"copy%1:10:01::","synset_id":"copy.n.03"},{"index":4,"sense_key":"copy%1:10:02::","synset_id":"copy.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"copy%1:06:00::","call_ids":["senseval2.d001.s016.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"copy%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":788,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001696,"input_uncached_usd":0.001576,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0457709999755025}
{"item_id":"senseval2.d001.s016.t006","gold_sense_keys":["damage%2:30:00::"],"candidates":[{"index":1,"sense_key":"damage%2:30:00::","synset_id":"damage.v.01"},{"index":2,"sense_key":"damage%2:30:01::","synset_id":"damage.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"damage%2:30:00::","call_ids":["senseval2.d001.s016.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"damage%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":680,"cached_input_tokens":0,"output_tokens":137,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00273,"input_uncached_usd":0.0013599999999999999,"input_cached_usd":0.0,"output_usd":0.0013700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.947147374972701}
{"item_id":"senseval2.d001.s016.t007","gold_sense_keys":["early%4:02:01::"],"candidates":[{"index":1,"sense_key":"early%4:02:01::","synset_id":"early_on.r.01"},{"index":2,"sense_key":"early%4:02:00::","synset_id":"early.r.02"},{"index":3,"sense_key":"early%4:02:02::","synset_id":"early.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"early%4:02:01::","call_ids":["senseval2.d001.s016.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"early%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":729,"cached_input_tokens":0,"output_tokens":69,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0021479999999999997,"input_uncached_usd":0.0014579999999999999,"input_cached_usd":0.0,"output_usd":0.0006900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3343509590486065}
{"item_id":"senseval2.d001.s016.t008","gold_sense_keys":["life%1:28:00::","life%1:28:01::"],"candidates":[{"index":1,"sense_key":"life%1:26:01::","synset_id":"life.n.01"},{"index":2,"sense_key":"life%1:09:00::","synset_id":"life.n.02"},{"index":3,"sense_key":"life%1:26:02::","synset_id":"life.n.03"},{"index":4,"sense_key":"life%1:26:00::","synset_id":"animation.n.01"},{"index":5,"sense_key":"life%1:28:00::","synset_id":"life.n.05"},{"index":6,"sense_key":"life%1:28:01::","synset_id":"life.n.06"},{"index":7,"sense_key":"life%1:28:02::","synset_id":"life.n.07"},{"index":8,"sense_key":"life%1:18:00::","synset_id":"life.n.08"},{"index":9,"sense_key":"life%1:07:01::","synset_id":"liveliness.n.02"},{"index":10,"sense_key":"life%1:03:00::","synset_id":"life.n.10"},{"index":11,"sense_key":"life%1:19:00::","synset_id":"life.n.11"},{"index":12,"sense_key":"life%1:10:00::","synset_id":"biography.n.01"},{"index":13,"sense_key":"life%1:16:00::","synset_id":"life.n.13"},{"index":14,"sense_key":"life%1:28:03::","synset_id":"life_sentence.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"life%1:28:00::","call_ids":["senseval2.d001.s016.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"life%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1334,"cached_input_tokens":0,"output_tokens":136,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004028,"input_uncached_usd":0.002668,"input_cached_usd":0.0,"output_usd":0.00136,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6969053340144455}
{"item_id":"senseval2.d001.s016.t009","gold_sense_keys":["especially%4:02:00::"],"candidates":[{"index":1,"sense_key":"especially%4:02:00::","synset_id":"particularly.r.01"},{"index":2,"sense_key":"especially%4:02:01::","synset_id":"specially.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"especially%4:02:00::","call_ids":["senseval2.d001.s016.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"especially%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":706,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001532,"input_uncached_usd":0.0014119999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0716364999534562}
{"item_id":"senseval2.d001.s016.t010","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s016.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":892,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001904,"input_uncached_usd":0.001784,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2282216660678387}
{"item_id":"senseval2.d001.s016.t012","gold_sense_keys":["only%4:02:02::"],"candidates":[{"index":1,"sense_key":"only%4:02:02::","synset_id":"merely.r.01"},{"index":2,"sense_key":"only%4:02:01::","synset_id":"entirely.r.02"},{"index":3,"sense_key":"only%4:02:00::","synset_id":"only.r.03"},{"index":4,"sense_key":"only%4:02:04::","synset_id":"only.r.04"},{"index":5,"sense_key":"only%4:02:06::","synset_id":"only.r.05"},{"index":6,"sense_key":"only%4:02:03::","synset_id":"only.r.06"},{"index":7,"sense_key":"only%4:02:05::","synset_id":"only.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"only%4:02:02::","call_ids":["senseval2.d001.s016.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"only%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":940,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00252,"input_uncached_usd":0.00188,"input_cached_usd":0.0,"output_usd":0.00064,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3310644589364529}
{"item_id":"senseval2.d001.s016.t013","gold_sense_keys":["lose%2:40:02::"],"candidates":[{"index":1,"sense_key":"lose%2:40:02::","synset_id":"lose.v.01"},{"index":2,"sense_key":"lose%2:33:00::","synset_id":"lose.v.02"},{"index":3,"sense_key":"lose%2:37:00::","synset_id":"lose.v.03"},{"index":4,"sense_key":"lose%2:35:00::","synset_id":"misplace.v.01"},{"index":5,"sense_key":"lose%2:40:01::","synset_id":"lose.v.05"},{"index":6,"sense_key":"lose%2:39:00::","synset_id":"lose.v.06"},{"index":7,"sense_key":"lose%2:40:06::","synset_id":"lose.v.07"},{"index":8,"sense_key":"lose%2:40:00::","synset_id":"lose.v.08"},{"index":9,"sense_key":"lose%2:33:01::","synset_id":"fall_back.v.04"},{"index":10,"sense_key":"lose%2:39:01::","synset_id":"miss.v.01"},{"index":11,"sense_key":"lose%2:30:03::","synset_id":"suffer.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"lose%2:40:02::","call_ids":["senseval2.d001.s016.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"lose%2:40:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1230,"cached_input_tokens":0,"output_tokens":96,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0034200000000000003,"input_uncached_usd":0.00246,"input_cached_usd":0.0,"output_usd":0.0009600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0505794580094516}
{"item_id":"senseval2.d001.s016.t014","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":2,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":3,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"other%3:00:00::","call_ids":["senseval2.d001.s016.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":808,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0017360000000000001,"input_uncached_usd":0.001616,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4180097499629483}
{"item_id":"senseval2.d001.s016.t015","gold_sense_keys":["copy%1:06:00::"],"candidates":[{"index":1,"sense_key":"copy%1:10:00::","synset_id":"transcript.n.02"},{"index":2,"sense_key":"copy%1:06:00::","synset_id":"copy.n.02"},{"index":3,"sense_key":"copy%1:10:01::","synset_id":"copy.n.03"},{"index":4,"sense_key":"copy%1:10:02::","synset_id":"copy.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"copy%1:06:00::","call_ids":["senseval2.d001.s016.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"copy%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":788,"cached_input_tokens":0,"output_tokens":87,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0024460000000000003,"input_uncached_usd":0.001576,"input_cached_usd":0.0,"output_usd":0.0008700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4579187909839675}
{"item_id":"senseval2.d001.s016.t016","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s016.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":892,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001904,"input_uncached_usd":0.001784,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.007351291947998}
{"item_id":"senseval2.d001.s016.t017","gold_sense_keys":["develop%2:30:00::","develop%2:42:00::"],"candidates":[{"index":1,"sense_key":"develop%2:36:01::","synset_id":"develop.v.01"},{"index":2,"sense_key":"develop%2:36:00::","synset_id":"evolve.v.01"},{"index":3,"sense_key":"develop%2:30:06::","synset_id":"develop.v.03"},{"index":4,"sense_key":"develop%2:29:00::","synset_id":"grow.v.08"},{"index":5,"sense_key":"develop%2:42:00::","synset_id":"originate.v.01"},{"index":6,"sense_key":"develop%2:30:04::","synset_id":"build_up.v.05"},{"index":7,"sense_key":"develop%2:32:00::","synset_id":"explicate.v.02"},{"index":8,"sense_key":"develop%2:31:00::","synset_id":"train.v.01"},{"index":9,"sense_key":"develop%2:30:05::","synset_id":"develop.v.09"},{"index":10,"sense_key":"develop%2:30:00::","synset_id":"develop.v.10"},{"index":11,"sense_key":"develop%2:30:09::","synset_id":"modernize.v.02"},{"index":12,"sense_key":"develop%2:30:01::","synset_id":"develop.v.12"},{"index":13,"sense_key":"develop%2:36:09::","synset_id":"develop.v.13"},{"index":14,"sense_key":"develop%2:30:10::","synset_id":"develop.v.14"},{"index":15,"sense_key":"develop%2:39:00::","synset_id":"develop.v.15"},{"index":16,"sense_key":"develop%2:35:04::","synset_id":"develop.v.16"},{"index":17,"sense_key":"develop%2:33:03::","synset_id":"develop.v.17"},{"index":18,"sense_key":"develop%2:33:02::","synset_id":"develop.v.18"},{"index":19,"sense_key":"develop%2:30:13::","synset_id":"develop.v.19"},{"index":20,"sense_key":"develop%2:30:03::","synset_id":"break.v.53"},{"index":21,"sense_key":"develop%2:30:12::","synset_id":"develop.v.21"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"develop%2:42:00::","call_ids":["senseval2.d001.s016.t017__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"develop%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1954,"cached_input_tokens":0,"output_tokens":155,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.005458,"input_uncached_usd":0.003908,"input_cached_usd":0.0,"output_usd":0.0015500000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.3760360840242356}
{"item_id":"senseval2.d001.s017.t000","gold_sense_keys":["emerging%5:00:00:nascent:00"],"candidates":[{"index":1,"sense_key":"emerging%5:00:00:future:00","synset_id":"emerging.s.01"},{"index":2,"sense_key":"emerging%5:00:00:nascent:00","synset_id":"emergent.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"emerging%5:00:00:nascent:00","call_ids":["senseval2.d001.s017.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"emerging%5:00:00:nascent:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":677,"cached_input_tokens":0,"output_tokens":86,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002214,"input_uncached_usd":0.001354,"input_cached_usd":0.0,"output_usd":0.0008600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.61070966697298}
{"item_id":"senseval2.d001.s017.t001","gold_sense_keys":["genetic%3:01:00::"],"candidates":[{"index":1,"sense_key":"genetic%5:00:00:inheritable:00","synset_id":"familial.s.01"},{"index":2,"sense_key":"genetic%3:01:02::","synset_id":"genic.a.01"},{"index":3,"sense_key":"genetic%3:01:01::","synset_id":"genetic.a.03"},{"index":4,"sense_key":"genetic%3:01:00::","synset_id":"genetic.a.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"genetic%3:01:02::","call_ids":["senseval2.d001.s017.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"genetic%3:01:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":829,"cached_input_tokens":0,"output_tokens":161,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003268,"input_uncached_usd":0.001658,"input_cached_usd":0.0,"output_usd":0.00161,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9814094579778612}
{"item_id":"senseval2.d001.s017.t003","gold_sense_keys":["able%3:00:00::"],"candidates":[{"index":1,"sense_key":"able%3:00:00::","synset_id":"able.a.01"},{"index":2,"sense_key":"able%5:00:00:competent:00","synset_id":"able.s.01"},{"index":3,"sense_key":"able%5:00:00:capable:00","synset_id":"able.s.02"},{"index":4,"sense_key":"able%5:00:00:fit:01","synset_id":"able.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"able%3:00:00::","call_ids":["senseval2.d001.s017.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"able%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":825,"cached_input_tokens":0,"output_tokens":131,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00296,"input_uncached_usd":0.00165,"input_cached_usd":0.0,"output_usd":0.0013100000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.849679374950938}
{"item_id":"senseval2.d001.s017.t006","gold_sense_keys":["individual%1:03:00::"],"candidates":[{"index":1,"sense_key":"individual%1:03:00::","synset_id":"person.n.01"},{"index":2,"sense_key":"individual%1:18:00::","synset_id":"individual.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"individual%1:03:00::","call_ids":["senseval2.d001.s017.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"individual%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":659,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001438,"input_uncached_usd":0.001318,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.558089500060305}
{"item_id":"senseval2.d001.s017.t009","gold_sense_keys":["believe%2:31:04::"],"candidates":[{"index":1,"sense_key":"believe%2:31:00::","synset_id":"believe.v.01"},{"index":2,"sense_key":"believe%2:31:04::","synset_id":"think.v.01"},{"index":3,"sense_key":"believe%2:31:03::","synset_id":"believe.v.03"},{"index":4,"sense_key":"believe%2:31:01::","synset_id":"believe.v.04"},{"index":5,"sense_key":"believe%2:31:02::","synset_id":"believe.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"believe%2:31:04::","call_ids":["senseval2.d001.s017.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"believe%2:31:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":862,"cached_input_tokens":0,"output_tokens":161,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003334,"input_uncached_usd":0.0017239999999999998,"input_cached_usd":0.0,"output_usd":0.00161,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.034998916904442}
{"item_id":"senseval2.d001.s017.t010","gold_sense_keys":["new%3:00:00::","new%5:00:00:original:00","new%5:00:00:other:00","new%5:00:00:unprecedented:00"],"candidates":[{"index":1,"sense_key":"new%3:00:00::","synset_id":"new.a.01"},{"index":2,"sense_key":"new%5:00:00:original:00","synset_id":"fresh.s.03"},{"index":3,"sense_key":"new%5:00:00:inexperienced:00","synset_id":"raw.s.11"},{"index":4,"sense_key":"new%5:00:00:unprecedented:00","synset_id":"new.s.03"},{"index":5,"sense_key":"new%5:00:00:other:00","synset_id":"new.s.04"},{"index":6,"sense_key":"new%3:00:09::","synset_id":"new.a.06"},{"index":7,"sense_key":"new%5:00:02:original:00","synset_id":"newfangled.s.01"},{"index":8,"sense_key":"new%5:00:02:late:01","synset_id":"new.s.06"},{"index":9,"sense_key":"new%5:00:01:late:01","synset_id":"modern.s.04"},{"index":10,"sense_key":"new%5:00:00:early:02","synset_id":"new.s.08"},{"index":11,"sense_key":"new%5:00:00:unaccustomed:00","synset_id":"new.s.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"new%3:00:00::","call_ids":["senseval2.d001.s017.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"new%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1262,"cached_input_tokens":0,"output_tokens":92,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003444,"input_uncached_usd":0.0025239999999999998,"input_cached_usd":0.0,"output_usd":0.00092,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5564887910149992}
{"item_id":"senseval2.d001.s017.t011","gold_sense_keys":["age%1:28:02::"],"candidates":[{"index":1,"sense_key":"age%1:07:00::","synset_id":"age.n.01"},{"index":2,"sense_key":"age%1:28:02::","synset_id":"historic_period.n.01"},{"index":3,"sense_key":"age%1:28:00::","synset_id":"age.n.03"},{"index":4,"sense_key":"age%1:28:03::","synset_id":"long_time.n.01"},{"index":5,"sense_key":"age%1:28:01::","synset_id":"old_age.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"age%1:28:02::","call_ids":["senseval2.d001.s017.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"age%1:28:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":861,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001842,"input_uncached_usd":0.001722,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5183572500245646}
{"item_id":"senseval2.d001.s017.t013","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s017.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":890,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0019,"input_uncached_usd":0.00178,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0099015000741929}
{"item_id":"senseval2.d001.s018.t000","gold_sense_keys":["first%3:00:00::"],"candidates":[{"index":1,"sense_key":"first%3:00:00::","synset_id":"first.a.01"},{"index":2,"sense_key":"first%5:00:00:ordinal:00","synset_id":"first.s.01"},{"index":3,"sense_key":"first%5:00:02:opening:00","synset_id":"inaugural.s.01"},{"index":4,"sense_key":"first%5:00:01:opening:00","synset_id":"beginning.s.01"},{"index":5,"sense_key":"first%5:00:00:best:00","synset_id":"first.s.04"},{"index":6,"sense_key":"first%3:00:03::","synset_id":"first.a.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"first%3:00:00::","call_ids":["senseval2.d001.s018.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"first%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":951,"cached_input_tokens":0,"output_tokens":112,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003022,"input_uncached_usd":0.0019019999999999998,"input_cached_usd":0.0,"output_usd":0.0011200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1294219999108464}
{"item_id":"senseval2.d001.s018.t001","gold_sense_keys":["beneficiary%1:18:00::"],"candidates":[{"index":1,"sense_key":"beneficiary%1:18:00::","synset_id":"beneficiary.n.01"},{"index":2,"sense_key":"beneficiary%1:10:00::","synset_id":"benefactive_role.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"beneficiary%1:18:00::","call_ids":["senseval2.d001.s018.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"beneficiary%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":709,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001538,"input_uncached_usd":0.001418,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5669319169828668}
{"item_id":"senseval2.d001.s018.t002","gold_sense_keys":["new%3:00:00::","new%5:00:00:original:00","new%5:00:00:other:00","new%5:00:00:unprecedented:00"],"candidates":[{"index":1,"sense_key":"new%3:00:00::","synset_id":"new.a.01"},{"index":2,"sense_key":"new%5:00:00:original:00","synset_id":"fresh.s.03"},{"index":3,"sense_key":"new%5:00:00:inexperienced:00","synset_id":"raw.s.11"},{"index":4,"sense_key":"new%5:00:00:unprecedented:00","synset_id":"new.s.03"},{"index":5,"sense_key":"new%5:00:00:other:00","synset_id":"new.s.04"},{"index":6,"sense_key":"new%3:00:09::","synset_id":"new.a.06"},{"index":7,"sense_key":"new%5:00:02:original:00","synset_id":"newfangled.s.01"},{"index":8,"sense_key":"new%5:00:02:late:01","synset_id":"new.s.06"},{"index":9,"sense_key":"new%5:00:01:late:01","synset_id":"modern.s.04"},{"index":10,"sense_key":"new%5:00:00:early:02","synset_id":"new.s.08"},{"index":11,"sense_key":"new%5:00:00:unaccustomed:00","synset_id":"new.s.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"new%3:00:00::","call_ids":["senseval2.d001.s018.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"new%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1279,"cached_input_tokens":0,"output_tokens":95,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003508,"input_uncached_usd":0.002558,"input_cached_usd":0.0,"output_usd":0.0009500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.4292745830025524}
{"item_id":"senseval2.d001.s018.t003","gold_sense_keys":["finding%1:17:00::"],"candidates":[{"index":1,"sense_key":"finding%1:04:00::","synset_id":"determination.n.01"},{"index":2,"sense_key":"finding%1:04:01::","synset_id":"finding.n.02"},{"index":3,"sense_key":"finding%1:17:00::","synset_id":"finding.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"finding%1:17:00::","call_ids":["senseval2.d001.s018.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"finding%1:17:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":757,"cached_input_tokens":0,"output_tokens":138,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002894,"input_uncached_usd":0.001514,"input_cached_usd":0.0,"output_usd":0.0013800000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.007802333100699}
{"item_id":"senseval2.d001.s019.t000","gold_sense_keys":["couple%1:14:01::"],"candidates":[{"index":1,"sense_key":"couple%1:14:00::","synset_id":"couple.n.01"},{"index":2,"sense_key":"couple%1:14:01::","synset_id":"couple.n.02"},{"index":3,"sense_key":"couple%1:23:01::","synset_id":"couple.n.03"},{"index":4,"sense_key":"couple%1:23:00::","synset_id":"couple.n.04"},{"index":5,"sense_key":"couple%1:17:00::","synset_id":"couple.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"couple%1:14:01::","call_ids":["senseval2.d001.s019.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"couple%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":857,"cached_input_tokens":0,"output_tokens":131,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003024,"input_uncached_usd":0.001714,"input_cached_usd":0.0,"output_usd":0.0013100000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7334632499841973}
{"item_id":"senseval2.d001.s019.t001","gold_sense_keys":["know%2:31:01::","know%2:31:02::"],"candidates":[{"index":1,"sense_key":"know%2:31:01::","synset_id":"know.v.01"},{"index":2,"sense_key":"know%2:31:03::","synset_id":"know.v.02"},{"index":3,"sense_key":"know%2:31:02::","synset_id":"know.v.03"},{"index":4,"sense_key":"know%2:31:00::","synset_id":"know.v.04"},{"index":5,"sense_key":"know%2:31:04::","synset_id":"know.v.05"},{"index":6,"sense_key":"know%2:31:06::","synset_id":"acknowledge.v.06"},{"index":7,"sense_key":"know%2:31:15::","synset_id":"know.v.07"},{"index":8,"sense_key":"know%2:35:00::","synset_id":"sleep_together.v.01"},{"index":9,"sense_key":"know%2:31:11::","synset_id":"know.v.09"},{"index":10,"sense_key":"know%2:31:12::","synset_id":"know.v.10"},{"index":11,"sense_key":"know%2:31:14::","synset_id":"know.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"know%2:31:01::","call_ids":["senseval2.d001.s019.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"know%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1317,"cached_input_tokens":0,"output_tokens":102,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003654,"input_uncached_usd":0.002634,"input_cached_usd":0.0,"output_usd":0.00102,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8052511250134557}
{"item_id":"senseval2.d001.s019.t002","gold_sense_keys":["even%4:02:00::"],"candidates":[{"index":1,"sense_key":"even%4:02:00::","synset_id":"even.r.01"},{"index":2,"sense_key":"even%4:02:03::","synset_id":"even.r.02"},{"index":3,"sense_key":"even%4:02:01::","synset_id":"even.r.03"},{"index":4,"sense_key":"even%4:02:04::","synset_id":"even.r.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"even%4:02:00::","call_ids":["senseval2.d001.s019.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"even%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":822,"cached_input_tokens":0,"output_tokens":154,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003184,"input_uncached_usd":0.0016439999999999998,"input_cached_usd":0.0,"output_usd":0.0015400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.095540542039089}
{"item_id":"senseval2.d001.s019.t003","gold_sense_keys":["before%4:02:03::"],"candidates":[{"index":1,"sense_key":"before%4:02:03::","synset_id":"earlier.r.01"},{"index":2,"sense_key":"before%4:02:01::","synset_id":"ahead.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"before%4:02:03::","call_ids":["senseval2.d001.s019.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"before%4:02:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":709,"cached_input_tokens":0,"output_tokens":91,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002328,"input_uncached_usd":0.001418,"input_cached_usd":0.0,"output_usd":0.0009100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.845291457953863}
{"item_id":"senseval2.d001.s019.t004","gold_sense_keys":["become%2:30:00::"],"candidates":[{"index":1,"sense_key":"become%2:30:00::","synset_id":"become.v.01"},{"index":2,"sense_key":"become%2:42:01::","synset_id":"become.v.02"},{"index":3,"sense_key":"become%2:42:00::","synset_id":"become.v.03"},{"index":4,"sense_key":"become%2:42:03::","synset_id":"become.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"become%2:30:00::","call_ids":["senseval2.d001.s019.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"become%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":823,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002266,"input_uncached_usd":0.001646,"input_cached_usd":0.0,"output_usd":0.00062,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5164135829545557}
{"item_id":"senseval2.d001.s019.t005","gold_sense_keys":["pregnant%3:00:00::"],"candidates":[{"index":1,"sense_key":"pregnant%3:00:00::","synset_id":"pregnant.a.01"},{"index":2,"sense_key":"pregnant%5:00:00:meaningful:00","synset_id":"meaning.s.01"},{"index":3,"sense_key":"pregnant%5:00:00:full:00","synset_id":"fraught.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"pregnant%3:00:00::","call_ids":["senseval2.d001.s019.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"pregnant%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":767,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001654,"input_uncached_usd":0.001534,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9879634999670088}
{"item_id":"senseval2.d001.s019.t006","gold_sense_keys":["child%1:18:01::"],"candidates":[{"index":1,"sense_key":"child%1:18:00::","synset_id":"child.n.01"},{"index":2,"sense_key":"child%1:18:01::","synset_id":"child.n.02"},{"index":3,"sense_key":"child%1:18:06::","synset_id":"child.n.03"},{"index":4,"sense_key":"child%1:18:07::","synset_id":"child.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"child%1:18:01::","call_ids":["senseval2.d001.s019.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"child%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":832,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002304,"input_uncached_usd":0.001664,"input_cached_usd":0.0,"output_usd":0.00064,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.352528874995187}
{"item_id":"senseval2.d001.s019.t007","gold_sense_keys":["chance%1:07:00::"],"candidates":[{"index":1,"sense_key":"chance%1:26:00::","synset_id":"opportunity.n.01"},{"index":2,"sense_key":"chance%1:19:00::","synset_id":"luck.n.02"},{"index":3,"sense_key":"chance%1:04:00::","synset_id":"chance.n.03"},{"index":4,"sense_key":"chance%1:07:00::","synset_id":"probability.n.01"},{"index":5,"sense_key":"chance%1:26:01::","synset_id":"prospect.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"chance%1:07:00::","call_ids":["senseval2.d001.s019.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"chance%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":961,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002042,"input_uncached_usd":0.0019219999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.513338541961275}
{"item_id":"senseval2.d001.s019.t010","gold_sense_keys":["eye%1:08:00::"],"candidates":[{"index":1,"sense_key":"eye%1:08:00::","synset_id":"eye.n.01"},{"index":2,"sense_key":"eye%1:09:00::","synset_id":"eye.n.02"},{"index":3,"sense_key":"eye%1:09:01::","synset_id":"eye.n.03"},{"index":4,"sense_key":"eye%1:15:00::","synset_id":"center.n.01"},{"index":5,"sense_key":"eye%1:06:00::","synset_id":"eye.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"eye%1:08:00::","call_ids":["senseval2.d001.s019.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"eye%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":868,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001856,"input_uncached_usd":0.001736,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4289831250207499}
{"item_id":"senseval2.d001.s019.t011","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s019.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":913,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001946,"input_uncached_usd":0.001826,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.118230584077537}
{"item_id":"senseval2.d001.s019.t012","gold_sense_keys":["occur%2:42:03::"],"candidates":[{"index":1,"sense_key":"occur%2:30:00::","synset_id":"happen.v.01"},{"index":2,"sense_key":"occur%2:31:00::","synset_id":"occur.v.02"},{"index":3,"sense_key":"occur%2:42:03::","synset_id":"occur.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"occur%2:42:03::","call_ids":["senseval2.d001.s019.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"occur%2:42:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":799,"cached_input_tokens":0,"output_tokens":140,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0029980000000000002,"input_uncached_usd":0.001598,"input_cached_usd":0.0,"output_usd":0.0014000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0499245420796797}
{"item_id":"senseval2.d001.s019.t013","gold_sense_keys":["about%4:02:00::"],"candidates":[{"index":1,"sense_key":"about%4:02:00::","synset_id":"approximately.r.01"},{"index":2,"sense_key":"about%4:02:06::","synset_id":"about.r.02"},{"index":3,"sense_key":"about%4:02:01::","synset_id":"about.r.03"},{"index":4,"sense_key":"about%4:02:04::","synset_id":"about.r.04"},{"index":5,"sense_key":"about%4:02:02::","synset_id":"about.r.05"},{"index":6,"sense_key":"about%4:02:03::","synset_id":"about.r.06"},{"index":7,"sense_key":"about%4:02:05::","synset_id":"about.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"about%4:02:00::","call_ids":["senseval2.d001.s019.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"about%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1045,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0022099999999999997,"input_uncached_usd":0.00209,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.803784416988492}
{"item_id":"senseval2.d001.s019.t014","gold_sense_keys":["once%4:02:00::"],"candidates":[{"index":1,"sense_key":"once%4:02:00::","synset_id":"once.r.01"},{"index":2,"sense_key":"once%4:02:02::","synset_id":"once.r.02"},{"index":3,"sense_key":"once%4:02:01::","synset_id":"once.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"once%4:02:00::","call_ids":["senseval2.d001.s019.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"once%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":748,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002226,"input_uncached_usd":0.001496,"input_cached_usd":0.0,"output_usd":0.0007300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.346848375047557}
{"item_id":"senseval2.d001.s019.t015","gold_sense_keys":["birth%1:18:00::"],"candidates":[{"index":1,"sense_key":"birth%1:28:00::","synset_id":"birth.n.01"},{"index":2,"sense_key":"birth%1:11:00::","synset_id":"birth.n.02"},{"index":3,"sense_key":"birth%1:22:00::","synset_id":"parturition.n.01"},{"index":4,"sense_key":"birth%1:24:00::","synset_id":"parentage.n.02"},{"index":5,"sense_key":"birth%1:18:00::","synset_id":"birth.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"birth%1:18:00::","call_ids":["senseval2.d001.s019.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"birth%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":869,"cached_input_tokens":0,"output_tokens":105,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002788,"input_uncached_usd":0.001738,"input_cached_usd":0.0,"output_usd":0.0010500000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6740295409690589}
{"item_id":"senseval2.d001.s020.t000","gold_sense_keys":["know%2:31:01::","know%2:31:02::"],"candidates":[{"index":1,"sense_key":"know%2:31:01::","synset_id":"know.v.01"},{"index":2,"sense_key":"know%2:31:03::","synset_id":"know.v.02"},{"index":3,"sense_key":"know%2:31:02::","synset_id":"know.v.03"},{"index":4,"sense_key":"know%2:31:00::","synset_id":"know.v.04"},{"index":5,"sense_key":"know%2:31:04::","synset_id":"know.v.05"},{"index":6,"sense_key":"know%2:31:06::","synset_id":"acknowledge.v.06"},{"index":7,"sense_key":"know%2:31:15::","synset_id":"know.v.07"},{"index":8,"sense_key":"know%2:35:00::","synset_id":"sleep_together.v.01"},{"index":9,"sense_key":"know%2:31:11::","synset_id":"know.v.09"},{"index":10,"sense_key":"know%2:31:12::","synset_id":"know.v.10"},{"index":11,"sense_key":"know%2:31:14::","synset_id":"know.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"know%2:31:01::","call_ids":["senseval2.d001.s020.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"know%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1336,"cached_input_tokens":0,"output_tokens":202,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004692,"input_uncached_usd":0.002672,"input_cached_usd":0.0,"output_usd":0.00202,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.965643374947831}
{"item_id":"senseval2.d001.s020.t001","gold_sense_keys":["carry%2:42:13::"],"candidates":[{"index":1,"sense_key":"carry%2:35:02::","synset_id":"transport.v.02"},{"index":2,"sense_key":"carry%2:42:01::","synset_id":"carry.v.02"},{"index":3,"sense_key":"carry%2:38:03::","synset_id":"impart.v.03"},{"index":4,"sense_key":"carry%2:32:02::","synset_id":"carry.v.04"},{"index":5,"sense_key":"carry%2:35:09::","synset_id":"carry.v.05"},{"index":6,"sense_key":"carry%2:35:01::","synset_id":"hold.v.14"},{"index":7,"sense_key":"carry%2:42:03::","synset_id":"hold.v.11"},{"index":8,"sense_key":"carry%2:41:00::","synset_id":"carry.v.08"},{"index":9,"sense_key":"carry%2:30:05::","synset_id":"carry.v.09"},{"index":10,"sense_key":"carry%2:42:02::","synset_id":"carry.v.10"},{"index":11,"sense_key":"carry%2:33:01::","synset_id":"carry.v.11"},{"index":12,"sense_key":"carry%2:31:12::","synset_id":"carry.v.12"},{"index":13,"sense_key":"carry%2:41:01::","synset_id":"behave.v.02"},{"index":14,"sense_key":"carry%2:40:00::","synset_id":"stock.v.01"},{"index":15,"sense_key":"carry%2:36:00::","synset_id":"carry.v.15"},{"index":16,"sense_key":"carry%2:35:08::","synset_id":"dribble.v.03"},{"index":17,"sense_key":"carry%2:32:07::","synset_id":"carry.v.17"},{"index":18,"sense_key":"carry%2:42:13::","synset_id":"carry.v.18"},{"index":19,"sense_key":"carry%2:42:15::","synset_id":"carry.v.19"},{"index":20,"sense_key":"carry%2:42:14::","synset_id":"carry.v.20"},{"index":21,"sense_key":"carry%2:42:12::","synset_id":"carry.v.21"},{"index":22,"sense_key":"carry%2:42:08::","synset_id":"carry.v.22"},{"index":23,"sense_key":"carry%2:41:02::","synset_id":"carry.v.23"},{"index":24,"sense_key":"carry%2:41:06::","synset_id":"carry.v.24"},{"index":25,"sense_key":"carry%2:41:12::","synset_id":"carry.v.25"},{"index":26,"sense_key":"carry%2:40:10::","synset_id":"carry.v.26"},{"index":27,"sense_key":"carry%2:40:04::","synset_id":"carry.v.27"},{"index":28,"sense_key":"carry%2:40:13::","synset_id":"post.v.07"},{"index":29,"sense_key":"carry%2:38:10::","synset_id":"carry.v.29"},{"index":30,"sense_key":"carry%2:38:07::","synset_id":"carry.v.30"},{"index":31,"sense_key":"carry%2:36:08::","synset_id":"carry.v.31"},{"index":32,"sense_key":"carry%2:35:12::","synset_id":"carry.v.32"},{"index":33,"sense_key":"carry%2:34:00::","synset_id":"carry.v.33"},{"index":34,"sense_key":"carry%2:34:01::","synset_id":"carry.v.34"},{"index":35,"sense_key":"carry%2:33:04::","synset_id":"carry.v.35"},{"index":36,"sense_key":"carry%2:33:03::","synset_id":"carry.v.36"},{"index":37,"sense_key":"carry%2:33:02::","synset_id":"carry.v.37"},{"index":38,"sense_key":"carry%2:33:12::","synset_id":"carry.v.38"},{"index":39,"sense_key":"carry%2:32:00::","synset_id":"carry.v.39"},{"index":40,"sense_key":"carry%2:29:00::","synset_id":"have_a_bun_in_the_oven.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"carry%2:42:03::","call_ids":["senseval2.d001.s020.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"carry%2:42:03::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":2841,"cached_input_tokens":0,"output_tokens":163,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.007312,"input_uncached_usd":0.0056819999999999996,"input_cached_usd":0.0,"output_usd":0.0016300000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.6386967500438914}
{"item_id":"senseval2.d001.s020.t002","gold_sense_keys":["damaged%3:00:00::"],"candidates":[{"index":1,"sense_key":"damaged%3:00:00::","synset_id":"damaged.a.01"},{"index":2,"sense_key":"damaged%5:00:00:disreputable:00","synset_id":"discredited.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"damaged%3:00:00::","call_ids":["senseval2.d001.s020.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"damaged%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":750,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0016200000000000001,"input_uncached_usd":0.0015,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.185549582936801}
{"item_id":"senseval2.d001.s020.t004","gold_sense_keys":["lose%2:40:02::"],"candidates":[{"index":1,"sense_key":"lose%2:40:02::","synset_id":"lose.v.01"},{"index":2,"sense_key":"lose%2:33:00::","synset_id":"lose.v.02"},{"index":3,"sense_key":"lose%2:37:00::","synset_id":"lose.v.03"},{"index":4,"sense_key":"lose%2:35:00::","synset_id":"misplace.v.01"},{"index":5,"sense_key":"lose%2:40:01::","synset_id":"lose.v.05"},{"index":6,"sense_key":"lose%2:39:00::","synset_id":"lose.v.06"},{"index":7,"sense_key":"lose%2:40:06::","synset_id":"lose.v.07"},{"index":8,"sense_key":"lose%2:40:00::","synset_id":"lose.v.08"},{"index":9,"sense_key":"lose%2:33:01::","synset_id":"fall_back.v.04"},{"index":10,"sense_key":"lose%2:39:01::","synset_id":"miss.v.01"},{"index":11,"sense_key":"lose%2:30:03::","synset_id":"suffer.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"lose%2:40:02::","call_ids":["senseval2.d001.s020.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"lose%2:40:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1270,"cached_input_tokens":0,"output_tokens":147,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00401,"input_uncached_usd":0.0025399999999999997,"input_cached_usd":0.0,"output_usd":0.0014700000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8851101660402492}
{"item_id":"senseval2.d001.s020.t005","gold_sense_keys":["eye%1:08:00::"],"candidates":[{"index":1,"sense_key":"eye%1:08:00::","synset_id":"eye.n.01"},{"index":2,"sense_key":"eye%1:09:00::","synset_id":"eye.n.02"},{"index":3,"sense_key":"eye%1:09:01::","synset_id":"eye.n.03"},{"index":4,"sense_key":"eye%1:15:00::","synset_id":"center.n.01"},{"index":5,"sense_key":"eye%1:06:00::","synset_id":"eye.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"eye%1:08:00::","call_ids":["senseval2.d001.s020.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"eye%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":887,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001894,"input_uncached_usd":0.001774,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0992566250497475}
{"item_id":"senseval2.d001.s020.t008","gold_sense_keys":["only%4:02:02::"],"candidates":[{"index":1,"sense_key":"only%4:02:02::","synset_id":"merely.r.01"},{"index":2,"sense_key":"only%4:02:01::","synset_id":"entirely.r.02"},{"index":3,"sense_key":"only%4:02:00::","synset_id":"only.r.03"},{"index":4,"sense_key":"only%4:02:04::","synset_id":"only.r.04"},{"index":5,"sense_key":"only%4:02:06::","synset_id":"only.r.05"},{"index":6,"sense_key":"only%4:02:03::","synset_id":"only.r.06"},{"index":7,"sense_key":"only%4:02:05::","synset_id":"only.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"only%4:02:02::","call_ids":["senseval2.d001.s020.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"only%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":980,"cached_input_tokens":0,"output_tokens":123,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00319,"input_uncached_usd":0.00196,"input_cached_usd":0.0,"output_usd":0.0012300000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9626291249878705}
{"item_id":"senseval2.d001.s020.t009","gold_sense_keys":["mother%1:18:00::"],"candidates":[{"index":1,"sense_key":"mother%1:18:00::","synset_id":"mother.n.01"},{"index":2,"sense_key":"mother%1:27:00::","synset_id":"mother.n.02"},{"index":3,"sense_key":"mother%1:18:02::","synset_id":"mother.n.03"},{"index":4,"sense_key":"mother%1:18:01::","synset_id":"mother.n.04"},{"index":5,"sense_key":"mother%1:09:00::","synset_id":"mother.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"mother%1:18:00::","call_ids":["senseval2.d001.s020.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"mother%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":867,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001854,"input_uncached_usd":0.0017339999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1762817079434171}
{"item_id":"senseval2.d001.s020.t010","gold_sense_keys":["suffer%2:39:01::","suffer%2:39:02::"],"candidates":[{"index":1,"sense_key":"suffer%2:39:01::","synset_id":"suffer.v.01"},{"index":2,"sense_key":"suffer%2:29:01::","synset_id":"suffer.v.02"},{"index":3,"sense_key":"suffer%2:37:00::","synset_id":"suffer.v.03"},{"index":4,"sense_key":"suffer%2:31:00::","synset_id":"digest.v.03"},{"index":5,"sense_key":"suffer%2:30:00::","synset_id":"suffer.v.05"},{"index":6,"sense_key":"suffer%2:29:00::","synset_id":"suffer.v.06"},{"index":7,"sense_key":"suffer%2:39:00::","synset_id":"hurt.v.06"},{"index":8,"sense_key":"suffer%2:29:03::","synset_id":"suffer.v.08"},{"index":9,"sense_key":"suffer%2:42:04::","synset_id":"suffer.v.09"},{"index":10,"sense_key":"suffer%2:39:02::","synset_id":"suffer.v.10"},{"index":11,"sense_key":"suffer%2:30:02::","synset_id":"suffer.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"suffer%2:39:02::","call_ids":["senseval2.d001.s020.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"suffer%2:39:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1255,"cached_input_tokens":0,"output_tokens":123,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0037400000000000003,"input_uncached_usd":0.00251,"input_cached_usd":0.0,"output_usd":0.0012300000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.084866249933839}
{"item_id":"senseval2.d001.s020.t011","gold_sense_keys":["same%3:00:00::"],"candidates":[{"index":1,"sense_key":"same%3:00:02::","synset_id":"same.a.01"},{"index":2,"sense_key":"same%3:00:00::","synset_id":"same.a.02"},{"index":3,"sense_key":"same%3:00:04::","synset_id":"like.a.02"},{"index":4,"sense_key":"same%5:00:00:unchanged:00","synset_id":"same.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"same%3:00:00::","call_ids":["senseval2.d001.s020.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"same%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":835,"cached_input_tokens":0,"output_tokens":178,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00345,"input_uncached_usd":0.0016699999999999998,"input_cached_usd":0.0,"output_usd":0.0017800000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.302942750044167}
{"item_id":"senseval2.d001.s020.t012","gold_sense_keys":["fate%1:26:00::"],"candidates":[{"index":1,"sense_key":"fate%1:11:00::","synset_id":"destiny.n.01"},{"index":2,"sense_key":"fate%1:18:00::","synset_id":"destiny.n.02"},{"index":3,"sense_key":"fate%1:26:00::","synset_id":"fortune.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"fate%1:26:00::","call_ids":["senseval2.d001.s020.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"fate%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":833,"cached_input_tokens":0,"output_tokens":151,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003176,"input_uncached_usd":0.001666,"input_cached_usd":0.0,"output_usd":0.00151,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.4310477500548586}
{"item_id":"senseval2.d001.s020.t013","gold_sense_keys":["baby%1:18:00::"],"candidates":[{"index":1,"sense_key":"baby%1:18:00::","synset_id":"baby.n.01"},{"index":2,"sense_key":"baby%1:18:01::","synset_id":"baby.n.02"},{"index":3,"sense_key":"baby%1:18:03::","synset_id":"child.n.03"},{"index":4,"sense_key":"baby%1:18:04::","synset_id":"baby.n.04"},{"index":5,"sense_key":"baby%1:18:02::","synset_id":"baby.n.05"},{"index":6,"sense_key":"baby%1:05:00::","synset_id":"baby.n.06"},{"index":7,"sense_key":"baby%1:04:00::","synset_id":"baby.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"baby%1:18:00::","call_ids":["senseval2.d001.s020.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"baby%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1021,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002162,"input_uncached_usd":0.002042,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.608865209040232}
{"item_id":"senseval2.d001.s021.t000","gold_sense_keys":["isolation%1:04:00::"],"candidates":[{"index":1,"sense_key":"isolation%1:26:00::","synset_id":"isolation.n.01"},{"index":2,"sense_key":"isolation%1:12:00::","synset_id":"isolation.n.02"},{"index":3,"sense_key":"isolation%1:04:00::","synset_id":"isolation.n.03"},{"index":4,"sense_key":"isolation%1:22:00::","synset_id":"isolation.n.04"},{"index":5,"sense_key":"isolation%1:04:01::","synset_id":"isolation.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"isolation%1:04:00::","call_ids":["senseval2.d001.s021.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"isolation%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":882,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001884,"input_uncached_usd":0.001764,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.234596083057113}
{"item_id":"senseval2.d001.s021.t003","gold_sense_keys":["become%2:30:00::"],"candidates":[{"index":1,"sense_key":"become%2:30:00::","synset_id":"become.v.01"},{"index":2,"sense_key":"become%2:42:01::","synset_id":"become.v.02"},{"index":3,"sense_key":"become%2:42:00::","synset_id":"become.v.03"},{"index":4,"sense_key":"become%2:42:03::","synset_id":"become.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"become%2:30:00::","call_ids":["senseval2.d001.s021.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"become%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":873,"cached_input_tokens":0,"output_tokens":123,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0029760000000000003,"input_uncached_usd":0.001746,"input_cached_usd":0.0,"output_usd":0.0012300000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.4679993339814246}
{"item_id":"senseval2.d001.s021.t004","gold_sense_keys":["possible%3:00:00::"],"candidates":[{"index":1,"sense_key":"possible%3:00:00::","synset_id":"possible.a.01"},{"index":2,"sense_key":"possible%3:00:04::","synset_id":"potential.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"possible%3:00:00::","call_ids":["senseval2.d001.s021.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"possible%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":754,"cached_input_tokens":0,"output_tokens":81,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002318,"input_uncached_usd":0.001508,"input_cached_usd":0.0,"output_usd":0.0008100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5446875829948112}
{"item_id":"senseval2.d001.s021.t005","gold_sense_keys":["last%5:00:00:past:00"],"candidates":[{"index":1,"sense_key":"last%5:00:00:past:00","synset_id":"last.s.01"},{"index":2,"sense_key":"last%3:00:00::","synset_id":"last.a.02"},{"index":3,"sense_key":"last%5:00:00:closing:00","synset_id":"concluding.s.01"},{"index":4,"sense_key":"last%5:00:00:unlikely:00","synset_id":"last.s.03"},{"index":5,"sense_key":"last%5:00:00:dying:00","synset_id":"last.s.04"},{"index":6,"sense_key":"last%5:00:00:ultimate:00","synset_id":"final.s.02"},{"index":7,"sense_key":"last%5:00:00:high:02","synset_id":"last.s.06"},{"index":8,"sense_key":"last%5:00:00:unalterable:00","synset_id":"final.s.03"},{"index":9,"sense_key":"last%5:00:00:worst:00","synset_id":"last.s.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"last%5:00:00:past:00","call_ids":["senseval2.d001.s021.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"last%5:00:00:past:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1197,"cached_input_tokens":0,"output_tokens":45,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0028439999999999997,"input_uncached_usd":0.002394,"input_cached_usd":0.0,"output_usd":0.00045000000000000004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4550393330864608}
{"item_id":"senseval2.d001.s021.t007","gold_sense_keys":["find_out%2:32:00::"],"candidates":[{"index":1,"sense_key":"find_out%2:32:00::","synset_id":"determine.v.01"},{"index":2,"sense_key":"find_out%2:31:00::","synset_id":"learn.v.02"},{"index":3,"sense_key":"find_out%2:32:01::","synset_id":"determine.v.08"},{"index":4,"sense_key":"find_out%2:31:01::","synset_id":"find_out.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"find_out%2:32:01::","call_ids":["senseval2.d001.s021.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"find_out%2:32:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1000,"cached_input_tokens":0,"output_tokens":117,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00317,"input_uncached_usd":0.002,"input_cached_usd":0.0,"output_usd":0.00117,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0863177499268204}
{"item_id":"senseval2.d001.s021.t008","gold_sense_keys":["threat%1:26:00::"],"candidates":[{"index":1,"sense_key":"threat%1:26:00::","synset_id":"menace.n.01"},{"index":2,"sense_key":"threat%1:10:00::","synset_id":"threat.n.02"},{"index":3,"sense_key":"threat%1:10:01::","synset_id":"threat.n.03"},{"index":4,"sense_key":"threat%1:18:00::","synset_id":"terror.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"threat%1:26:00::","call_ids":["senseval2.d001.s021.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"threat%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":871,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001862,"input_uncached_usd":0.0017419999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0235324170207605}
{"item_id":"senseval2.d001.s021.t009","gold_sense_keys":["baby%1:18:00::"],"candidates":[{"index":1,"sense_key":"baby%1:18:00::","synset_id":"baby.n.01"},{"index":2,"sense_key":"baby%1:18:01::","synset_id":"baby.n.02"},{"index":3,"sense_key":"baby%1:18:03::","synset_id":"child.n.03"},{"index":4,"sense_key":"baby%1:18:04::","synset_id":"baby.n.04"},{"index":5,"sense_key":"baby%1:18:02::","synset_id":"baby.n.05"},{"index":6,"sense_key":"baby%1:05:00::","synset_id":"baby.n.06"},{"index":7,"sense_key":"baby%1:04:00::","synset_id":"baby.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"baby%1:18:00::","call_ids":["senseval2.d001.s021.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"baby%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1052,"cached_input_tokens":0,"output_tokens":209,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004194,"input_uncached_usd":0.002104,"input_cached_usd":0.0,"output_usd":0.0020900000000000003,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.6833149580052122}
{"item_id":"senseval2.d001.s021.t010","gold_sense_keys":["face%2:32:00::"],"candidates":[{"index":1,"sense_key":"face%2:32:00::","synset_id":"confront.v.02"},{"index":2,"sense_key":"face%2:33:00::","synset_id":"confront.v.01"},{"index":3,"sense_key":"face%2:42:00::","synset_id":"front.v.01"},{"index":4,"sense_key":"face%2:42:02::","synset_id":"face.v.04"},{"index":5,"sense_key":"face%2:38:00::","synset_id":"face.v.05"},{"index":6,"sense_key":"face%2:32:03::","synset_id":"confront.v.03"},{"index":7,"sense_key":"face%2:39:04::","synset_id":"face.v.07"},{"index":8,"sense_key":"face%2:35:00::","synset_id":"face.v.08"},{"index":9,"sense_key":"face%2:35:01::","synset_id":"face.v.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"face%2:32:00::","call_ids":["senseval2.d001.s021.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"face%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1174,"cached_input_tokens":0,"output_tokens":184,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004188,"input_uncached_usd":0.002348,"input_cached_usd":0.0,"output_usd":0.00184,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.5516880420036614}
{"item_id":"senseval2.d001.s022.t001","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":2,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":3,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":4,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":5,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":6,"sense_key":"use%2:41:03::","synset_id":"use.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%2:34:01::","call_ids":["senseval2.d001.s022.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":988,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0020959999999999998,"input_uncached_usd":0.001976,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2609098750399426}
{"item_id":"senseval2.d001.s022.t002","gold_sense_keys":["new%3:00:00::","new%5:00:00:original:00","new%5:00:00:other:00","new%5:00:00:unprecedented:00"],"candidates":[{"index":1,"sense_key":"new%3:00:00::","synset_id":"new.a.01"},{"index":2,"sense_key":"new%5:00:00:original:00","synset_id":"fresh.s.03"},{"index":3,"sense_key":"new%5:00:00:inexperienced:00","synset_id":"raw.s.11"},{"index":4,"sense_key":"new%5:00:00:unprecedented:00","synset_id":"new.s.03"},{"index":5,"sense_key":"new%5:00:00:other:00","synset_id":"new.s.04"},{"index":6,"sense_key":"new%3:00:09::","synset_id":"new.a.06"},{"index":7,"sense_key":"new%5:00:02:original:00","synset_id":"newfangled.s.01"},{"index":8,"sense_key":"new%5:00:02:late:01","synset_id":"new.s.06"},{"index":9,"sense_key":"new%5:00:01:late:01","synset_id":"modern.s.04"},{"index":10,"sense_key":"new%5:00:00:early:02","synset_id":"new.s.08"},{"index":11,"sense_key":"new%5:00:00:unaccustomed:00","synset_id":"new.s.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"new%3:00:00::","call_ids":["senseval2.d001.s022.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"new%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1292,"cached_input_tokens":0,"output_tokens":106,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003644,"input_uncached_usd":0.002584,"input_cached_usd":0.0,"output_usd":0.0010600000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6368266249774024}
{"item_id":"senseval2.d001.s022.t003","gold_sense_keys":["genetic%3:01:00::","genetic%3:01:02::"],"candidates":[{"index":1,"sense_key":"genetic%5:00:00:inheritable:00","synset_id":"familial.s.01"},{"index":2,"sense_key":"genetic%3:01:02::","synset_id":"genic.a.01"},{"index":3,"sense_key":"genetic%3:01:01::","synset_id":"genetic.a.03"},{"index":4,"sense_key":"genetic%3:01:00::","synset_id":"genetic.a.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"genetic%3:01:02::","call_ids":["senseval2.d001.s022.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"genetic%3:01:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":859,"cached_input_tokens":0,"output_tokens":130,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003018,"input_uncached_usd":0.001718,"input_cached_usd":0.0,"output_usd":0.0013000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1219025829341263}
{"item_id":"senseval2.d001.s022.t004","gold_sense_keys":["show%2:31:00::","show%2:32:00::"],"candidates":[{"index":1,"sense_key":"show%2:39:02::","synset_id":"show.v.01"},{"index":2,"sense_key":"show%2:31:00::","synset_id":"prove.v.02"},{"index":3,"sense_key":"show%2:32:00::","synset_id":"testify.v.02"},{"index":4,"sense_key":"show%2:39:00::","synset_id":"show.v.04"},{"index":5,"sense_key":"show%2:36:00::","synset_id":"picture.v.02"},{"index":6,"sense_key":"show%2:32:02::","synset_id":"express.v.01"},{"index":7,"sense_key":"show%2:32:01::","synset_id":"indicate.v.02"},{"index":8,"sense_key":"show%2:39:01::","synset_id":"show.v.08"},{"index":9,"sense_key":"show%2:32:04::","synset_id":"read.v.08"},{"index":10,"sense_key":"show%2:32:03::","synset_id":"show.v.10"},{"index":11,"sense_key":"show%2:38:00::","synset_id":"usher.v.01"},{"index":12,"sense_key":"show%2:33:00::","synset_id":"show.v.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"show%2:32:00::","call_ids":["senseval2.d001.s022.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"show%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1416,"cached_input_tokens":0,"output_tokens":80,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003632,"input_uncached_usd":0.002832,"input_cached_usd":0.0,"output_usd":0.0008,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6118872499791905}
{"item_id":"senseval2.d001.s022.t005","gold_sense_keys":["little%5:00:00:emotional:00","little%5:00:00:young:00"],"candidates":[{"index":1,"sense_key":"little%3:00:01::","synset_id":"small.a.01"},{"index":2,"sense_key":"little%3:00:03::","synset_id":"little.a.02"},{"index":3,"sense_key":"little%5:00:00:young:00","synset_id":"little.s.01"},{"index":4,"sense_key":"little%5:00:00:unimportant:00","synset_id":"fiddling.s.01"},{"index":5,"sense_key":"little%5:00:00:soft:04","synset_id":"little.s.03"},{"index":6,"sense_key":"little%3:00:00::","synset_id":"short.a.03"},{"index":7,"sense_key":"little%5:00:00:lowercase:00","synset_id":"little.s.04"},{"index":8,"sense_key":"little%5:00:00:emotional:00","synset_id":"little.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"little%5:00:00:young:00","call_ids":["senseval2.d001.s022.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"little%5:00:00:young:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1118,"cached_input_tokens":0,"output_tokens":89,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003126,"input_uncached_usd":0.0022359999999999997,"input_cached_usd":0.0,"output_usd":0.0008900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8041715419385582}
{"item_id":"senseval2.d001.s022.t006","gold_sense_keys":["inherit%2:40:01::"],"candidates":[{"index":1,"sense_key":"inherit%2:40:00::","synset_id":"inherit.v.01"},{"index":2,"sense_key":"inherit%2:40:02::","synset_id":"inherit.v.02"},{"index":3,"sense_key":"inherit%2:40:01::","synset_id":"inherit.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"inherit%2:40:01::","call_ids":["senseval2.d001.s022.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"inherit%2:40:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":757,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001634,"input_uncached_usd":0.001514,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9818967909086496}
{"item_id":"senseval2.d001.s022.t007","gold_sense_keys":["damaged%3:00:00::"],"candidates":[{"index":1,"sense_key":"damaged%3:00:00::","synset_id":"damaged.a.01"},{"index":2,"sense_key":"damaged%5:00:00:disreputable:00","synset_id":"discredited.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"damaged%3:00:00::","call_ids":["senseval2.d001.s022.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"damaged%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":738,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001596,"input_uncached_usd":0.0014759999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0209672909695655}
{"item_id":"senseval2.d001.s022.t011","gold_sense_keys":["face%2:32:00::"],"candidates":[{"index":1,"sense_key":"face%2:32:00::","synset_id":"confront.v.02"},{"index":2,"sense_key":"face%2:33:00::","synset_id":"confront.v.01"},{"index":3,"sense_key":"face%2:42:00::","synset_id":"front.v.01"},{"index":4,"sense_key":"face%2:42:02::","synset_id":"face.v.04"},{"index":5,"sense_key":"face%2:38:00::","synset_id":"face.v.05"},{"index":6,"sense_key":"face%2:32:03::","synset_id":"confront.v.03"},{"index":7,"sense_key":"face%2:39:04::","synset_id":"face.v.07"},{"index":8,"sense_key":"face%2:35:00::","synset_id":"face.v.08"},{"index":9,"sense_key":"face%2:35:01::","synset_id":"face.v.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"face%2:32:00::","call_ids":["senseval2.d001.s022.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"face%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1131,"cached_input_tokens":0,"output_tokens":163,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0038919999999999996,"input_uncached_usd":0.0022619999999999997,"input_cached_usd":0.0,"output_usd":0.0016300000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9690839999821037}
{"item_id":"senseval2.d001.s022.t012","gold_sense_keys":["more%3:00:01::"],"candidates":[{"index":1,"sense_key":"more%3:00:01::","synset_id":"more.a.01"},{"index":2,"sense_key":"more%3:00:02::","synset_id":"more.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"more%3:00:01::","call_ids":["senseval2.d001.s022.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"more%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":742,"cached_input_tokens":0,"output_tokens":75,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002234,"input_uncached_usd":0.0014839999999999999,"input_cached_usd":0.0,"output_usd":0.00075,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.296198416966945}
{"item_id":"senseval2.d001.s022.t013","gold_sense_keys":["risk%1:26:00::"],"candidates":[{"index":1,"sense_key":"risk%1:26:00::","synset_id":"hazard.n.01"},{"index":2,"sense_key":"risk%1:04:00::","synset_id":"risk.n.02"},{"index":3,"sense_key":"risk%1:07:02::","synset_id":"risk.n.03"},{"index":4,"sense_key":"risk%1:07:01::","synset_id":"risk.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"risk%1:26:00::","call_ids":["senseval2.d001.s022.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"risk%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":878,"cached_input_tokens":0,"output_tokens":122,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0029760000000000003,"input_uncached_usd":0.001756,"input_cached_usd":0.0,"output_usd":0.0012200000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6983113340102136}
{"item_id":"senseval2.d001.s022.t014","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":2,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":3,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"other%3:00:00::","call_ids":["senseval2.d001.s022.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":836,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001792,"input_uncached_usd":0.0016719999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.741742082987912}
{"item_id":"senseval2.d001.s022.t015","gold_sense_keys":["child%1:18:00::"],"candidates":[{"index":1,"sense_key":"child%1:18:00::","synset_id":"child.n.01"},{"index":2,"sense_key":"child%1:18:01::","synset_id":"child.n.02"},{"index":3,"sense_key":"child%1:18:06::","synset_id":"child.n.03"},{"index":4,"sense_key":"child%1:18:07::","synset_id":"child.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"child%1:18:00::","call_ids":["senseval2.d001.s022.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"child%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":839,"cached_input_tokens":0,"output_tokens":71,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002388,"input_uncached_usd":0.001678,"input_cached_usd":0.0,"output_usd":0.00071,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.6996810000855476}
{"item_id":"senseval2.d001.s022.t016","gold_sense_keys":["develop%2:29:00::"],"candidates":[{"index":1,"sense_key":"develop%2:36:01::","synset_id":"develop.v.01"},{"index":2,"sense_key":"develop%2:36:00::","synset_id":"evolve.v.01"},{"index":3,"sense_key":"develop%2:30:06::","synset_id":"develop.v.03"},{"index":4,"sense_key":"develop%2:29:00::","synset_id":"grow.v.08"},{"index":5,"sense_key":"develop%2:42:00::","synset_id":"originate.v.01"},{"index":6,"sense_key":"develop%2:30:04::","synset_id":"build_up.v.05"},{"index":7,"sense_key":"develop%2:32:00::","synset_id":"explicate.v.02"},{"index":8,"sense_key":"develop%2:31:00::","synset_id":"train.v.01"},{"index":9,"sense_key":"develop%2:30:05::","synset_id":"develop.v.09"},{"index":10,"sense_key":"develop%2:30:00::","synset_id":"develop.v.10"},{"index":11,"sense_key":"develop%2:30:09::","synset_id":"modernize.v.02"},{"index":12,"sense_key":"develop%2:30:01::","synset_id":"develop.v.12"},{"index":13,"sense_key":"develop%2:36:09::","synset_id":"develop.v.13"},{"index":14,"sense_key":"develop%2:30:10::","synset_id":"develop.v.14"},{"index":15,"sense_key":"develop%2:39:00::","synset_id":"develop.v.15"},{"index":16,"sense_key":"develop%2:35:04::","synset_id":"develop.v.16"},{"index":17,"sense_key":"develop%2:33:03::","synset_id":"develop.v.17"},{"index":18,"sense_key":"develop%2:33:02::","synset_id":"develop.v.18"},{"index":19,"sense_key":"develop%2:30:13::","synset_id":"develop.v.19"},{"index":20,"sense_key":"develop%2:30:03::","synset_id":"break.v.53"},{"index":21,"sense_key":"develop%2:30:12::","synset_id":"develop.v.21"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"develop%2:29:00::","call_ids":["senseval2.d001.s022.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"develop%2:29:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1982,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004594,"input_uncached_usd":0.003964,"input_cached_usd":0.0,"output_usd":0.00063,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":4.433599791955203}
{"item_id":"senseval2.d001.s022.t018","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s022.t018__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":920,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00196,"input_uncached_usd":0.0018399999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.096251125098206}
{"item_id":"senseval2.d001.s023.t000","gold_sense_keys":["make%2:30:10::"],"candidates":[{"index":1,"sense_key":"make%2:41:00::","synset_id":"make.v.01"},{"index":2,"sense_key":"make%2:30:00::","synset_id":"make.v.02"},{"index":3,"sense_key":"make%2:36:00::","synset_id":"make.v.03"},{"index":4,"sense_key":"make%2:32:00::","synset_id":"induce.v.02"},{"index":5,"sense_key":"make%2:36:08::","synset_id":"cause.v.01"},{"index":6,"sense_key":"make%2:36:01::","synset_id":"produce.v.02"},{"index":7,"sense_key":"make%2:31:00::","synset_id":"draw.v.04"},{"index":8,"sense_key":"make%2:36:15::","synset_id":"make.v.08"},{"index":9,"sense_key":"make%2:36:09::","synset_id":"create.v.05"},{"index":10,"sense_key":"make%2:40:01::","synset_id":"gain.v.08"},{"index":11,"sense_key":"make%2:36:05::","synset_id":"do.v.08"},{"index":12,"sense_key":"make%2:42:05::","synset_id":"form.v.02"},{"index":13,"sense_key":"make%2:38:02::","synset_id":"reach.v.07"},{"index":14,"sense_key":"make%2:42:00::","synset_id":"make.v.14"},{"index":15,"sense_key":"make%2:36:11::","synset_id":"make.v.15"},{"index":16,"sense_key":"make%2:36:12::","synset_id":"make.v.16"},{"index":17,"sense_key":"make%2:36:13::","synset_id":"construct.v.01"},{"index":18,"sense_key":"make%2:30:15::","synset_id":"make.v.18"},{"index":19,"sense_key":"make%2:29:08::","synset_id":"make.v.19"},{"index":20,"sense_key":"make%2:41:03::","synset_id":"name.v.03"},{"index":21,"sense_key":"make%2:40:02::","synset_id":"have.v.17"},{"index":22,"sense_key":"make%2:38:00::","synset_id":"reach.v.01"},{"index":23,"sense_key":"make%2:31:13::","synset_id":"lay_down.v.01"},{"index":24,"sense_key":"make%2:41:13::","synset_id":"make.v.24"},{"index":25,"sense_key":"make%2:41:01::","synset_id":"make.v.25"},{"index":26,"sense_key":"make%2:36:04::","synset_id":"hold.v.03"},{"index":27,"sense_key":"make%2:30:02::","synset_id":"make.v.27"},{"index":28,"sense_key":"make%2:38:05::","synset_id":"take.v.27"},{"index":29,"sense_key":"make%2:29:00::","synset_id":"stool.v.04"},{"index":30,"sense_key":"make%2:42:09::","synset_id":"make.v.30"},{"index":31,"sense_key":"make%2:42:13::","synset_id":"make.v.31"},{"index":32,"sense_key":"make%2:42:12::","synset_id":"make.v.32"},{"index":33,"sense_key":"make%2:42:11::","synset_id":"make.v.33"},{"index":34,"sense_key":"make%2:42:15::","synset_id":"make.v.34"},{"index":35,"sense_key":"make%2:39:12::","synset_id":"make.v.35"},{"index":36,"sense_key":"make%2:38:10::","synset_id":"make.v.36"},{"index":37,"sense_key":"make%2:38:15::","synset_id":"make.v.37"},{"index":38,"sense_key":"make%2:36:10::","synset_id":"make.v.38"},{"index":39,"sense_key":"make%2:36:07::","synset_id":"cook.v.02"},{"index":40,"sense_key":"make%2:35:13::","synset_id":"seduce.v.01"},{"index":41,"sense_key":"make%2:32:14::","synset_id":"make.v.41"},{"index":42,"sense_key":"make%2:32:09::","synset_id":"make.v.42"},{"index":43,"sense_key":"make%2:31:10::","synset_id":"make.v.43"},{"index":44,"sense_key":"make%2:31:12::","synset_id":"make.v.44"},{"index":45,"sense_key":"make%2:30:10::","synset_id":"make.v.45"},{"index":46,"sense_key":"make%2:30:14::","synset_id":"make.v.46"},{"index":47,"sense_key":"make%2:30:12::","synset_id":"make.v.47"},{"index":48,"sense_key":"make%2:29:06::","synset_id":"make.v.48"},{"index":49,"sense_key":"make%2:29:01::","synset_id":"make.v.49"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":45,"chosen_sense_key":"make%2:30:10::","call_ids":["senseval2.d001.s023.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":45,"predicted_sense_key":"make%2:30:10::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":3238,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.007136,"input_uncached_usd":0.006476,"input_cached_usd":0.0,"output_usd":0.0006600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3806906669633463}
{"item_id":"senseval2.d001.s023.t002","gold_sense_keys":["say%2:32:00::","say%2:32:15::"],"candidates":[{"index":1,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":6,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":9,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":10,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":11,"sense_key":"say%2:32:09::","synset_id":"say.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"say%2:32:00::","call_ids":["senseval2.d001.s023.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"say%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1230,"cached_input_tokens":0,"output_tokens":146,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00392,"input_uncached_usd":0.00246,"input_cached_usd":0.0,"output_usd":0.0014600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0989270000718534}
{"item_id":"senseval2.d001.s024.t001","gold_sense_keys":["first%3:00:00::"],"candidates":[{"index":1,"sense_key":"first%3:00:00::","synset_id":"first.a.01"},{"index":2,"sense_key":"first%5:00:00:ordinal:00","synset_id":"first.s.01"},{"index":3,"sense_key":"first%5:00:02:opening:00","synset_id":"inaugural.s.01"},{"index":4,"sense_key":"first%5:00:01:opening:00","synset_id":"beginning.s.01"},{"index":5,"sense_key":"first%5:00:00:best:00","synset_id":"first.s.04"},{"index":6,"sense_key":"first%3:00:03::","synset_id":"first.a.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"first%3:00:00::","call_ids":["senseval2.d001.s024.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"first%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":972,"cached_input_tokens":0,"output_tokens":132,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0032640000000000004,"input_uncached_usd":0.001944,"input_cached_usd":0.0,"output_usd":0.0013200000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.837129333987832}
{"item_id":"senseval2.d001.s024.t004","gold_sense_keys":["individual%1:03:00::"],"candidates":[{"index":1,"sense_key":"individual%1:03:00::","synset_id":"person.n.01"},{"index":2,"sense_key":"individual%1:18:00::","synset_id":"individual.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"individual%1:03:00::","call_ids":["senseval2.d001.s024.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"individual%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":696,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001512,"input_uncached_usd":0.001392,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1460237500723451}
{"item_id":"senseval2.d001.s024.t005","gold_sense_keys":["expect%2:31:00::"],"candidates":[{"index":1,"sense_key":"expect%2:31:00::","synset_id":"expect.v.01"},{"index":2,"sense_key":"expect%2:32:00::","synset_id":"ask.v.04"},{"index":3,"sense_key":"expect%2:31:01::","synset_id":"expect.v.03"},{"index":4,"sense_key":"expect%2:31:05::","synset_id":"expect.v.04"},{"index":5,"sense_key":"expect%2:29:02::","synset_id":"expect.v.05"},{"index":6,"sense_key":"expect%2:29:00::","synset_id":"have_a_bun_in_the_oven.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"expect%2:31:00::","call_ids":["senseval2.d001.s024.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"expect%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":989,"cached_input_tokens":0,"output_tokens":89,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002868,"input_uncached_usd":0.0019779999999999997,"input_cached_usd":0.0,"output_usd":0.0008900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5891274169553071}
{"item_id":"senseval2.d001.s024.t006","gold_sense_keys":["develop%2:29:00::"],"candidates":[{"index":1,"sense_key":"develop%2:36:01::","synset_id":"develop.v.01"},{"index":2,"sense_key":"develop%2:36:00::","synset_id":"evolve.v.01"},{"index":3,"sense_key":"develop%2:30:06::","synset_id":"develop.v.03"},{"index":4,"sense_key":"develop%2:29:00::","synset_id":"grow.v.08"},{"index":5,"sense_key":"develop%2:42:00::","synset_id":"originate.v.01"},{"index":6,"sense_key":"develop%2:30:04::","synset_id":"build_up.v.05"},{"index":7,"sense_key":"develop%2:32:00::","synset_id":"explicate.v.02"},{"index":8,"sense_key":"develop%2:31:00::","synset_id":"train.v.01"},{"index":9,"sense_key":"develop%2:30:05::","synset_id":"develop.v.09"},{"index":10,"sense_key":"develop%2:30:00::","synset_id":"develop.v.10"},{"index":11,"sense_key":"develop%2:30:09::","synset_id":"modernize.v.02"},{"index":12,"sense_key":"develop%2:30:01::","synset_id":"develop.v.12"},{"index":13,"sense_key":"develop%2:36:09::","synset_id":"develop.v.13"},{"index":14,"sense_key":"develop%2:30:10::","synset_id":"develop.v.14"},{"index":15,"sense_key":"develop%2:39:00::","synset_id":"develop.v.15"},{"index":16,"sense_key":"develop%2:35:04::","synset_id":"develop.v.16"},{"index":17,"sense_key":"develop%2:33:03::","synset_id":"develop.v.17"},{"index":18,"sense_key":"develop%2:33:02::","synset_id":"develop.v.18"},{"index":19,"sense_key":"develop%2:30:13::","synset_id":"develop.v.19"},{"index":20,"sense_key":"develop%2:30:03::","synset_id":"break.v.53"},{"index":21,"sense_key":"develop%2:30:12::","synset_id":"develop.v.21"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"develop%2:29:00::","call_ids":["senseval2.d001.s024.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"develop%2:29:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1990,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0046,"input_uncached_usd":0.00398,"input_cached_usd":0.0,"output_usd":0.00062,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3020793750183657}
{"item_id":"senseval2.d001.s024.t007","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s024.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":928,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001976,"input_uncached_usd":0.001856,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1933704579714686}
{"item_id":"senseval2.d001.s025.t000","gold_sense_keys":["equally%4:02:01::"],"candidates":[{"index":1,"sense_key":"equally%4:02:01::","synset_id":"equally.r.01"},{"index":2,"sense_key":"equally%4:02:02::","synset_id":"evenly.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"equally%4:02:01::","call_ids":["senseval2.d001.s025.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"equally%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":748,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001616,"input_uncached_usd":0.001496,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0427863750373945}
{"item_id":"senseval2.d001.s025.t001","gold_sense_keys":["important%3:00:00::","important%3:00:02::","important%3:00:04::"],"candidates":[{"index":1,"sense_key":"important%3:00:00::","synset_id":"important.a.01"},{"index":2,"sense_key":"important%3:00:04::","synset_id":"significant.a.01"},{"index":3,"sense_key":"important%3:00:02::","synset_id":"crucial.a.01"},{"index":4,"sense_key":"important%5:00:00:influential:00","synset_id":"authoritative.s.01"},{"index":5,"sense_key":"important%5:00:00:immodest:02","synset_id":"important.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"important%3:00:00::","call_ids":["senseval2.d001.s025.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"important%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":903,"cached_input_tokens":0,"output_tokens":83,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002636,"input_uncached_usd":0.0018059999999999999,"input_cached_usd":0.0,"output_usd":0.0008300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8987253330415115}
{"item_id":"senseval2.d001.s025.t003","gold_sense_keys":["discovery%1:04:00::"],"candidates":[{"index":1,"sense_key":"discovery%1:04:00::","synset_id":"discovery.n.01"},{"index":2,"sense_key":"discovery%1:10:00::","synset_id":"discovery.n.02"},{"index":3,"sense_key":"discovery%1:09:00::","synset_id":"discovery.n.03"},{"index":4,"sense_key":"discovery%1:10:01::","synset_id":"discovery.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"discovery%1:04:00::","call_ids":["senseval2.d001.s025.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"discovery%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":785,"cached_input_tokens":0,"output_tokens":138,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0029500000000000004,"input_uncached_usd":0.00157,"input_cached_usd":0.0,"output_usd":0.0013800000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8537281670141965}
{"item_id":"senseval2.d001.s025.t005","gold_sense_keys":["control%2:41:01::"],"candidates":[{"index":1,"sense_key":"control%2:41:00::","synset_id":"control.v.01"},{"index":2,"sense_key":"control%2:41:01::","synset_id":"control.v.02"},{"index":3,"sense_key":"control%2:35:00::","synset_id":"operate.v.03"},{"index":4,"sense_key":"control%2:37:06::","synset_id":"manipulate.v.05"},{"index":5,"sense_key":"control%2:41:02::","synset_id":"control.v.05"},{"index":6,"sense_key":"control%2:31:11::","synset_id":"control.v.06"},{"index":7,"sense_key":"control%2:31:01::","synset_id":"see.v.10"},{"index":8,"sense_key":"control%2:31:00::","synset_id":"master.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"control%2:41:00::","call_ids":["senseval2.d001.s025.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"control%2:41:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1158,"cached_input_tokens":0,"output_tokens":148,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003796,"input_uncached_usd":0.002316,"input_cached_usd":0.0,"output_usd":0.0014800000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1103869999060407}
{"item_id":"senseval2.d001.s025.t007","gold_sense_keys":["cell%1:03:00::"],"candidates":[{"index":1,"sense_key":"cell%1:06:03::","synset_id":"cell.n.01"},{"index":2,"sense_key":"cell%1:03:00::","synset_id":"cell.n.02"},{"index":3,"sense_key":"cell%1:06:00::","synset_id":"cell.n.03"},{"index":4,"sense_key":"cell%1:14:00::","synset_id":"cell.n.04"},{"index":5,"sense_key":"cell%1:06:04::","synset_id":"cellular_telephone.n.01"},{"index":6,"sense_key":"cell%1:06:02::","synset_id":"cell.n.06"},{"index":7,"sense_key":"cell%1:06:01::","synset_id":"cell.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cell%1:03:00::","call_ids":["senseval2.d001.s025.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cell%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1005,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00213,"input_uncached_usd":0.00201,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4416397080058232}
{"item_id":"senseval2.d001.s025.t008","gold_sense_keys":["growth%1:22:00::"],"candidates":[{"index":1,"sense_key":"growth%1:22:00::","synset_id":"growth.n.01"},{"index":2,"sense_key":"growth%1:22:01::","synset_id":"growth.n.02"},{"index":3,"sense_key":"growth%1:22:02::","synset_id":"increase.n.03"},{"index":4,"sense_key":"growth%1:14:00::","synset_id":"growth.n.04"},{"index":5,"sense_key":"growth%1:11:01::","synset_id":"emergence.n.01"},{"index":6,"sense_key":"growth%1:26:00::","synset_id":"growth.n.06"},{"index":7,"sense_key":"growth%1:17:00::","synset_id":"growth.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"growth%1:22:02::","call_ids":["senseval2.d001.s025.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"growth%1:22:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1036,"cached_input_tokens":0,"output_tokens":219,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004262,"input_uncached_usd":0.002072,"input_cached_usd":0.0,"output_usd":0.00219,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.793368541984819}
{"item_id":"senseval2.d001.s025.t009","gold_sense_keys":["make%2:36:12::"],"candidates":[{"index":1,"sense_key":"make%2:41:00::","synset_id":"make.v.01"},{"index":2,"sense_key":"make%2:30:00::","synset_id":"make.v.02"},{"index":3,"sense_key":"make%2:36:00::","synset_id":"make.v.03"},{"index":4,"sense_key":"make%2:32:00::","synset_id":"induce.v.02"},{"index":5,"sense_key":"make%2:36:08::","synset_id":"cause.v.01"},{"index":6,"sense_key":"make%2:36:01::","synset_id":"produce.v.02"},{"index":7,"sense_key":"make%2:31:00::","synset_id":"draw.v.04"},{"index":8,"sense_key":"make%2:36:15::","synset_id":"make.v.08"},{"index":9,"sense_key":"make%2:36:09::","synset_id":"create.v.05"},{"index":10,"sense_key":"make%2:40:01::","synset_id":"gain.v.08"},{"index":11,"sense_key":"make%2:36:05::","synset_id":"do.v.08"},{"index":12,"sense_key":"make%2:42:05::","synset_id":"form.v.02"},{"index":13,"sense_key":"make%2:38:02::","synset_id":"reach.v.07"},{"index":14,"sense_key":"make%2:42:00::","synset_id":"make.v.14"},{"index":15,"sense_key":"make%2:36:11::","synset_id":"make.v.15"},{"index":16,"sense_key":"make%2:36:12::","synset_id":"make.v.16"},{"index":17,"sense_key":"make%2:36:13::","synset_id":"construct.v.01"},{"index":18,"sense_key":"make%2:30:15::","synset_id":"make.v.18"},{"index":19,"sense_key":"make%2:29:08::","synset_id":"make.v.19"},{"index":20,"sense_key":"make%2:41:03::","synset_id":"name.v.03"},{"index":21,"sense_key":"make%2:40:02::","synset_id":"have.v.17"},{"index":22,"sense_key":"make%2:38:00::","synset_id":"reach.v.01"},{"index":23,"sense_key":"make%2:31:13::","synset_id":"lay_down.v.01"},{"index":24,"sense_key":"make%2:41:13::","synset_id":"make.v.24"},{"index":25,"sense_key":"make%2:41:01::","synset_id":"make.v.25"},{"index":26,"sense_key":"make%2:36:04::","synset_id":"hold.v.03"},{"index":27,"sense_key":"make%2:30:02::","synset_id":"make.v.27"},{"index":28,"sense_key":"make%2:38:05::","synset_id":"take.v.27"},{"index":29,"sense_key":"make%2:29:00::","synset_id":"stool.v.04"},{"index":30,"sense_key":"make%2:42:09::","synset_id":"make.v.30"},{"index":31,"sense_key":"make%2:42:13::","synset_id":"make.v.31"},{"index":32,"sense_key":"make%2:42:12::","synset_id":"make.v.32"},{"index":33,"sense_key":"make%2:42:11::","synset_id":"make.v.33"},{"index":34,"sense_key":"make%2:42:15::","synset_id":"make.v.34"},{"index":35,"sense_key":"make%2:39:12::","synset_id":"make.v.35"},{"index":36,"sense_key":"make%2:38:10::","synset_id":"make.v.36"},{"index":37,"sense_key":"make%2:38:15::","synset_id":"make.v.37"},{"index":38,"sense_key":"make%2:36:10::","synset_id":"make.v.38"},{"index":39,"sense_key":"make%2:36:07::","synset_id":"cook.v.02"},{"index":40,"sense_key":"make%2:35:13::","synset_id":"seduce.v.01"},{"index":41,"sense_key":"make%2:32:14::","synset_id":"make.v.41"},{"index":42,"sense_key":"make%2:32:09::","synset_id":"make.v.42"},{"index":43,"sense_key":"make%2:31:10::","synset_id":"make.v.43"},{"index":44,"sense_key":"make%2:31:12::","synset_id":"make.v.44"},{"index":45,"sense_key":"make%2:30:10::","synset_id":"make.v.45"},{"index":46,"sense_key":"make%2:30:14::","synset_id":"make.v.46"},{"index":47,"sense_key":"make%2:30:12::","synset_id":"make.v.47"},{"index":48,"sense_key":"make%2:29:06::","synset_id":"make.v.48"},{"index":49,"sense_key":"make%2:29:01::","synset_id":"make.v.49"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":16,"chosen_sense_key":"make%2:36:12::","call_ids":["senseval2.d001.s025.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":16,"predicted_sense_key":"make%2:36:12::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":3261,"cached_input_tokens":0,"output_tokens":108,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.007602,"input_uncached_usd":0.006522,"input_cached_usd":0.0,"output_usd":0.00108,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1803624579915777}
{"item_id":"senseval2.d001.s025.t010","gold_sense_keys":["doctor%1:18:00::"],"candidates":[{"index":1,"sense_key":"doctor%1:18:00::","synset_id":"doctor.n.01"},{"index":2,"sense_key":"doctor%1:18:02::","synset_id":"doctor_of_the_church.n.01"},{"index":3,"sense_key":"doctor%1:04:00::","synset_id":"doctor.n.03"},{"index":4,"sense_key":"doctor%1:18:01::","synset_id":"doctor.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"doctor%1:18:00::","call_ids":["senseval2.d001.s025.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"doctor%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":937,"cached_input_tokens":0,"output_tokens":85,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002724,"input_uncached_usd":0.001874,"input_cached_usd":0.0,"output_usd":0.0008500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.218782459036447}
{"item_id":"senseval2.d001.s025.t011","gold_sense_keys":["name%2:32:03::"],"candidates":[{"index":1,"sense_key":"name%2:32:03::","synset_id":"name.v.01"},{"index":2,"sense_key":"name%2:32:02::","synset_id":"name.v.02"},{"index":3,"sense_key":"name%2:41:01::","synset_id":"name.v.03"},{"index":4,"sense_key":"name%2:41:00::","synset_id":"appoint.v.01"},{"index":5,"sense_key":"name%2:32:07::","synset_id":"name.v.05"},{"index":6,"sense_key":"name%2:32:04::","synset_id":"mention.v.01"},{"index":7,"sense_key":"name%2:31:01::","synset_id":"identify.v.05"},{"index":8,"sense_key":"name%2:32:09::","synset_id":"list.v.01"},{"index":9,"sense_key":"name%2:31:00::","synset_id":"diagnose.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"name%2:32:03::","call_ids":["senseval2.d001.s025.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"name%2:32:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1152,"cached_input_tokens":0,"output_tokens":84,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003144,"input_uncached_usd":0.002304,"input_cached_usd":0.0,"output_usd":0.00084,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4413822919595987}
{"item_id":"senseval2.d001.s025.t012","gold_sense_keys":["open%2:30:00::"],"candidates":[{"index":1,"sense_key":"open%2:35:00::","synset_id":"open.v.01"},{"index":2,"sense_key":"open%2:41:00::","synset_id":"open.v.02"},{"index":3,"sense_key":"open%2:35:06::","synset_id":"open.v.03"},{"index":4,"sense_key":"open%2:41:01::","synset_id":"open.v.04"},{"index":5,"sense_key":"open%2:35:08::","synset_id":"unfold.v.04"},{"index":6,"sense_key":"open%2:30:00::","synset_id":"open.v.06"},{"index":7,"sense_key":"open%2:30:01::","synset_id":"open.v.07"},{"index":8,"sense_key":"open%2:42:00::","synset_id":"open.v.08"},{"index":9,"sense_key":"open%2:33:00::","synset_id":"open.v.09"},{"index":10,"sense_key":"open%2:40:04::","synset_id":"afford.v.04"},{"index":11,"sense_key":"open%2:39:10::","synset_id":"open.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"open%2:30:00::","call_ids":["senseval2.d001.s025.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"open%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1132,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002924,"input_uncached_usd":0.002264,"input_cached_usd":0.0,"output_usd":0.0006600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.776791333919391}
{"item_id":"senseval2.d001.s025.t014","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s025.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":913,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001946,"input_uncached_usd":0.001826,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9881014579441398}
{"item_id":"senseval2.d001.s025.t016","gold_sense_keys":["recent%5:00:00:past:00"],"candidates":[{"index":1,"sense_key":"recent%5:00:00:new:00","synset_id":"recent.s.01"},{"index":2,"sense_key":"recent%5:00:00:past:00","synset_id":"late.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"recent%5:00:00:past:00","call_ids":["senseval2.d001.s025.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"recent%5:00:00:past:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":700,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00207,"input_uncached_usd":0.0014,"input_cached_usd":0.0,"output_usd":0.00067,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6739668749505654}
{"item_id":"senseval2.d001.s025.t017","gold_sense_keys":["month%1:28:01::"],"candidates":[{"index":1,"sense_key":"month%1:28:01::","synset_id":"calendar_month.n.01"},{"index":2,"sense_key":"month%1:28:00::","synset_id":"month.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"month%1:28:00::","call_ids":["senseval2.d001.s025.t017__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"month%1:28:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":700,"cached_input_tokens":0,"output_tokens":75,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00215,"input_uncached_usd":0.0014,"input_cached_usd":0.0,"output_usd":0.00075,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6509103749413043}
{"item_id":"senseval2.d001.s025.t018","gold_sense_keys":["explode%2:30:02::"],"candidates":[{"index":1,"sense_key":"explode%2:30:01::","synset_id":"explode.v.01"},{"index":2,"sense_key":"explode%2:30:00::","synset_id":"explode.v.02"},{"index":3,"sense_key":"explode%2:31:00::","synset_id":"explode.v.03"},{"index":4,"sense_key":"explode%2:30:03::","synset_id":"explode.v.04"},{"index":5,"sense_key":"explode%2:35:00::","synset_id":"explode.v.05"},{"index":6,"sense_key":"explode%2:32:02::","synset_id":"explode.v.06"},{"index":7,"sense_key":"explode%2:32:00::","synset_id":"explode.v.07"},{"index":8,"sense_key":"explode%2:31:01::","synset_id":"explode.v.08"},{"index":9,"sense_key":"explode%2:30:04::","synset_id":"detonate.v.02"},{"index":10,"sense_key":"explode%2:30:02::","synset_id":"explode.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"explode%2:30:02::","call_ids":["senseval2.d001.s025.t018__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"explode%2:30:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1171,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002972,"input_uncached_usd":0.002342,"input_cached_usd":0.0,"output_usd":0.00063,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.092824917053804}
{"item_id":"senseval2.d001.s026.t000","gold_sense_keys":["turn_out%2:42:00::","turn_out%2:42:01::"],"candidates":[{"index":1,"sense_key":"turn_out%2:42:00::","synset_id":"prove.v.01"},{"index":2,"sense_key":"turn_out%2:42:01::","synset_id":"turn_out.v.02"},{"index":3,"sense_key":"turn_out%2:36:01::","synset_id":"turn_out.v.03"},{"index":4,"sense_key":"turn_out%2:42:02::","synset_id":"turn_out.v.04"},{"index":5,"sense_key":"turn_out%2:30:00::","synset_id":"turn_out.v.05"},{"index":6,"sense_key":"turn_out%2:36:00::","synset_id":"bear.v.05"},{"index":7,"sense_key":"turn_out%2:35:00::","synset_id":"eject.v.01"},{"index":8,"sense_key":"turn_out%2:41:03::","synset_id":"turn_out.v.08"},{"index":9,"sense_key":"turn_out%2:40:03::","synset_id":"turn_out.v.09"},{"index":10,"sense_key":"turn_out%2:38:00::","synset_id":"turn_out.v.10"},{"index":11,"sense_key":"turn_out%2:35:01::","synset_id":"switch_off.v.01"},{"index":12,"sense_key":"turn_out%2:29:00::","synset_id":"get_up.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"turn_out%2:42:01::","call_ids":["senseval2.d001.s026.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"turn_out%2:42:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1322,"cached_input_tokens":0,"output_tokens":80,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003444,"input_uncached_usd":0.002644,"input_cached_usd":0.0,"output_usd":0.0008,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.6713564170058817}
{"item_id":"senseval2.d001.s026.t001","gold_sense_keys":["study%2:31:02::"],"candidates":[{"index":1,"sense_key":"study%2:31:02::","synset_id":"analyze.v.01"},{"index":2,"sense_key":"study%2:31:03::","synset_id":"study.v.02"},{"index":3,"sense_key":"study%2:39:00::","synset_id":"study.v.03"},{"index":4,"sense_key":"study%2:31:00::","synset_id":"learn.v.04"},{"index":5,"sense_key":"study%2:31:01::","synset_id":"study.v.05"},{"index":6,"sense_key":"study%2:31:04::","synset_id":"study.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"study%2:31:02::","call_ids":["senseval2.d001.s026.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"study%2:31:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":955,"cached_input_tokens":0,"output_tokens":121,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00312,"input_uncached_usd":0.0019099999999999998,"input_cached_usd":0.0,"output_usd":0.0012100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9565025420160964}
{"item_id":"senseval2.d001.s026.t002","gold_sense_keys":["tragic%5:00:00:sad:00"],"candidates":[{"index":1,"sense_key":"tragic%5:00:00:sad:00","synset_id":"tragic.s.01"},{"index":2,"sense_key":"tragic%3:01:00::","synset_id":"tragic.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"tragic%5:00:00:sad:00","call_ids":["senseval2.d001.s026.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"tragic%5:00:00:sad:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":691,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001502,"input_uncached_usd":0.001382,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2641079999739304}
{"item_id":"senseval2.d001.s026.t003","gold_sense_keys":["uncommon%3:00:00::"],"candidates":[{"index":1,"sense_key":"uncommon%3:00:00::","synset_id":"uncommon.a.01"},{"index":2,"sense_key":"uncommon%5:00:00:extraordinary:00","synset_id":"rare.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"uncommon%3:00:00::","call_ids":["senseval2.d001.s026.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"uncommon%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":748,"cached_input_tokens":0,"output_tokens":98,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002476,"input_uncached_usd":0.001496,"input_cached_usd":0.0,"output_usd":0.0009800000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7247626670869067}
{"item_id":"senseval2.d001.s026.t005","gold_sense_keys":["make%2:30:00::"],"candidates":[{"index":1,"sense_key":"make%2:41:00::","synset_id":"make.v.01"},{"index":2,"sense_key":"make%2:30:00::","synset_id":"make.v.02"},{"index":3,"sense_key":"make%2:36:00::","synset_id":"make.v.03"},{"index":4,"sense_key":"make%2:32:00::","synset_id":"induce.v.02"},{"index":5,"sense_key":"make%2:36:08::","synset_id":"cause.v.01"},{"index":6,"sense_key":"make%2:36:01::","synset_id":"produce.v.02"},{"index":7,"sense_key":"make%2:31:00::","synset_id":"draw.v.04"},{"index":8,"sense_key":"make%2:36:15::","synset_id":"make.v.08"},{"index":9,"sense_key":"make%2:36:09::","synset_id":"create.v.05"},{"index":10,"sense_key":"make%2:40:01::","synset_id":"gain.v.08"},{"index":11,"sense_key":"make%2:36:05::","synset_id":"do.v.08"},{"index":12,"sense_key":"make%2:42:05::","synset_id":"form.v.02"},{"index":13,"sense_key":"make%2:38:02::","synset_id":"reach.v.07"},{"index":14,"sense_key":"make%2:42:00::","synset_id":"make.v.14"},{"index":15,"sense_key":"make%2:36:11::","synset_id":"make.v.15"},{"index":16,"sense_key":"make%2:36:12::","synset_id":"make.v.16"},{"index":17,"sense_key":"make%2:36:13::","synset_id":"construct.v.01"},{"index":18,"sense_key":"make%2:30:15::","synset_id":"make.v.18"},{"index":19,"sense_key":"make%2:29:08::","synset_id":"make.v.19"},{"index":20,"sense_key":"make%2:41:03::","synset_id":"name.v.03"},{"index":21,"sense_key":"make%2:40:02::","synset_id":"have.v.17"},{"index":22,"sense_key":"make%2:38:00::","synset_id":"reach.v.01"},{"index":23,"sense_key":"make%2:31:13::","synset_id":"lay_down.v.01"},{"index":24,"sense_key":"make%2:41:13::","synset_id":"make.v.24"},{"index":25,"sense_key":"make%2:41:01::","synset_id":"make.v.25"},{"index":26,"sense_key":"make%2:36:04::","synset_id":"hold.v.03"},{"index":27,"sense_key":"make%2:30:02::","synset_id":"make.v.27"},{"index":28,"sense_key":"make%2:38:05::","synset_id":"take.v.27"},{"index":29,"sense_key":"make%2:29:00::","synset_id":"stool.v.04"},{"index":30,"sense_key":"make%2:42:09::","synset_id":"make.v.30"},{"index":31,"sense_key":"make%2:42:13::","synset_id":"make.v.31"},{"index":32,"sense_key":"make%2:42:12::","synset_id":"make.v.32"},{"index":33,"sense_key":"make%2:42:11::","synset_id":"make.v.33"},{"index":34,"sense_key":"make%2:42:15::","synset_id":"make.v.34"},{"index":35,"sense_key":"make%2:39:12::","synset_id":"make.v.35"},{"index":36,"sense_key":"make%2:38:10::","synset_id":"make.v.36"},{"index":37,"sense_key":"make%2:38:15::","synset_id":"make.v.37"},{"index":38,"sense_key":"make%2:36:10::","synset_id":"make.v.38"},{"index":39,"sense_key":"make%2:36:07::","synset_id":"cook.v.02"},{"index":40,"sense_key":"make%2:35:13::","synset_id":"seduce.v.01"},{"index":41,"sense_key":"make%2:32:14::","synset_id":"make.v.41"},{"index":42,"sense_key":"make%2:32:09::","synset_id":"make.v.42"},{"index":43,"sense_key":"make%2:31:10::","synset_id":"make.v.43"},{"index":44,"sense_key":"make%2:31:12::","synset_id":"make.v.44"},{"index":45,"sense_key":"make%2:30:10::","synset_id":"make.v.45"},{"index":46,"sense_key":"make%2:30:14::","synset_id":"make.v.46"},{"index":47,"sense_key":"make%2:30:12::","synset_id":"make.v.47"},{"index":48,"sense_key":"make%2:29:06::","synset_id":"make.v.48"},{"index":49,"sense_key":"make%2:29:01::","synset_id":"make.v.49"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"make%2:36:00::","call_ids":["senseval2.d001.s026.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"make%2:36:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":3239,"cached_input_tokens":0,"output_tokens":136,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.007838,"input_uncached_usd":0.006477999999999999,"input_cached_usd":0.0,"output_usd":0.00136,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.4986067920690402}
{"item_id":"senseval2.d001.s026.t006","gold_sense_keys":["possible%3:00:00::"],"candidates":[{"index":1,"sense_key":"possible%3:00:00::","synset_id":"possible.a.01"},{"index":2,"sense_key":"possible%3:00:04::","synset_id":"potential.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"possible%3:00:00::","call_ids":["senseval2.d001.s026.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"possible%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":682,"cached_input_tokens":0,"output_tokens":108,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002444,"input_uncached_usd":0.001364,"input_cached_usd":0.0,"output_usd":0.00108,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9644701250363141}
{"item_id":"senseval2.d001.s026.t008","gold_sense_keys":["insight%1:09:01::","insight%1:09:02::"],"candidates":[{"index":1,"sense_key":"insight%1:09:02::","synset_id":"penetration.n.02"},{"index":2,"sense_key":"insight%1:12:00::","synset_id":"insight.n.02"},{"index":3,"sense_key":"insight%1:09:01::","synset_id":"insight.n.03"},{"index":4,"sense_key":"insight%1:09:00::","synset_id":"insight.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"insight%1:09:02::","call_ids":["senseval2.d001.s026.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"insight%1:09:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":768,"cached_input_tokens":0,"output_tokens":132,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002856,"input_uncached_usd":0.001536,"input_cached_usd":0.0,"output_usd":0.0013200000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8600022500613704}
{"item_id":"senseval2.d001.s026.t009","gold_sense_keys":["basic%3:00:00::"],"candidates":[{"index":1,"sense_key":"basic%3:00:00::","synset_id":"basic.a.01"},{"index":2,"sense_key":"basic%5:00:00:standard:01","synset_id":"basic.s.01"},{"index":3,"sense_key":"basic%5:00:02:first:00","synset_id":"basic.s.02"},{"index":4,"sense_key":"basic%5:00:00:alkaline:00","synset_id":"basic.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"basic%3:00:00::","call_ids":["senseval2.d001.s026.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"basic%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":816,"cached_input_tokens":0,"output_tokens":132,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002952,"input_uncached_usd":0.001632,"input_cached_usd":0.0,"output_usd":0.0013200000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.198250791989267}
{"item_id":"senseval2.d001.s026.t010","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s026.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":891,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001902,"input_uncached_usd":0.001782,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4388099999632686}
{"item_id":"senseval2.d001.s026.t011","gold_sense_keys":["say%2:32:00::","say%2:32:15::"],"candidates":[{"index":1,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":6,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":9,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":10,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":11,"sense_key":"say%2:32:09::","synset_id":"say.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"say%2:32:00::","call_ids":["senseval2.d001.s026.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"say%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1231,"cached_input_tokens":0,"output_tokens":99,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0034519999999999998,"input_uncached_usd":0.0024619999999999998,"input_cached_usd":0.0,"output_usd":0.00099,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1255914999637753}
{"item_id":"senseval2.d001.s026.t012","gold_sense_keys":["director%1:18:00::"],"candidates":[{"index":1,"sense_key":"director%1:18:00::","synset_id":"director.n.01"},{"index":2,"sense_key":"director%1:18:03::","synset_id":"director.n.02"},{"index":3,"sense_key":"director%1:18:02::","synset_id":"director.n.03"},{"index":4,"sense_key":"director%1:18:04::","synset_id":"film_director.n.01"},{"index":5,"sense_key":"director%1:18:01::","synset_id":"conductor.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"director%1:18:00::","call_ids":["senseval2.d001.s026.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"director%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":781,"cached_input_tokens":0,"output_tokens":107,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002632,"input_uncached_usd":0.001562,"input_cached_usd":0.0,"output_usd":0.00107,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.803606666973792}
{"item_id":"senseval2.d001.s027.t001","gold_sense_keys":["public%1:14:00::"],"candidates":[{"index":1,"sense_key":"public%1:14:00::","synset_id":"populace.n.01"},{"index":2,"sense_key":"public%1:14:01::","synset_id":"public.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"public%1:14:00::","call_ids":["senseval2.d001.s027.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"public%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":641,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001402,"input_uncached_usd":0.001282,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4964152500033379}
{"item_id":"senseval2.d001.s027.t002","gold_sense_keys":["concerned%3:00:00::"],"candidates":[{"index":1,"sense_key":"concerned%3:00:00::","synset_id":"concerned.a.01"},{"index":2,"sense_key":"concerned%5:00:00:involved:00","synset_id":"concerned.s.01"},{"index":3,"sense_key":"concerned%5:00:01:involved:00","synset_id":"implicated.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"concerned%3:00:00::","call_ids":["senseval2.d001.s027.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"concerned%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":759,"cached_input_tokens":0,"output_tokens":100,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0025180000000000003,"input_uncached_usd":0.001518,"input_cached_usd":0.0,"output_usd":0.001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6540069159818813}
{"item_id":"senseval2.d001.s027.t003","gold_sense_keys":["advance%1:11:01::"],"candidates":[{"index":1,"sense_key":"advance%1:11:00::","synset_id":"progress.n.03"},{"index":2,"sense_key":"advance%1:11:01::","synset_id":"improvement.n.01"},{"index":3,"sense_key":"advance%1:10:00::","synset_id":"overture.n.03"},{"index":4,"sense_key":"advance%1:04:00::","synset_id":"progress.n.02"},{"index":5,"sense_key":"advance%1:21:00::","synset_id":"advance.n.05"},{"index":6,"sense_key":"advance%1:04:01::","synset_id":"advance.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"advance%1:11:01::","call_ids":["senseval2.d001.s027.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"advance%1:11:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":872,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001864,"input_uncached_usd":0.0017439999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.448061874951236}
{"item_id":"senseval2.d001.s027.t004","gold_sense_keys":["treatment%1:04:00::"],"candidates":[{"index":1,"sense_key":"treatment%1:04:00::","synset_id":"treatment.n.01"},{"index":2,"sense_key":"treatment%1:04:01::","synset_id":"treatment.n.02"},{"index":3,"sense_key":"treatment%1:07:00::","synset_id":"treatment.n.03"},{"index":4,"sense_key":"treatment%1:10:00::","synset_id":"discussion.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"treatment%1:04:00::","call_ids":["senseval2.d001.s027.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"treatment%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":819,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001758,"input_uncached_usd":0.001638,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4320244579575956}
{"item_id":"senseval2.d001.s027.t005","gold_sense_keys":["basic%3:00:00::","basic%5:00:02:first:00"],"candidates":[{"index":1,"sense_key":"basic%3:00:00::","synset_id":"basic.a.01"},{"index":2,"sense_key":"basic%5:00:00:standard:01","synset_id":"basic.s.01"},{"index":3,"sense_key":"basic%5:00:02:first:00","synset_id":"basic.s.02"},{"index":4,"sense_key":"basic%5:00:00:alkaline:00","synset_id":"basic.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"basic%3:00:00::","call_ids":["senseval2.d001.s027.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"basic%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":788,"cached_input_tokens":0,"output_tokens":135,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0029259999999999998,"input_uncached_usd":0.001576,"input_cached_usd":0.0,"output_usd":0.00135,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6792462919838727}
{"item_id":"senseval2.d001.s027.t007","gold_sense_keys":["begin%2:30:00::"],"candidates":[{"index":1,"sense_key":"begin%2:30:00::","synset_id":"get_down.v.07"},{"index":2,"sense_key":"begin%2:42:00::","synset_id":"begin.v.02"},{"index":3,"sense_key":"begin%2:30:01::","synset_id":"begin.v.03"},{"index":4,"sense_key":"begin%2:32:03::","synset_id":"begin.v.04"},{"index":5,"sense_key":"begin%2:42:02::","synset_id":"begin.v.05"},{"index":6,"sense_key":"begin%2:42:04::","synset_id":"begin.v.06"},{"index":7,"sense_key":"begin%2:42:03::","synset_id":"begin.v.07"},{"index":8,"sense_key":"begin%2:41:04::","synset_id":"begin.v.08"},{"index":9,"sense_key":"begin%2:41:00::","synset_id":"begin.v.09"},{"index":10,"sense_key":"begin%2:32:04::","synset_id":"begin.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"begin%2:30:00::","call_ids":["senseval2.d001.s027.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"begin%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1259,"cached_input_tokens":0,"output_tokens":69,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003208,"input_uncached_usd":0.002518,"input_cached_usd":0.0,"output_usd":0.0006900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6118769160239026}
{"item_id":"senseval2.d001.s027.t008","gold_sense_keys":["show%2:39:00::"],"candidates":[{"index":1,"sense_key":"show%2:39:02::","synset_id":"show.v.01"},{"index":2,"sense_key":"show%2:31:00::","synset_id":"prove.v.02"},{"index":3,"sense_key":"show%2:32:00::","synset_id":"testify.v.02"},{"index":4,"sense_key":"show%2:39:00::","synset_id":"show.v.04"},{"index":5,"sense_key":"show%2:36:00::","synset_id":"picture.v.02"},{"index":6,"sense_key":"show%2:32:02::","synset_id":"express.v.01"},{"index":7,"sense_key":"show%2:32:01::","synset_id":"indicate.v.02"},{"index":8,"sense_key":"show%2:39:01::","synset_id":"show.v.08"},{"index":9,"sense_key":"show%2:32:04::","synset_id":"read.v.08"},{"index":10,"sense_key":"show%2:32:03::","synset_id":"show.v.10"},{"index":11,"sense_key":"show%2:38:00::","synset_id":"usher.v.01"},{"index":12,"sense_key":"show%2:33:00::","synset_id":"show.v.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"show%2:39:00::","call_ids":["senseval2.d001.s027.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"show%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1359,"cached_input_tokens":0,"output_tokens":202,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004738,"input_uncached_usd":0.002718,"input_cached_usd":0.0,"output_usd":0.00202,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.756257083034143}
{"item_id":"senseval2.d001.s027.t009","gold_sense_keys":["result%1:11:00::","result%1:19:00::"],"candidates":[{"index":1,"sense_key":"result%1:19:00::","synset_id":"consequence.n.01"},{"index":2,"sense_key":"result%1:10:00::","synset_id":"solution.n.02"},{"index":3,"sense_key":"result%1:11:00::","synset_id":"result.n.03"},{"index":4,"sense_key":"result%1:10:01::","synset_id":"resultant_role.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"result%1:19:00::","call_ids":["senseval2.d001.s027.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"result%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":862,"cached_input_tokens":0,"output_tokens":154,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003264,"input_uncached_usd":0.0017239999999999998,"input_cached_usd":0.0,"output_usd":0.0015400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.145974666927941}
{"item_id":"senseval2.d001.s027.t010","gold_sense_keys":["there%4:02:02::"],"candidates":[{"index":1,"sense_key":"there%4:02:00::","synset_id":"there.r.01"},{"index":2,"sense_key":"there%4:02:02::","synset_id":"there.r.02"},{"index":3,"sense_key":"there%4:02:01::","synset_id":"there.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"there%4:02:00::","call_ids":["senseval2.d001.s027.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"there%4:02:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":700,"cached_input_tokens":0,"output_tokens":183,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0032300000000000002,"input_uncached_usd":0.0014,"input_cached_usd":0.0,"output_usd":0.0018300000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2208364580292255}
{"item_id":"senseval2.d001.s029.t000","gold_sense_keys":["make%2:36:12::"],"candidates":[{"index":1,"sense_key":"make%2:41:00::","synset_id":"make.v.01"},{"index":2,"sense_key":"make%2:30:00::","synset_id":"make.v.02"},{"index":3,"sense_key":"make%2:36:00::","synset_id":"make.v.03"},{"index":4,"sense_key":"make%2:32:00::","synset_id":"induce.v.02"},{"index":5,"sense_key":"make%2:36:08::","synset_id":"cause.v.01"},{"index":6,"sense_key":"make%2:36:01::","synset_id":"produce.v.02"},{"index":7,"sense_key":"make%2:31:00::","synset_id":"draw.v.04"},{"index":8,"sense_key":"make%2:36:15::","synset_id":"make.v.08"},{"index":9,"sense_key":"make%2:36:09::","synset_id":"create.v.05"},{"index":10,"sense_key":"make%2:40:01::","synset_id":"gain.v.08"},{"index":11,"sense_key":"make%2:36:05::","synset_id":"do.v.08"},{"index":12,"sense_key":"make%2:42:05::","synset_id":"form.v.02"},{"index":13,"sense_key":"make%2:38:02::","synset_id":"reach.v.07"},{"index":14,"sense_key":"make%2:42:00::","synset_id":"make.v.14"},{"index":15,"sense_key":"make%2:36:11::","synset_id":"make.v.15"},{"index":16,"sense_key":"make%2:36:12::","synset_id":"make.v.16"},{"index":17,"sense_key":"make%2:36:13::","synset_id":"construct.v.01"},{"index":18,"sense_key":"make%2:30:15::","synset_id":"make.v.18"},{"index":19,"sense_key":"make%2:29:08::","synset_id":"make.v.19"},{"index":20,"sense_key":"make%2:41:03::","synset_id":"name.v.03"},{"index":21,"sense_key":"make%2:40:02::","synset_id":"have.v.17"},{"index":22,"sense_key":"make%2:38:00::","synset_id":"reach.v.01"},{"index":23,"sense_key":"make%2:31:13::","synset_id":"lay_down.v.01"},{"index":24,"sense_key":"make%2:41:13::","synset_id":"make.v.24"},{"index":25,"sense_key":"make%2:41:01::","synset_id":"make.v.25"},{"index":26,"sense_key":"make%2:36:04::","synset_id":"hold.v.03"},{"index":27,"sense_key":"make%2:30:02::","synset_id":"make.v.27"},{"index":28,"sense_key":"make%2:38:05::","synset_id":"take.v.27"},{"index":29,"sense_key":"make%2:29:00::","synset_id":"stool.v.04"},{"index":30,"sense_key":"make%2:42:09::","synset_id":"make.v.30"},{"index":31,"sense_key":"make%2:42:13::","synset_id":"make.v.31"},{"index":32,"sense_key":"make%2:42:12::","synset_id":"make.v.32"},{"index":33,"sense_key":"make%2:42:11::","synset_id":"make.v.33"},{"index":34,"sense_key":"make%2:42:15::","synset_id":"make.v.34"},{"index":35,"sense_key":"make%2:39:12::","synset_id":"make.v.35"},{"index":36,"sense_key":"make%2:38:10::","synset_id":"make.v.36"},{"index":37,"sense_key":"make%2:38:15::","synset_id":"make.v.37"},{"index":38,"sense_key":"make%2:36:10::","synset_id":"make.v.38"},{"index":39,"sense_key":"make%2:36:07::","synset_id":"cook.v.02"},{"index":40,"sense_key":"make%2:35:13::","synset_id":"seduce.v.01"},{"index":41,"sense_key":"make%2:32:14::","synset_id":"make.v.41"},{"index":42,"sense_key":"make%2:32:09::","synset_id":"make.v.42"},{"index":43,"sense_key":"make%2:31:10::","synset_id":"make.v.43"},{"index":44,"sense_key":"make%2:31:12::","synset_id":"make.v.44"},{"index":45,"sense_key":"make%2:30:10::","synset_id":"make.v.45"},{"index":46,"sense_key":"make%2:30:14::","synset_id":"make.v.46"},{"index":47,"sense_key":"make%2:30:12::","synset_id":"make.v.47"},{"index":48,"sense_key":"make%2:29:06::","synset_id":"make.v.48"},{"index":49,"sense_key":"make%2:29:01::","synset_id":"make.v.49"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":16,"chosen_sense_key":"make%2:36:12::","call_ids":["senseval2.d001.s029.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":16,"predicted_sense_key":"make%2:36:12::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":3189,"cached_input_tokens":0,"output_tokens":144,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.007818,"input_uncached_usd":0.006378,"input_cached_usd":0.0,"output_usd":0.00144,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":8.497840250027366}
{"item_id":"senseval2.d001.s029.t002","gold_sense_keys":["discovery%1:04:00::"],"candidates":[{"index":1,"sense_key":"discovery%1:04:00::","synset_id":"discovery.n.01"},{"index":2,"sense_key":"discovery%1:10:00::","synset_id":"discovery.n.02"},{"index":3,"sense_key":"discovery%1:09:00::","synset_id":"discovery.n.03"},{"index":4,"sense_key":"discovery%1:10:01::","synset_id":"discovery.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"discovery%1:04:00::","call_ids":["senseval2.d001.s029.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"discovery%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":713,"cached_input_tokens":0,"output_tokens":138,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0028060000000000003,"input_uncached_usd":0.001426,"input_cached_usd":0.0,"output_usd":0.0013800000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6251584580168128}
{"item_id":"senseval2.d001.s030.t000","gold_sense_keys":["then%4:02:00::"],"candidates":[{"index":1,"sense_key":"then%4:02:00::","synset_id":"then.r.01"},{"index":2,"sense_key":"then%4:02:02::","synset_id":"then.r.02"},{"index":3,"sense_key":"then%4:02:01::","synset_id":"then.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"then%4:02:00::","call_ids":["senseval2.d001.s030.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"then%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":707,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002184,"input_uncached_usd":0.001414,"input_cached_usd":0.0,"output_usd":0.0007700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4395131249912083}
{"item_id":"senseval2.d001.s030.t001","gold_sense_keys":["last%5:00:00:past:00"],"candidates":[{"index":1,"sense_key":"last%5:00:00:past:00","synset_id":"last.s.01"},{"index":2,"sense_key":"last%3:00:00::","synset_id":"last.a.02"},{"index":3,"sense_key":"last%5:00:00:closing:00","synset_id":"concluding.s.01"},{"index":4,"sense_key":"last%5:00:00:unlikely:00","synset_id":"last.s.03"},{"index":5,"sense_key":"last%5:00:00:dying:00","synset_id":"last.s.04"},{"index":6,"sense_key":"last%5:00:00:ultimate:00","synset_id":"final.s.02"},{"index":7,"sense_key":"last%5:00:00:high:02","synset_id":"last.s.06"},{"index":8,"sense_key":"last%5:00:00:unalterable:00","synset_id":"final.s.03"},{"index":9,"sense_key":"last%5:00:00:worst:00","synset_id":"last.s.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"last%5:00:00:past:00","call_ids":["senseval2.d001.s030.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"last%5:00:00:past:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1095,"cached_input_tokens":0,"output_tokens":48,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00267,"input_uncached_usd":0.00219,"input_cached_usd":0.0,"output_usd":0.00048000000000000007,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8100415410008281}
{"item_id":"senseval2.d001.s030.t002","gold_sense_keys":["spring%1:28:00::"],"candidates":[{"index":1,"sense_key":"spring%1:28:00::","synset_id":"spring.n.01"},{"index":2,"sense_key":"spring%1:06:00::","synset_id":"spring.n.02"},{"index":3,"sense_key":"spring%1:17:00::","synset_id":"spring.n.03"},{"index":4,"sense_key":"spring%1:15:00::","synset_id":"spring.n.04"},{"index":5,"sense_key":"spring%1:07:00::","synset_id":"give.n.01"},{"index":6,"sense_key":"spring%1:04:00::","synset_id":"leap.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"spring%1:28:00::","call_ids":["senseval2.d001.s030.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"spring%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":872,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001864,"input_uncached_usd":0.0017439999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5236084170173854}
{"item_id":"senseval2.d001.s030.t004","gold_sense_keys":["report%2:32:04::"],"candidates":[{"index":1,"sense_key":"report%2:32:00::","synset_id":"report.v.01"},{"index":2,"sense_key":"report%2:32:04::","synset_id":"report.v.02"},{"index":3,"sense_key":"report%2:32:03::","synset_id":"report.v.03"},{"index":4,"sense_key":"report%2:32:01::","synset_id":"report.v.04"},{"index":5,"sense_key":"report%2:32:13::","synset_id":"report.v.05"},{"index":6,"sense_key":"report%2:32:02::","synset_id":"report.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"report%2:32:04::","call_ids":["senseval2.d001.s030.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"report%2:32:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":887,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001894,"input_uncached_usd":0.001774,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5056768329814076}
{"item_id":"senseval2.d001.s030.t005","gold_sense_keys":["find%2:31:09::","find%2:32:00::","find%2:36:00::","find%2:39:02::"],"candidates":[{"index":1,"sense_key":"find%2:40:02::","synset_id":"find.v.01"},{"index":2,"sense_key":"find%2:39:02::","synset_id":"detect.v.01"},{"index":3,"sense_key":"find%2:40:00::","synset_id":"find.v.03"},{"index":4,"sense_key":"find%2:32:00::","synset_id":"determine.v.01"},{"index":5,"sense_key":"find%2:31:10::","synset_id":"find.v.05"},{"index":6,"sense_key":"find%2:39:01::","synset_id":"witness.v.02"},{"index":7,"sense_key":"find%2:40:01::","synset_id":"line_up.v.02"},{"index":8,"sense_key":"find%2:36:00::","synset_id":"discover.v.03"},{"index":9,"sense_key":"find%2:31:09::","synset_id":"discover.v.04"},{"index":10,"sense_key":"find%2:40:03::","synset_id":"find.v.10"},{"index":11,"sense_key":"find%2:32:01::","synset_id":"rule.v.04"},{"index":12,"sense_key":"find%2:30:00::","synset_id":"receive.v.02"},{"index":13,"sense_key":"find%2:39:05::","synset_id":"find.v.13"},{"index":14,"sense_key":"find%2:40:15::","synset_id":"recover.v.01"},{"index":15,"sense_key":"find%2:38:10::","synset_id":"find.v.15"},{"index":16,"sense_key":"find%2:30:13::","synset_id":"find_oneself.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"find%2:36:00::","call_ids":["senseval2.d001.s030.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"find%2:36:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1694,"cached_input_tokens":0,"output_tokens":185,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.005238,"input_uncached_usd":0.003388,"input_cached_usd":0.0,"output_usd":0.00185,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.4556364589370787}
{"item_id":"senseval2.d001.s030.t007","gold_sense_keys":["call%2:32:02::"],"candidates":[{"index":1,"sense_key":"call%2:32:02::","synset_id":"name.v.01"},{"index":2,"sense_key":"call%2:32:00::","synset_id":"call.v.02"},{"index":3,"sense_key":"call%2:32:01::","synset_id":"call.v.03"},{"index":4,"sense_key":"call%2:32:09::","synset_id":"shout.v.02"},{"index":5,"sense_key":"call%2:32:05::","synset_id":"call.v.05"},{"index":6,"sense_key":"call%2:41:00::","synset_id":"visit.v.03"},{"index":7,"sense_key":"call%2:41:04::","synset_id":"call.v.07"},{"index":8,"sense_key":"call%2:32:13::","synset_id":"call.v.08"},{"index":9,"sense_key":"call%2:32:04::","synset_id":"call.v.09"},{"index":10,"sense_key":"call%2:32:03::","synset_id":"call.v.10"},{"index":11,"sense_key":"call%2:42:09::","synset_id":"call.v.11"},{"index":12,"sense_key":"call%2:41:14::","synset_id":"address.v.06"},{"index":13,"sense_key":"call%2:41:12::","synset_id":"call.v.13"},{"index":14,"sense_key":"call%2:40:09::","synset_id":"call.v.14"},{"index":15,"sense_key":"call%2:40:05::","synset_id":"bid.v.04"},{"index":16,"sense_key":"call%2:38:00::","synset_id":"call.v.16"},{"index":17,"sense_key":"call%2:32:06::","synset_id":"call.v.17"},{"index":18,"sense_key":"call%2:32:10::","synset_id":"predict.v.01"},{"index":19,"sense_key":"call%2:32:12::","synset_id":"call.v.19"},{"index":20,"sense_key":"call%2:32:07::","synset_id":"call.v.20"},{"index":21,"sense_key":"call%2:32:08::","synset_id":"call.v.21"},{"index":22,"sense_key":"call%2:32:11::","synset_id":"call.v.22"},{"index":23,"sense_key":"call%2:32:15::","synset_id":"call.v.23"},{"index":24,"sense_key":"call%2:32:14::","synset_id":"call.v.24"},{"index":25,"sense_key":"call%2:31:13::","synset_id":"call.v.25"},{"index":26,"sense_key":"call%2:31:00::","synset_id":"call.v.26"},{"index":27,"sense_key":"call%2:31:05::","synset_id":"call.v.27"},{"index":28,"sense_key":"call%2:29:05::","synset_id":"call.v.28"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"call%2:32:02::","call_ids":["senseval2.d001.s030.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"call%2:32:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2165,"cached_input_tokens":0,"output_tokens":69,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00502,"input_uncached_usd":0.00433,"input_cached_usd":0.0,"output_usd":0.0006900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7273454170208424}
{"item_id":"senseval2.d001.s030.t008","gold_sense_keys":["impair%2:30:00::"],"candidates":[{"index":1,"sense_key":"impair%2:30:00::","synset_id":"impair.v.01"},{"index":2,"sense_key":"impair%2:30:01::","synset_id":"mar.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"impair%2:30:00::","call_ids":["senseval2.d001.s030.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"impair%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":646,"cached_input_tokens":0,"output_tokens":114,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002432,"input_uncached_usd":0.001292,"input_cached_usd":0.0,"output_usd":0.0011400000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.145660832989961}
{"item_id":"senseval2.d001.s030.t009","gold_sense_keys":["turn%2:42:13::"],"candidates":[{"index":1,"sense_key":"turn%2:38:00::","synset_id":"turn.v.01"},{"index":2,"sense_key":"turn%2:30:04::","synset_id":"change_state.v.01"},{"index":3,"sense_key":"turn%2:42:00::","synset_id":"become.v.02"},{"index":4,"sense_key":"turn%2:38:02::","synset_id":"turn.v.04"},{"index":5,"sense_key":"turn%2:30:00::","synset_id":"change_by_reversal.v.01"},{"index":6,"sense_key":"turn%2:38:03::","synset_id":"turn.v.06"},{"index":7,"sense_key":"turn%2:30:03::","synset_id":"turn.v.07"},{"index":8,"sense_key":"turn%2:35:08::","synset_id":"turn.v.08"},{"index":9,"sense_key":"turn%2:38:04::","synset_id":"turn.v.09"},{"index":10,"sense_key":"turn%2:38:01::","synset_id":"turn.v.10"},{"index":11,"sense_key":"turn%2:38:06::","synset_id":"turn.v.11"},{"index":12,"sense_key":"turn%2:36:00::","synset_id":"plow.v.01"},{"index":13,"sense_key":"turn%2:35:06::","synset_id":"turn.v.13"},{"index":14,"sense_key":"turn%2:30:09::","synset_id":"turn.v.14"},{"index":15,"sense_key":"turn%2:29:00::","synset_id":"twist.v.10"},{"index":16,"sense_key":"turn%2:42:13::","synset_id":"turn.v.16"},{"index":17,"sense_key":"turn%2:41:10::","synset_id":"turn.v.17"},{"index":18,"sense_key":"turn%2:40:13::","synset_id":"turn.v.18"},{"index":19,"sense_key":"turn%2:38:13::","synset_id":"turn.v.19"},{"index":20,"sense_key":"turn%2:38:14::","synset_id":"turn.v.20"},{"index":21,"sense_key":"turn%2:35:11::","synset_id":"flex.v.05"},{"index":22,"sense_key":"turn%2:35:10::","synset_id":"turn.v.22"},{"index":23,"sense_key":"turn%2:33:13::","synset_id":"turn.v.23"},{"index":24,"sense_key":"turn%2:32:14::","synset_id":"call_on.v.01"},{"index":25,"sense_key":"turn%2:30:01::","synset_id":"sour.v.01"},{"index":26,"sense_key":"turn%2:30:14::","synset_id":"turn.v.26"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":16,"chosen_sense_key":"turn%2:42:13::","call_ids":["senseval2.d001.s030.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":16,"predicted_sense_key":"turn%2:42:13::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2105,"cached_input_tokens":0,"output_tokens":123,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00544,"input_uncached_usd":0.00421,"input_cached_usd":0.0,"output_usd":0.0012300000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5824339999817312}
{"item_id":"senseval2.d001.s030.t010","gold_sense_keys":["healthy%3:00:00::"],"candidates":[{"index":1,"sense_key":"healthy%3:00:00::","synset_id":"healthy.a.01"},{"index":2,"sense_key":"healthy%5:00:00:sound:00","synset_id":"healthy.s.01"},{"index":3,"sense_key":"healthy%5:00:00:wholesome:00","synset_id":"healthy.s.02"},{"index":4,"sense_key":"healthy%5:00:00:reasonable:00","synset_id":"healthy.s.03"},{"index":5,"sense_key":"healthy%5:00:00:considerable:00","synset_id":"goodly.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"healthy%3:00:00::","call_ids":["senseval2.d001.s030.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"healthy%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":916,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001952,"input_uncached_usd":0.0018319999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.435834166011773}
{"item_id":"senseval2.d001.s030.t011","gold_sense_keys":["colon%1:08:00::"],"candidates":[{"index":1,"sense_key":"colon%1:08:00::","synset_id":"colon.n.01"},{"index":2,"sense_key":"colon%1:23:01::","synset_id":"colon.n.02"},{"index":3,"sense_key":"colon%1:23:00::","synset_id":"colon.n.03"},{"index":4,"sense_key":"colon%1:15:00::","synset_id":"colon.n.04"},{"index":5,"sense_key":"colon%1:10:00::","synset_id":"colon.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"colon%1:08:00::","call_ids":["senseval2.d001.s030.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"colon%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":839,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0017980000000000001,"input_uncached_usd":0.001678,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4062493330566213}
{"item_id":"senseval2.d001.s030.t012","gold_sense_keys":["cell%1:03:00::"],"candidates":[{"index":1,"sense_key":"cell%1:06:03::","synset_id":"cell.n.01"},{"index":2,"sense_key":"cell%1:03:00::","synset_id":"cell.n.02"},{"index":3,"sense_key":"cell%1:06:00::","synset_id":"cell.n.03"},{"index":4,"sense_key":"cell%1:14:00::","synset_id":"cell.n.04"},{"index":5,"sense_key":"cell%1:06:04::","synset_id":"cellular_telephone.n.01"},{"index":6,"sense_key":"cell%1:06:02::","synset_id":"cell.n.06"},{"index":7,"sense_key":"cell%1:06:01::","synset_id":"cell.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cell%1:03:00::","call_ids":["senseval2.d001.s030.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cell%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":953,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002026,"input_uncached_usd":0.001906,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5066483750706539}
{"item_id":"senseval2.d001.s030.t013","gold_sense_keys":["cancerous%5:00:00:malignant:02"],"candidates":[{"index":1,"sense_key":"cancerous%5:00:00:malignant:02","synset_id":"cancerous.s.01"},{"index":2,"sense_key":"cancerous%5:00:00:malign:00","synset_id":"cancerous.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancerous%5:00:00:malignant:02","call_ids":["senseval2.d001.s030.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancerous%5:00:00:malignant:02","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":673,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001466,"input_uncached_usd":0.001346,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.160374957951717}
{"item_id":"senseval2.d001.s031.t001","gold_sense_keys":["report%1:10:03::"],"candidates":[{"index":1,"sense_key":"report%1:10:03::","synset_id":"report.n.01"},{"index":2,"sense_key":"report%1:10:01::","synset_id":"report.n.02"},{"index":3,"sense_key":"report%1:10:00::","synset_id":"report.n.03"},{"index":4,"sense_key":"report%1:11:00::","synset_id":"report.n.04"},{"index":5,"sense_key":"report%1:10:04::","synset_id":"report_card.n.01"},{"index":6,"sense_key":"report%1:10:02::","synset_id":"composition.n.08"},{"index":7,"sense_key":"report%1:09:00::","synset_id":"reputation.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"report%1:10:03::","call_ids":["senseval2.d001.s031.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"report%1:10:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":968,"cached_input_tokens":0,"output_tokens":290,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004836,"input_uncached_usd":0.001936,"input_cached_usd":0.0,"output_usd":0.0029000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.7866536249639466}
{"item_id":"senseval2.d001.s031.t002","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":2,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":3,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"other%3:00:00::","call_ids":["senseval2.d001.s031.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":754,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001628,"input_uncached_usd":0.001508,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4022816669894382}
{"item_id":"senseval2.d001.s031.t004","gold_sense_keys":["team%1:14:00::"],"candidates":[{"index":1,"sense_key":"team%1:14:00::","synset_id":"team.n.01"},{"index":2,"sense_key":"team%1:14:01::","synset_id":"team.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"team%1:14:00::","call_ids":["senseval2.d001.s031.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"team%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":598,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0013160000000000001,"input_uncached_usd":0.001196,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9858659999445081}
{"item_id":"senseval2.d001.s031.t005","gold_sense_keys":["evidence%1:09:00::","evidence%1:10:01::"],"candidates":[{"index":1,"sense_key":"evidence%1:09:00::","synset_id":"evidence.n.01"},{"index":2,"sense_key":"evidence%1:10:01::","synset_id":"evidence.n.02"},{"index":3,"sense_key":"evidence%1:10:00::","synset_id":"evidence.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"evidence%1:09:00::","call_ids":["senseval2.d001.s031.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"evidence%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":709,"cached_input_tokens":0,"output_tokens":163,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0030480000000000004,"input_uncached_usd":0.001418,"input_cached_usd":0.0,"output_usd":0.0016300000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.6903143749805167}
{"item_id":"senseval2.d001.s031.t006","gold_sense_keys":["same%3:00:00::"],"candidates":[{"index":1,"sense_key":"same%3:00:02::","synset_id":"same.a.01"},{"index":2,"sense_key":"same%3:00:00::","synset_id":"same.a.02"},{"index":3,"sense_key":"same%3:00:04::","synset_id":"like.a.02"},{"index":4,"sense_key":"same%5:00:00:unchanged:00","synset_id":"same.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"same%3:00:02::","call_ids":["senseval2.d001.s031.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"same%3:00:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":741,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0021520000000000003,"input_uncached_usd":0.001482,"input_cached_usd":0.0,"output_usd":0.00067,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3896079579135403}
{"item_id":"senseval2.d001.s031.t007","gold_sense_keys":["damaged%3:00:00::"],"candidates":[{"index":1,"sense_key":"damaged%3:00:00::","synset_id":"damaged.a.01"},{"index":2,"sense_key":"damaged%5:00:00:disreputable:00","synset_id":"discredited.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"damaged%3:00:00::","call_ids":["senseval2.d001.s031.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"damaged%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":656,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0014320000000000001,"input_uncached_usd":0.001312,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2232149159535766}
{"item_id":"senseval2.d001.s031.t009","gold_sense_keys":["present%3:00:02::"],"candidates":[{"index":1,"sense_key":"present%3:00:01::","synset_id":"present.a.01"},{"index":2,"sense_key":"present%3:00:02::","synset_id":"present.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"present%3:00:02::","call_ids":["senseval2.d001.s031.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"present%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":630,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00138,"input_uncached_usd":0.0012599999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9675250829895958}
{"item_id":"senseval2.d001.s031.t010","gold_sense_keys":["tissue%1:08:00::"],"candidates":[{"index":1,"sense_key":"tissue%1:08:00::","synset_id":"tissue.n.01"},{"index":2,"sense_key":"tissue%1:27:02::","synset_id":"tissue.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"tissue%1:08:00::","call_ids":["senseval2.d001.s031.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"tissue%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":619,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001358,"input_uncached_usd":0.001238,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0452402910450473}
{"item_id":"senseval2.d001.s032.t004","gold_sense_keys":["most%4:02:00::"],"candidates":[{"index":1,"sense_key":"most%4:02:00::","synset_id":"most.r.01"},{"index":2,"sense_key":"most%4:02:01::","synset_id":"most.r.02"},{"index":3,"sense_key":"most%4:02:02::","synset_id":"about.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"most%4:02:00::","call_ids":["senseval2.d001.s032.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"most%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":693,"cached_input_tokens":0,"output_tokens":45,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001836,"input_uncached_usd":0.0013859999999999999,"input_cached_usd":0.0,"output_usd":0.00045000000000000004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3165796250104904}
{"item_id":"senseval2.d001.s032.t005","gold_sense_keys":["common%3:00:01::","common%5:00:00:familiar:02"],"candidates":[{"index":1,"sense_key":"common%3:00:02::","synset_id":"common.a.01"},{"index":2,"sense_key":"common%3:00:01::","synset_id":"common.a.02"},{"index":3,"sense_key":"common%5:00:00:shared:00","synset_id":"common.s.01"},{"index":4,"sense_key":"common%5:00:00:familiar:02","synset_id":"common.s.02"},{"index":5,"sense_key":"common%5:00:00:informal:02","synset_id":"common.s.03"},{"index":6,"sense_key":"common%5:00:00:lowborn:00","synset_id":"common.s.04"},{"index":7,"sense_key":"common%5:00:00:inferior:02","synset_id":"coarse.s.02"},{"index":8,"sense_key":"common%5:00:00:unrefined:01","synset_id":"coarse.s.01"},{"index":9,"sense_key":"common%5:00:00:ordinary:00","synset_id":"common.s.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"common%5:00:00:familiar:02","call_ids":["senseval2.d001.s032.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"common%5:00:00:familiar:02","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1134,"cached_input_tokens":0,"output_tokens":102,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003288,"input_uncached_usd":0.002268,"input_cached_usd":0.0,"output_usd":0.00102,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0429446670459583}
{"item_id":"senseval2.d001.s032.t007","gold_sense_keys":["form%1:09:01::"],"candidates":[{"index":1,"sense_key":"form%1:10:00::","synset_id":"form.n.01"},{"index":2,"sense_key":"form%1:09:01::","synset_id":"kind.n.01"},{"index":3,"sense_key":"form%1:09:00::","synset_id":"form.n.03"},{"index":4,"sense_key":"form%1:07:01::","synset_id":"shape.n.01"},{"index":5,"sense_key":"form%1:08:00::","synset_id":"human_body.n.01"},{"index":6,"sense_key":"form%1:03:00::","synset_id":"shape.n.02"},{"index":7,"sense_key":"form%1:07:00::","synset_id":"form.n.07"},{"index":8,"sense_key":"form%1:10:01::","synset_id":"form.n.08"},{"index":9,"sense_key":"form%1:14:02::","synset_id":"form.n.09"},{"index":10,"sense_key":"form%1:10:02::","synset_id":"form.n.10"},{"index":11,"sense_key":"form%1:07:03::","synset_id":"form.n.11"},{"index":12,"sense_key":"form%1:26:00::","synset_id":"phase.n.02"},{"index":13,"sense_key":"form%1:14:00::","synset_id":"class.n.02"},{"index":14,"sense_key":"form%1:07:02::","synset_id":"form.n.14"},{"index":15,"sense_key":"form%1:06:00::","synset_id":"mannequin.n.02"},{"index":16,"sense_key":"form%1:06:01::","synset_id":"form.n.16"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"form%1:09:01::","call_ids":["senseval2.d001.s032.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"form%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1578,"cached_input_tokens":0,"output_tokens":91,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004066,"input_uncached_usd":0.003156,"input_cached_usd":0.0,"output_usd":0.0009100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4725979999639094}
{"item_id":"senseval2.d001.s032.t010","gold_sense_keys":["kill%2:35:02::"],"candidates":[{"index":1,"sense_key":"kill%2:35:00::","synset_id":"kill.v.01"},{"index":2,"sense_key":"kill%2:41:01::","synset_id":"kill.v.02"},{"index":3,"sense_key":"kill%2:30:08::","synset_id":"stamp_out.v.01"},{"index":4,"sense_key":"kill%2:42:00::","synset_id":"kill.v.04"},{"index":5,"sense_key":"kill%2:39:00::","synset_id":"kill.v.05"},{"index":6,"sense_key":"kill%2:37:00::","synset_id":"kill.v.06"},{"index":7,"sense_key":"kill%2:35:06::","synset_id":"kill.v.07"},{"index":8,"sense_key":"kill%2:35:05::","synset_id":"kill.v.08"},{"index":9,"sense_key":"kill%2:35:02::","synset_id":"kill.v.09"},{"index":10,"sense_key":"kill%2:35:01::","synset_id":"kill.v.10"},{"index":11,"sense_key":"kill%2:34:00::","synset_id":"toss_off.v.02"},{"index":12,"sense_key":"kill%2:30:00::","synset_id":"kill.v.12"},{"index":13,"sense_key":"kill%2:30:04::","synset_id":"kill.v.13"},{"index":14,"sense_key":"kill%2:30:03::","synset_id":"kill.v.14"},{"index":15,"sense_key":"kill%2:30:02::","synset_id":"kill.v.15"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"kill%2:35:02::","call_ids":["senseval2.d001.s032.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"kill%2:35:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1328,"cached_input_tokens":0,"output_tokens":122,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003876,"input_uncached_usd":0.002656,"input_cached_usd":0.0,"output_usd":0.0012200000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8171105840010568}
{"item_id":"senseval2.d001.s032.t012","gold_sense_keys":["american%1:18:00::"],"candidates":[{"index":1,"sense_key":"american%1:18:00::","synset_id":"american.n.01"},{"index":2,"sense_key":"american%1:10:00::","synset_id":"american_english.n.01"},{"index":3,"sense_key":"american%1:18:01::","synset_id":"american.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"american%1:18:00::","call_ids":["senseval2.d001.s032.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"american%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":621,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0013620000000000001,"input_uncached_usd":0.001242,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0545961670577526}
{"item_id":"senseval2.d001.s032.t013","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["senseval2.d001.s032.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":764,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001648,"input_uncached_usd":0.0015279999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0372457079356536}
{"item_id":"senseval2.d001.s033.t000","gold_sense_keys":["right%4:02:04::","right%4:02:08::"],"candidates":[{"index":1,"sense_key":"right%4:02:04::","synset_id":"right.r.01"},{"index":2,"sense_key":"right%4:02:08::","synset_id":"right.r.02"},{"index":3,"sense_key":"right%4:02:09::","synset_id":"right.r.03"},{"index":4,"sense_key":"right%4:02:03::","synset_id":"right.r.04"},{"index":5,"sense_key":"right%4:02:01::","synset_id":"properly.r.01"},{"index":6,"sense_key":"right%4:02:00::","synset_id":"right.r.06"},{"index":7,"sense_key":"right%4:02:07::","synset_id":"right.r.07"},{"index":8,"sense_key":"right%4:02:06::","synset_id":"mighty.r.01"},{"index":9,"sense_key":"right%4:02:05::","synset_id":"justly.r.02"},{"index":10,"sense_key":"right%4:02:02::","synset_id":"correctly.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"right%4:02:08::","call_ids":["senseval2.d001.s033.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"right%4:02:08::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":978,"cached_input_tokens":0,"output_tokens":146,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0034159999999999998,"input_uncached_usd":0.001956,"input_cached_usd":0.0,"output_usd":0.0014600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.4303094170754775}
{"item_id":"senseval2.d001.s033.t001","gold_sense_keys":["now%4:02:04::"],"candidates":[{"index":1,"sense_key":"now%4:02:05::","synset_id":"now.r.01"},{"index":2,"sense_key":"now%4:02:00::","synset_id":"nowadays.r.01"},{"index":3,"sense_key":"now%4:02:02::","synset_id":"now.r.03"},{"index":4,"sense_key":"now%4:02:04::","synset_id":"now.r.04"},{"index":5,"sense_key":"now%4:02:01::","synset_id":"immediately.r.01"},{"index":6,"sense_key":"now%4:02:07::","synset_id":"now.r.06"},{"index":7,"sense_key":"now%4:02:06::","synset_id":"now.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"now%4:02:04::","call_ids":["senseval2.d001.s033.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"now%4:02:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":906,"cached_input_tokens":0,"output_tokens":94,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002752,"input_uncached_usd":0.001812,"input_cached_usd":0.0,"output_usd":0.0009400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3648847920121625}
{"item_id":"senseval2.d001.s033.t002","gold_sense_keys":["about%4:02:00::"],"candidates":[{"index":1,"sense_key":"about%4:02:00::","synset_id":"approximately.r.01"},{"index":2,"sense_key":"about%4:02:06::","synset_id":"about.r.02"},{"index":3,"sense_key":"about%4:02:01::","synset_id":"about.r.03"},{"index":4,"sense_key":"about%4:02:04::","synset_id":"about.r.04"},{"index":5,"sense_key":"about%4:02:02::","synset_id":"about.r.05"},{"index":6,"sense_key":"about%4:02:03::","synset_id":"about.r.06"},{"index":7,"sense_key":"about%4:02:05::","synset_id":"about.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"about%4:02:00::","call_ids":["senseval2.d001.s033.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"about%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":917,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001954,"input_uncached_usd":0.001834,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4006391670554876}
{"item_id":"senseval2.d001.s033.t004","gold_sense_keys":["laboratory%1:06:00::"],"candidates":[{"index":1,"sense_key":"laboratory%1:06:00::","synset_id":"lab.n.01"},{"index":2,"sense_key":"laboratory%1:15:00::","synset_id":"testing_ground.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"laboratory%1:06:00::","call_ids":["senseval2.d001.s033.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"laboratory%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":657,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001434,"input_uncached_usd":0.0013139999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1994712080340832}
{"item_id":"senseval2.d001.s033.t005","gold_sense_keys":["u.s.%1:15:00::"],"candidates":[{"index":1,"sense_key":"u.s.%1:14:00::","synset_id":"united_states_government.n.01"},{"index":2,"sense_key":"us%1:15:00::","synset_id":"united_states.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"us%1:15:00::","call_ids":["senseval2.d001.s033.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"us%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":645,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00196,"input_uncached_usd":0.00129,"input_cached_usd":0.0,"output_usd":0.00067,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4679873330751434}
{"item_id":"senseval2.d001.s033.t008","gold_sense_keys":["race%2:41:03::"],"candidates":[{"index":1,"sense_key":"race%2:38:00::","synset_id":"rush.v.01"},{"index":2,"sense_key":"race%2:33:00::","synset_id":"race.v.02"},{"index":3,"sense_key":"race%2:41:03::","synset_id":"race.v.03"},{"index":4,"sense_key":"race%2:38:10::","synset_id":"race.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"race%2:41:03::","call_ids":["senseval2.d001.s033.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"race%2:41:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":727,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001574,"input_uncached_usd":0.001454,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0160832919646055}
{"item_id":"senseval2.d001.s033.t009","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":2,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":3,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"other%3:00:00::","call_ids":["senseval2.d001.s033.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":701,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0015220000000000001,"input_uncached_usd":0.001402,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2319654579041526}
{"item_id":"senseval2.d001.s034.t000","gold_sense_keys":["about%4:02:00::"],"candidates":[{"index":1,"sense_key":"about%4:02:00::","synset_id":"approximately.r.01"},{"index":2,"sense_key":"about%4:02:06::","synset_id":"about.r.02"},{"index":3,"sense_key":"about%4:02:01::","synset_id":"about.r.03"},{"index":4,"sense_key":"about%4:02:04::","synset_id":"about.r.04"},{"index":5,"sense_key":"about%4:02:02::","synset_id":"about.r.05"},{"index":6,"sense_key":"about%4:02:03::","synset_id":"about.r.06"},{"index":7,"sense_key":"about%4:02:05::","synset_id":"about.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"about%4:02:00::","call_ids":["senseval2.d001.s034.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"about%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":963,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002046,"input_uncached_usd":0.001926,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3056770409457386}
{"item_id":"senseval2.d001.s035.t001","gold_sense_keys":["say%2:32:01::"],"candidates":[{"index":1,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":6,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":9,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":10,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":11,"sense_key":"say%2:32:09::","synset_id":"say.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"say%2:32:01::","call_ids":["senseval2.d001.s035.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"say%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1203,"cached_input_tokens":0,"output_tokens":123,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003636,"input_uncached_usd":0.0024059999999999997,"input_cached_usd":0.0,"output_usd":0.0012300000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.574112290982157}
{"item_id":"senseval2.d001.s035.t003","gold_sense_keys":["alone%4:02:01::"],"candidates":[{"index":1,"sense_key":"alone%4:02:00::","synset_id":"entirely.r.02"},{"index":2,"sense_key":"alone%4:02:01::","synset_id":"alone.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"alone%4:02:00::","call_ids":["senseval2.d001.s035.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"alone%4:02:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":697,"cached_input_tokens":0,"output_tokens":98,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0023740000000000002,"input_uncached_usd":0.0013939999999999998,"input_cached_usd":0.0,"output_usd":0.0009800000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.260649958043359}
{"item_id":"senseval2.d001.s035.t004","gold_sense_keys":["combination%1:14:00::"],"candidates":[{"index":1,"sense_key":"combination%1:14:00::","synset_id":"combination.n.01"},{"index":2,"sense_key":"combination%1:14:04::","synset_id":"combination.n.02"},{"index":3,"sense_key":"combination%1:14:02::","synset_id":"combination.n.03"},{"index":4,"sense_key":"combination%1:14:01::","synset_id":"combination.n.04"},{"index":5,"sense_key":"combination%1:14:03::","synset_id":"combination.n.05"},{"index":6,"sense_key":"combination%1:04:01::","synset_id":"combination.n.06"},{"index":7,"sense_key":"combination%1:04:00::","synset_id":"combination.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"combination%1:14:00::","call_ids":["senseval2.d001.s035.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"combination%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":886,"cached_input_tokens":0,"output_tokens":144,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003212,"input_uncached_usd":0.001772,"input_cached_usd":0.0,"output_usd":0.00144,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.174446541001089}
{"item_id":"senseval2.d001.s035.t005","gold_sense_keys":["appear%2:39:00::","appear%2:39:01::"],"candidates":[{"index":1,"sense_key":"appear%2:39:00::","synset_id":"look.v.02"},{"index":2,"sense_key":"appear%2:30:00::","synset_id":"appear.v.02"},{"index":3,"sense_key":"appear%2:30:01::","synset_id":"appear.v.03"},{"index":4,"sense_key":"appear%2:39:01::","synset_id":"appear.v.04"},{"index":5,"sense_key":"appear%2:30:02::","synset_id":"appear.v.05"},{"index":6,"sense_key":"appear%2:36:00::","synset_id":"appear.v.06"},{"index":7,"sense_key":"appear%2:41:03::","synset_id":"appear.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"appear%2:39:00::","call_ids":["senseval2.d001.s035.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"appear%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1034,"cached_input_tokens":0,"output_tokens":206,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004128,"input_uncached_usd":0.002068,"input_cached_usd":0.0,"output_usd":0.00206,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.3665632089832798}
{"item_id":"senseval2.d001.s035.t006","gold_sense_keys":["crucial%5:00:00:important:00","crucial%5:00:00:material:02"],"candidates":[{"index":1,"sense_key":"crucial%3:00:00::","synset_id":"crucial.a.01"},{"index":2,"sense_key":"crucial%5:00:00:material:02","synset_id":"crucial.s.01"},{"index":3,"sense_key":"crucial%5:00:00:important:00","synset_id":"all-important.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"crucial%5:00:00:material:02","call_ids":["senseval2.d001.s035.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"crucial%5:00:00:material:02","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":736,"cached_input_tokens":0,"output_tokens":167,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003142,"input_uncached_usd":0.001472,"input_cached_usd":0.0,"output_usd":0.00167,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.7861975410487503}
{"item_id":"senseval2.d001.s035.t007","gold_sense_keys":["development%1:22:02::"],"candidates":[{"index":1,"sense_key":"development%1:04:01::","synset_id":"development.n.01"},{"index":2,"sense_key":"development%1:22:02::","synset_id":"development.n.02"},{"index":3,"sense_key":"development%1:22:01::","synset_id":"growth.n.01"},{"index":4,"sense_key":"development%1:11:01::","synset_id":"development.n.04"},{"index":5,"sense_key":"development%1:04:02::","synset_id":"exploitation.n.01"},{"index":6,"sense_key":"development%1:15:00::","synset_id":"development.n.06"},{"index":7,"sense_key":"development%1:26:00::","synset_id":"development.n.07"},{"index":8,"sense_key":"development%1:22:00::","synset_id":"development.n.08"},{"index":9,"sense_key":"development%1:10:00::","synset_id":"development.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"development%1:22:02::","call_ids":["senseval2.d001.s035.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"development%1:22:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1209,"cached_input_tokens":0,"output_tokens":135,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003768,"input_uncached_usd":0.002418,"input_cached_usd":0.0,"output_usd":0.00135,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.975222249981016}
{"item_id":"senseval2.d001.s035.t009","gold_sense_keys":["scourge%1:26:00::"],"candidates":[{"index":1,"sense_key":"scourge%1:06:00::","synset_id":"scourge.n.01"},{"index":2,"sense_key":"scourge%1:26:00::","synset_id":"bane.n.01"},{"index":3,"sense_key":"scourge%1:18:00::","synset_id":"terror.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"scourge%1:26:00::","call_ids":["senseval2.d001.s035.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"scourge%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":720,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00156,"input_uncached_usd":0.0014399999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.107800625031814}
{"item_id":"senseval2.d001.s035.t010","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s035.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":863,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001846,"input_uncached_usd":0.0017259999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1829814999364316}
{"item_id":"senseval2.d001.s035.t011","gold_sense_keys":["brain%1:08:00::"],"candidates":[{"index":1,"sense_key":"brain%1:08:00::","synset_id":"brain.n.01"},{"index":2,"sense_key":"brain%1:09:00::","synset_id":"brain.n.02"},{"index":3,"sense_key":"brain%1:09:01::","synset_id":"mind.n.01"},{"index":4,"sense_key":"brain%1:18:00::","synset_id":"genius.n.01"},{"index":5,"sense_key":"brain%1:13:00::","synset_id":"brain.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"brain%1:08:00::","call_ids":["senseval2.d001.s035.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"brain%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":904,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001928,"input_uncached_usd":0.001808,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6822209999663755}
{"item_id":"senseval2.d001.s035.t012","gold_sense_keys":["skin%1:08:00::"],"candidates":[{"index":1,"sense_key":"skin%1:08:00::","synset_id":"skin.n.01"},{"index":2,"sense_key":"skin%1:06:01::","synset_id":"skin.n.02"},{"index":3,"sense_key":"skin%1:05:01::","synset_id":"hide.n.02"},{"index":4,"sense_key":"skin%1:26:00::","synset_id":"skin.n.04"},{"index":5,"sense_key":"skin%1:13:00::","synset_id":"peel.n.02"},{"index":6,"sense_key":"skin%1:06:02::","synset_id":"skin.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"skin%1:08:00::","call_ids":["senseval2.d001.s035.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"skin%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":835,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00179,"input_uncached_usd":0.0016699999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.440580541966483}
{"item_id":"senseval2.d001.s035.t015","gold_sense_keys":["cervix%1:08:02::"],"candidates":[{"index":1,"sense_key":"cervix%1:08:01::","synset_id":"neck.n.01"},{"index":2,"sense_key":"cervix%1:08:02::","synset_id":"cervix.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cervix%1:08:02::","call_ids":["senseval2.d001.s035.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cervix%1:08:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":678,"cached_input_tokens":0,"output_tokens":70,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002056,"input_uncached_usd":0.001356,"input_cached_usd":0.0,"output_usd":0.0007000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.452722082962282}
{"item_id":"senseval2.d001.s036.t000","gold_sense_keys":["evidence%1:09:00::","evidence%1:10:01::"],"candidates":[{"index":1,"sense_key":"evidence%1:09:00::","synset_id":"evidence.n.01"},{"index":2,"sense_key":"evidence%1:10:01::","synset_id":"evidence.n.02"},{"index":3,"sense_key":"evidence%1:10:00::","synset_id":"evidence.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"evidence%1:09:00::","call_ids":["senseval2.d001.s036.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"evidence%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":763,"cached_input_tokens":0,"output_tokens":141,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0029360000000000002,"input_uncached_usd":0.001526,"input_cached_usd":0.0,"output_usd":0.00141,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8289137500105426}
{"item_id":"senseval2.d001.s036.t001","gold_sense_keys":["people%1:14:00::"],"candidates":[{"index":1,"sense_key":"people%1:14:00::","synset_id":"people.n.01"},{"index":2,"sense_key":"people%1:14:01::","synset_id":"citizenry.n.01"},{"index":3,"sense_key":"people%1:14:02::","synset_id":"people.n.03"},{"index":4,"sense_key":"people%1:14:03::","synset_id":"multitude.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"people%1:14:00::","call_ids":["senseval2.d001.s036.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"people%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":807,"cached_input_tokens":0,"output_tokens":86,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002474,"input_uncached_usd":0.001614,"input_cached_usd":0.0,"output_usd":0.0008600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3772112079896033}
{"item_id":"senseval2.d001.s036.t002","gold_sense_keys":["inherit%2:40:01::"],"candidates":[{"index":1,"sense_key":"inherit%2:40:00::","synset_id":"inherit.v.01"},{"index":2,"sense_key":"inherit%2:40:02::","synset_id":"inherit.v.02"},{"index":3,"sense_key":"inherit%2:40:01::","synset_id":"inherit.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"inherit%2:40:01::","call_ids":["senseval2.d001.s036.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"inherit%2:40:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":729,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001578,"input_uncached_usd":0.0014579999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4364668750204146}
{"item_id":"senseval2.d001.s036.t004","gold_sense_keys":["version%1:09:01::"],"candidates":[{"index":1,"sense_key":"version%1:10:01::","synset_id":"version.n.01"},{"index":2,"sense_key":"version%1:09:01::","synset_id":"version.n.02"},{"index":3,"sense_key":"version%1:10:02::","synset_id":"adaptation.n.01"},{"index":4,"sense_key":"version%1:10:00::","synset_id":"translation.n.01"},{"index":5,"sense_key":"version%1:09:00::","synset_id":"interpretation.n.01"},{"index":6,"sense_key":"version%1:04:00::","synset_id":"version.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"version%1:09:01::","call_ids":["senseval2.d001.s036.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"version%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":920,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00196,"input_uncached_usd":0.0018399999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0453211249550804}
{"item_id":"senseval2.d001.s036.t006","gold_sense_keys":["especially%4:02:00::"],"candidates":[{"index":1,"sense_key":"especially%4:02:00::","synset_id":"particularly.r.01"},{"index":2,"sense_key":"especially%4:02:01::","synset_id":"specially.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"especially%4:02:00::","call_ids":["senseval2.d001.s036.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"especially%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":706,"cached_input_tokens":0,"output_tokens":90,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002312,"input_uncached_usd":0.0014119999999999998,"input_cached_usd":0.0,"output_usd":0.0009000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.172124959062785}
{"item_id":"senseval2.d001.s036.t007","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s036.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":892,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001904,"input_uncached_usd":0.001784,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.075133917038329}
{"item_id":"senseval2.d001.s036.t009","gold_sense_keys":["explain%2:32:02::"],"candidates":[{"index":1,"sense_key":"explain%2:32:00::","synset_id":"explain.v.01"},{"index":2,"sense_key":"explain%2:32:01::","synset_id":"explain.v.02"},{"index":3,"sense_key":"explain%2:32:02::","synset_id":"excuse.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"explain%2:32:02::","call_ids":["senseval2.d001.s036.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"explain%2:32:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":765,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00219,"input_uncached_usd":0.00153,"input_cached_usd":0.0,"output_usd":0.0006600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4868449590867385}
{"item_id":"senseval2.d001.s036.t010","gold_sense_keys":["finally%4:02:02::"],"candidates":[{"index":1,"sense_key":"finally%4:02:02::","synset_id":"finally.r.01"},{"index":2,"sense_key":"finally%4:02:00::","synset_id":"ultimately.r.01"},{"index":3,"sense_key":"finally%4:02:03::","synset_id":"last.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"finally%4:02:02::","call_ids":["senseval2.d001.s036.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"finally%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":755,"cached_input_tokens":0,"output_tokens":175,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00326,"input_uncached_usd":0.0015099999999999998,"input_cached_usd":0.0,"output_usd":0.00175,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.062375291949138}
{"item_id":"senseval2.d001.s036.t011","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s036.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":892,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001904,"input_uncached_usd":0.001784,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5120512919966131}
{"item_id":"senseval2.d001.s036.t012","gold_sense_keys":["seem%2:39:00::"],"candidates":[{"index":1,"sense_key":"seem%2:39:00::","synset_id":"look.v.02"},{"index":2,"sense_key":"seem%2:39:01::","synset_id":"appear.v.04"},{"index":3,"sense_key":"seem%2:42:00::","synset_id":"seem.v.03"},{"index":4,"sense_key":"seem%2:39:02::","synset_id":"seem.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"seem%2:39:00::","call_ids":["senseval2.d001.s036.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"seem%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":834,"cached_input_tokens":0,"output_tokens":136,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0030280000000000003,"input_uncached_usd":0.001668,"input_cached_usd":0.0,"output_usd":0.00136,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.849132249946706}
{"item_id":"senseval2.d001.s036.t014","gold_sense_keys":["certain%5:00:00:definite:00"],"candidates":[{"index":1,"sense_key":"certain%5:00:00:definite:00","synset_id":"certain.s.01"},{"index":2,"sense_key":"certain%3:00:02::","synset_id":"certain.a.02"},{"index":3,"sense_key":"certain%3:00:01::","synset_id":"certain.a.03"},{"index":4,"sense_key":"certain%3:00:03::","synset_id":"certain.a.04"},{"index":5,"sense_key":"certain%3:00:05::","synset_id":"sealed.a.01"},{"index":6,"sense_key":"certain%5:00:00:reliable:00","synset_id":"certain.s.02"},{"index":7,"sense_key":"certain%5:00:00:careful:00","synset_id":"certain.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"certain%5:00:00:definite:00","call_ids":["senseval2.d001.s036.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"certain%5:00:00:definite:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1044,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002208,"input_uncached_usd":0.002088,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5426732499618083}
{"item_id":"senseval2.d001.s036.t015","gold_sense_keys":["family%1:14:01::"],"candidates":[{"index":1,"sense_key":"family%1:14:02::","synset_id":"family.n.01"},{"index":2,"sense_key":"family%1:14:00::","synset_id":"family.n.02"},{"index":3,"sense_key":"family%1:14:04::","synset_id":"class.n.01"},{"index":4,"sense_key":"family%1:14:01::","synset_id":"family.n.04"},{"index":5,"sense_key":"family%1:18:00::","synset_id":"kin.n.01"},{"index":6,"sense_key":"family%1:14:03::","synset_id":"family.n.06"},{"index":7,"sense_key":"family%1:14:06::","synset_id":"syndicate.n.01"},{"index":8,"sense_key":"family%1:14:05::","synset_id":"family.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"family%1:14:01::","call_ids":["senseval2.d001.s036.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"family%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1092,"cached_input_tokens":0,"output_tokens":123,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003414,"input_uncached_usd":0.0021839999999999997,"input_cached_usd":0.0,"output_usd":0.0012300000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.367656167014502}
{"item_id":"senseval2.d001.s037.t000","gold_sense_keys":["story%1:10:04::"],"candidates":[{"index":1,"sense_key":"story%1:10:03::","synset_id":"narrative.n.01"},{"index":2,"sense_key":"story%1:10:00::","synset_id":"story.n.02"},{"index":3,"sense_key":"story%1:06:00::","synset_id":"floor.n.02"},{"index":4,"sense_key":"story%1:10:04::","synset_id":"history.n.02"},{"index":5,"sense_key":"story%1:10:01::","synset_id":"report.n.03"},{"index":6,"sense_key":"story%1:10:02::","synset_id":"fib.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"story%1:10:04::","call_ids":["senseval2.d001.s037.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"story%1:10:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1019,"cached_input_tokens":0,"output_tokens":84,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002878,"input_uncached_usd":0.002038,"input_cached_usd":0.0,"output_usd":0.00084,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5984627499710768}
{"item_id":"senseval2.d001.s037.t002","gold_sense_keys":["go_back%2:42:00::"],"candidates":[{"index":1,"sense_key":"go_back%2:42:00::","synset_id":"go_back.v.01"},{"index":2,"sense_key":"go_back%2:32:00::","synset_id":"recur.v.02"},{"index":3,"sense_key":"go_back%2:30:00::","synset_id":"recover.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"go_back%2:42:00::","call_ids":["senseval2.d001.s037.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"go_back%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":746,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001612,"input_uncached_usd":0.0014919999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.367278250050731}
{"item_id":"senseval2.d001.s037.t005","gold_sense_keys":["name%2:32:03::"],"candidates":[{"index":1,"sense_key":"name%2:32:03::","synset_id":"name.v.01"},{"index":2,"sense_key":"name%2:32:02::","synset_id":"name.v.02"},{"index":3,"sense_key":"name%2:41:01::","synset_id":"name.v.03"},{"index":4,"sense_key":"name%2:41:00::","synset_id":"appoint.v.01"},{"index":5,"sense_key":"name%2:32:07::","synset_id":"name.v.05"},{"index":6,"sense_key":"name%2:32:04::","synset_id":"mention.v.01"},{"index":7,"sense_key":"name%2:31:01::","synset_id":"identify.v.05"},{"index":8,"sense_key":"name%2:32:09::","synset_id":"list.v.01"},{"index":9,"sense_key":"name%2:31:00::","synset_id":"diagnose.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"name%2:32:03::","call_ids":["senseval2.d001.s037.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"name%2:32:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1128,"cached_input_tokens":0,"output_tokens":109,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003346,"input_uncached_usd":0.0022559999999999998,"input_cached_usd":0.0,"output_usd":0.00109,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4328043749555945}
{"item_id":"senseval2.d001.s037.t006","gold_sense_keys":["propose%2:31:00::"],"candidates":[{"index":1,"sense_key":"propose%2:32:00::","synset_id":"propose.v.01"},{"index":2,"sense_key":"propose%2:31:00::","synset_id":"project.v.08"},{"index":3,"sense_key":"propose%2:31:01::","synset_id":"aim.v.02"},{"index":4,"sense_key":"propose%2:41:00::","synset_id":"nominate.v.02"},{"index":5,"sense_key":"propose%2:32:01::","synset_id":"propose.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"propose%2:31:00::","call_ids":["senseval2.d001.s037.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"propose%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":933,"cached_input_tokens":0,"output_tokens":99,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002856,"input_uncached_usd":0.0018659999999999998,"input_cached_usd":0.0,"output_usd":0.00099,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8791490419534966}
{"item_id":"senseval2.d001.s037.t008","gold_sense_keys":["stem%2:42:00::"],"candidates":[{"index":1,"sense_key":"stem%2:42:00::","synset_id":"stem.v.01"},{"index":2,"sense_key":"stem%2:42:03::","synset_id":"stem.v.02"},{"index":3,"sense_key":"stem%2:30:00::","synset_id":"stem.v.03"},{"index":4,"sense_key":"stem%2:30:01::","synset_id":"stem.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"stem%2:42:00::","call_ids":["senseval2.d001.s037.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"stem%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":780,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00168,"input_uncached_usd":0.00156,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.102049749926664}
{"item_id":"senseval2.d001.s037.t009","gold_sense_keys":["separate%3:00:00::"],"candidates":[{"index":1,"sense_key":"separate%3:00:00::","synset_id":"separate.a.01"},{"index":2,"sense_key":"separate%5:00:00:detached:00","synset_id":"freestanding.s.01"},{"index":3,"sense_key":"separate%5:00:02:segregated:00","synset_id":"separate.s.02"},{"index":4,"sense_key":"separate%5:00:02:unconnected:00","synset_id":"disjoined.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"separate%3:00:00::","call_ids":["senseval2.d001.s037.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"separate%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":809,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001738,"input_uncached_usd":0.0016179999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.483079000026919}
{"item_id":"senseval2.d001.s037.t010","gold_sense_keys":["genetic%3:01:02::"],"candidates":[{"index":1,"sense_key":"genetic%5:00:00:inheritable:00","synset_id":"familial.s.01"},{"index":2,"sense_key":"genetic%3:01:02::","synset_id":"genic.a.01"},{"index":3,"sense_key":"genetic%3:01:01::","synset_id":"genetic.a.03"},{"index":4,"sense_key":"genetic%3:01:00::","synset_id":"genetic.a.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"genetic%3:01:02::","call_ids":["senseval2.d001.s037.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"genetic%3:01:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":828,"cached_input_tokens":0,"output_tokens":156,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003216,"input_uncached_usd":0.001656,"input_cached_usd":0.0,"output_usd":0.0015600000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.204335834016092}
{"item_id":"senseval2.d001.s037.t011","gold_sense_keys":["defect%1:26:01::"],"candidates":[{"index":1,"sense_key":"defect%1:26:01::","synset_id":"defect.n.01"},{"index":2,"sense_key":"defect%1:07:01::","synset_id":"defect.n.02"},{"index":3,"sense_key":"defect%1:26:00::","synset_id":"defect.n.03"},{"index":4,"sense_key":"defect%1:07:00::","synset_id":"blemish.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"defect%1:26:01::","call_ids":["senseval2.d001.s037.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"defect%1:26:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":845,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00181,"input_uncached_usd":0.0016899999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1273474580375478}
{"item_id":"senseval2.d001.s038.t000","gold_sense_keys":["theorize%2:31:00::"],"candidates":[{"index":1,"sense_key":"theorize%2:31:00::","synset_id":"speculate.v.01"},{"index":2,"sense_key":"theorize%2:31:02::","synset_id":"theorize.v.02"},{"index":3,"sense_key":"theorize%2:31:01::","synset_id":"theorize.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"theorize%2:31:00::","call_ids":["senseval2.d001.s038.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"theorize%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":747,"cached_input_tokens":0,"output_tokens":76,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002254,"input_uncached_usd":0.0014939999999999999,"input_cached_usd":0.0,"output_usd":0.00076,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0067639589542523}
{"item_id":"senseval2.d001.s038.t001","gold_sense_keys":["eye%1:08:00::"],"candidates":[{"index":1,"sense_key":"eye%1:08:00::","synset_id":"eye.n.01"},{"index":2,"sense_key":"eye%1:09:00::","synset_id":"eye.n.02"},{"index":3,"sense_key":"eye%1:09:01::","synset_id":"eye.n.03"},{"index":4,"sense_key":"eye%1:15:00::","synset_id":"center.n.01"},{"index":5,"sense_key":"eye%1:06:00::","synset_id":"eye.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"eye%1:08:00::","call_ids":["senseval2.d001.s038.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"eye%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":825,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00177,"input_uncached_usd":0.00165,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0071831250097603}
{"item_id":"senseval2.d001.s038.t002","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s038.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":870,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00186,"input_uncached_usd":0.00174,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0417914160061628}
{"item_id":"senseval2.d001.s038.t004","gold_sense_keys":["inherit%2:40:01::"],"candidates":[{"index":1,"sense_key":"inherit%2:40:00::","synset_id":"inherit.v.01"},{"index":2,"sense_key":"inherit%2:40:02::","synset_id":"inherit.v.02"},{"index":3,"sense_key":"inherit%2:40:01::","synset_id":"inherit.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"inherit%2:40:01::","call_ids":["senseval2.d001.s038.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"inherit%2:40:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":707,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001534,"input_uncached_usd":0.001414,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1308011250803247}
{"item_id":"senseval2.d001.s038.t005","gold_sense_keys":["damaged%3:00:00::"],"candidates":[{"index":1,"sense_key":"damaged%3:00:00::","synset_id":"damaged.a.01"},{"index":2,"sense_key":"damaged%5:00:00:disreputable:00","synset_id":"discredited.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"damaged%3:00:00::","call_ids":["senseval2.d001.s038.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"damaged%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":688,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001496,"input_uncached_usd":0.0013759999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5199322090484202}
{"item_id":"senseval2.d001.s038.t006","gold_sense_keys":["copy%1:06:00::"],"candidates":[{"index":1,"sense_key":"copy%1:10:00::","synset_id":"transcript.n.02"},{"index":2,"sense_key":"copy%1:06:00::","synset_id":"copy.n.02"},{"index":3,"sense_key":"copy%1:10:01::","synset_id":"copy.n.03"},{"index":4,"sense_key":"copy%1:10:02::","synset_id":"copy.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"copy%1:06:00::","call_ids":["senseval2.d001.s038.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"copy%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":766,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001652,"input_uncached_usd":0.001532,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0681272080400959}
{"item_id":"senseval2.d001.s038.t008","gold_sense_keys":["parent%1:18:00::"],"candidates":[{"index":1,"sense_key":"parent%1:18:00::","synset_id":"parent.n.01"},{"index":2,"sense_key":"parent%1:03:00::","synset_id":"parent.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"parent%1:18:00::","call_ids":["senseval2.d001.s038.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"parent%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":658,"cached_input_tokens":0,"output_tokens":151,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002826,"input_uncached_usd":0.001316,"input_cached_usd":0.0,"output_usd":0.00151,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.056985207949765}
{"item_id":"senseval2.d001.s038.t009","gold_sense_keys":["normal%3:00:01::"],"candidates":[{"index":1,"sense_key":"normal%3:00:01::","synset_id":"normal.a.01"},{"index":2,"sense_key":"normal%3:00:02::","synset_id":"normal.a.02"},{"index":3,"sense_key":"normal%3:00:03::","synset_id":"normal.a.03"},{"index":4,"sense_key":"normal%5:00:00:perpendicular:00","synset_id":"normal.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"normal%3:00:01::","call_ids":["senseval2.d001.s038.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"normal%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":759,"cached_input_tokens":0,"output_tokens":144,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002958,"input_uncached_usd":0.001518,"input_cached_usd":0.0,"output_usd":0.00144,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.3474138329038396}
{"item_id":"senseval2.d001.s038.t010","gold_sense_keys":["copy%1:06:00::"],"candidates":[{"index":1,"sense_key":"copy%1:10:00::","synset_id":"transcript.n.02"},{"index":2,"sense_key":"copy%1:06:00::","synset_id":"copy.n.02"},{"index":3,"sense_key":"copy%1:10:01::","synset_id":"copy.n.03"},{"index":4,"sense_key":"copy%1:10:02::","synset_id":"copy.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"copy%1:06:00::","call_ids":["senseval2.d001.s038.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"copy%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":766,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001652,"input_uncached_usd":0.001532,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.456756709027104}
{"item_id":"senseval2.d001.s039.t002","gold_sense_keys":["develop%2:30:00::","develop%2:42:00::"],"candidates":[{"index":1,"sense_key":"develop%2:36:01::","synset_id":"develop.v.01"},{"index":2,"sense_key":"develop%2:36:00::","synset_id":"evolve.v.01"},{"index":3,"sense_key":"develop%2:30:06::","synset_id":"develop.v.03"},{"index":4,"sense_key":"develop%2:29:00::","synset_id":"grow.v.08"},{"index":5,"sense_key":"develop%2:42:00::","synset_id":"originate.v.01"},{"index":6,"sense_key":"develop%2:30:04::","synset_id":"build_up.v.05"},{"index":7,"sense_key":"develop%2:32:00::","synset_id":"explicate.v.02"},{"index":8,"sense_key":"develop%2:31:00::","synset_id":"train.v.01"},{"index":9,"sense_key":"develop%2:30:05::","synset_id":"develop.v.09"},{"index":10,"sense_key":"develop%2:30:00::","synset_id":"develop.v.10"},{"index":11,"sense_key":"develop%2:30:09::","synset_id":"modernize.v.02"},{"index":12,"sense_key":"develop%2:30:01::","synset_id":"develop.v.12"},{"index":13,"sense_key":"develop%2:36:09::","synset_id":"develop.v.13"},{"index":14,"sense_key":"develop%2:30:10::","synset_id":"develop.v.14"},{"index":15,"sense_key":"develop%2:39:00::","synset_id":"develop.v.15"},{"index":16,"sense_key":"develop%2:35:04::","synset_id":"develop.v.16"},{"index":17,"sense_key":"develop%2:33:03::","synset_id":"develop.v.17"},{"index":18,"sense_key":"develop%2:33:02::","synset_id":"develop.v.18"},{"index":19,"sense_key":"develop%2:30:13::","synset_id":"develop.v.19"},{"index":20,"sense_key":"develop%2:30:03::","synset_id":"break.v.53"},{"index":21,"sense_key":"develop%2:30:12::","synset_id":"develop.v.21"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"develop%2:42:00::","call_ids":["senseval2.d001.s039.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"develop%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1909,"cached_input_tokens":0,"output_tokens":171,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.005528,"input_uncached_usd":0.0038179999999999998,"input_cached_usd":0.0,"output_usd":0.0017100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1704588750144467}
{"item_id":"senseval2.d001.s039.t003","gold_sense_keys":["second%5:00:00:ordinal:00"],"candidates":[{"index":1,"sense_key":"second%5:00:00:ordinal:00","synset_id":"second.s.01"},{"index":2,"sense_key":"second%3:00:00::","synset_id":"second.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"second%5:00:00:ordinal:00","call_ids":["senseval2.d001.s039.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"second%5:00:00:ordinal:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":651,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001422,"input_uncached_usd":0.001302,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.7725425419630483}
{"item_id":"senseval2.d001.s039.t004","gold_sense_keys":["normal%3:00:01::"],"candidates":[{"index":1,"sense_key":"normal%3:00:01::","synset_id":"normal.a.01"},{"index":2,"sense_key":"normal%3:00:02::","synset_id":"normal.a.02"},{"index":3,"sense_key":"normal%3:00:03::","synset_id":"normal.a.03"},{"index":4,"sense_key":"normal%5:00:00:perpendicular:00","synset_id":"normal.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"normal%3:00:01::","call_ids":["senseval2.d001.s039.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"normal%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":736,"cached_input_tokens":0,"output_tokens":96,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002432,"input_uncached_usd":0.001472,"input_cached_usd":0.0,"output_usd":0.0009600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9692353749414906}
{"item_id":"senseval2.d001.s039.t005","gold_sense_keys":["copy%1:06:00::"],"candidates":[{"index":1,"sense_key":"copy%1:10:00::","synset_id":"transcript.n.02"},{"index":2,"sense_key":"copy%1:06:00::","synset_id":"copy.n.02"},{"index":3,"sense_key":"copy%1:10:01::","synset_id":"copy.n.03"},{"index":4,"sense_key":"copy%1:10:02::","synset_id":"copy.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"copy%1:06:00::","call_ids":["senseval2.d001.s039.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"copy%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":743,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001606,"input_uncached_usd":0.001486,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4372192081063986}
{"item_id":"senseval2.d001.s039.t007","gold_sense_keys":["damaged%3:00:00::"],"candidates":[{"index":1,"sense_key":"damaged%3:00:00::","synset_id":"damaged.a.01"},{"index":2,"sense_key":"damaged%5:00:00:disreputable:00","synset_id":"discredited.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"damaged%3:00:00::","call_ids":["senseval2.d001.s039.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"damaged%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":665,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0014500000000000001,"input_uncached_usd":0.00133,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1282430840656161}
{"item_id":"senseval2.d001.s040.t000","gold_sense_keys":["way%1:04:01::","way%1:07:01::"],"candidates":[{"index":1,"sense_key":"way%1:07:01::","synset_id":"manner.n.01"},{"index":2,"sense_key":"way%1:04:01::","synset_id":"means.n.01"},{"index":3,"sense_key":"way%1:15:00::","synset_id":"direction.n.01"},{"index":4,"sense_key":"way%1:26:00::","synset_id":"way.n.04"},{"index":5,"sense_key":"way%1:04:00::","synset_id":"way.n.05"},{"index":6,"sense_key":"way%1:06:00::","synset_id":"way.n.06"},{"index":7,"sense_key":"way%1:04:02::","synset_id":"way.n.07"},{"index":8,"sense_key":"way%1:23:00::","synset_id":"room.n.02"},{"index":9,"sense_key":"way%1:07:02::","synset_id":"way.n.09"},{"index":10,"sense_key":"way%1:09:00::","synset_id":"way.n.10"},{"index":11,"sense_key":"way%1:09:01::","synset_id":"way.n.11"},{"index":12,"sense_key":"way%1:21:00::","synset_id":"way.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"way%1:04:01::","call_ids":["senseval2.d001.s040.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"way%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1192,"cached_input_tokens":0,"output_tokens":91,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003294,"input_uncached_usd":0.002384,"input_cached_usd":0.0,"output_usd":0.0009100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8337695829104632}
{"item_id":"senseval2.d001.s040.t001","gold_sense_keys":["prove%2:31:00::"],"candidates":[{"index":1,"sense_key":"prove%2:42:00::","synset_id":"prove.v.01"},{"index":2,"sense_key":"prove%2:31:00::","synset_id":"prove.v.02"},{"index":3,"sense_key":"prove%2:32:00::","synset_id":"testify.v.02"},{"index":4,"sense_key":"prove%2:31:02::","synset_id":"prove.v.04"},{"index":5,"sense_key":"prove%2:41:03::","synset_id":"test.v.01"},{"index":6,"sense_key":"prove%2:38:01::","synset_id":"rise.v.15"},{"index":7,"sense_key":"prove%2:38:00::","synset_id":"raise.v.15"},{"index":8,"sense_key":"prove%2:36:00::","synset_id":"prove.v.08"},{"index":9,"sense_key":"prove%2:31:01::","synset_id":"prove.v.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"prove%2:31:00::","call_ids":["senseval2.d001.s040.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"prove%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1108,"cached_input_tokens":0,"output_tokens":102,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003236,"input_uncached_usd":0.002216,"input_cached_usd":0.0,"output_usd":0.00102,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.138874833006412}
{"item_id":"senseval2.d001.s040.t002","gold_sense_keys":["theory%1:09:01::"],"candidates":[{"index":1,"sense_key":"theory%1:09:00::","synset_id":"theory.n.01"},{"index":2,"sense_key":"theory%1:09:01::","synset_id":"hypothesis.n.02"},{"index":3,"sense_key":"theory%1:09:02::","synset_id":"theory.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"theory%1:09:01::","call_ids":["senseval2.d001.s040.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"theory%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":884,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0023279999999999998,"input_uncached_usd":0.0017679999999999998,"input_cached_usd":0.0,"output_usd":0.0005600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8292470829328522}
{"item_id":"senseval2.d001.s041.t000","gold_sense_keys":["back%4:02:01::"],"candidates":[{"index":1,"sense_key":"back%4:02:04::","synset_id":"back.r.01"},{"index":2,"sense_key":"back%4:02:00::","synset_id":"back.r.02"},{"index":3,"sense_key":"back%4:02:05::","synset_id":"back.r.03"},{"index":4,"sense_key":"back%4:02:01::","synset_id":"back.r.04"},{"index":5,"sense_key":"back%4:02:06::","synset_id":"back.r.05"},{"index":6,"sense_key":"back%4:02:03::","synset_id":"back.r.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"back%4:02:01::","call_ids":["senseval2.d001.s041.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"back%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":829,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002198,"input_uncached_usd":0.001658,"input_cached_usd":0.0,"output_usd":0.00054,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":5.288230250007473}
{"item_id":"senseval2.d001.s041.t001","gold_sense_keys":["then%4:02:01::"],"candidates":[{"index":1,"sense_key":"then%4:02:00::","synset_id":"then.r.01"},{"index":2,"sense_key":"then%4:02:02::","synset_id":"then.r.02"},{"index":3,"sense_key":"then%4:02:01::","synset_id":"then.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"then%4:02:01::","call_ids":["senseval2.d001.s041.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"then%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":687,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001494,"input_uncached_usd":0.001374,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.09867812495213}
{"item_id":"senseval2.d001.s041.t003","gold_sense_keys":["way%1:04:01::","way%1:07:01::"],"candidates":[{"index":1,"sense_key":"way%1:07:01::","synset_id":"manner.n.01"},{"index":2,"sense_key":"way%1:04:01::","synset_id":"means.n.01"},{"index":3,"sense_key":"way%1:15:00::","synset_id":"direction.n.01"},{"index":4,"sense_key":"way%1:26:00::","synset_id":"way.n.04"},{"index":5,"sense_key":"way%1:04:00::","synset_id":"way.n.05"},{"index":6,"sense_key":"way%1:06:00::","synset_id":"way.n.06"},{"index":7,"sense_key":"way%1:04:02::","synset_id":"way.n.07"},{"index":8,"sense_key":"way%1:23:00::","synset_id":"room.n.02"},{"index":9,"sense_key":"way%1:07:02::","synset_id":"way.n.09"},{"index":10,"sense_key":"way%1:09:00::","synset_id":"way.n.10"},{"index":11,"sense_key":"way%1:09:01::","synset_id":"way.n.11"},{"index":12,"sense_key":"way%1:21:00::","synset_id":"way.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"way%1:04:01::","call_ids":["senseval2.d001.s041.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"way%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1142,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002964,"input_uncached_usd":0.002284,"input_cached_usd":0.0,"output_usd":0.00068,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.4101447080029175}
{"item_id":"senseval2.d001.s041.t008","gold_sense_keys":["see%2:39:00::"],"candidates":[{"index":1,"sense_key":"see%2:39:00::","synset_id":"see.v.01"},{"index":2,"sense_key":"see%2:31:01::","synset_id":"understand.v.02"},{"index":3,"sense_key":"see%2:39:02::","synset_id":"witness.v.02"},{"index":4,"sense_key":"see%2:36:00::","synset_id":"visualize.v.01"},{"index":5,"sense_key":"see%2:31:00::","synset_id":"see.v.05"},{"index":6,"sense_key":"see%2:31:02::","synset_id":"learn.v.02"},{"index":7,"sense_key":"see%2:39:03::","synset_id":"watch.v.03"},{"index":8,"sense_key":"see%2:38:00::","synset_id":"meet.v.01"},{"index":9,"sense_key":"see%2:32:00::","synset_id":"determine.v.08"},{"index":10,"sense_key":"see%2:31:03::","synset_id":"see.v.10"},{"index":11,"sense_key":"see%2:41:11::","synset_id":"see.v.11"},{"index":12,"sense_key":"see%2:41:12::","synset_id":"see.v.12"},{"index":13,"sense_key":"see%2:41:10::","synset_id":"visit.v.01"},{"index":14,"sense_key":"see%2:41:14::","synset_id":"attend.v.02"},{"index":15,"sense_key":"see%2:35:09::","synset_id":"see.v.15"},{"index":16,"sense_key":"see%2:41:00::","synset_id":"go_steady.v.01"},{"index":17,"sense_key":"see%2:39:05::","synset_id":"see.v.17"},{"index":18,"sense_key":"see%2:31:08::","synset_id":"see.v.18"},{"index":19,"sense_key":"see%2:39:11::","synset_id":"see.v.19"},{"index":20,"sense_key":"see%2:39:13::","synset_id":"examine.v.02"},{"index":21,"sense_key":"see%2:39:12::","synset_id":"experience.v.01"},{"index":22,"sense_key":"see%2:38:03::","synset_id":"see.v.22"},{"index":23,"sense_key":"see%2:33:00::","synset_id":"see.v.23"},{"index":24,"sense_key":"see%2:31:13::","synset_id":"interpret.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"see%2:39:00::","call_ids":["senseval2.d001.s041.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"see%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2078,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004766,"input_uncached_usd":0.004156,"input_cached_usd":0.0,"output_usd":0.0006100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.805492499959655}
{"item_id":"senseval2.d001.s041.t009","gold_sense_keys":["pair%1:14:01::"],"candidates":[{"index":1,"sense_key":"pair%1:14:01::","synset_id":"pair.n.01"},{"index":2,"sense_key":"pair%1:23:00::","synset_id":"couple.n.04"},{"index":3,"sense_key":"pair%1:14:00::","synset_id":"pair.n.03"},{"index":4,"sense_key":"pair%1:14:02::","synset_id":"pair.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"pair%1:23:00::","call_ids":["senseval2.d001.s041.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"pair%1:23:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":689,"cached_input_tokens":0,"output_tokens":176,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003138,"input_uncached_usd":0.0013779999999999999,"input_cached_usd":0.0,"output_usd":0.00176,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0203325839247555}
{"item_id":"senseval2.d001.s041.t011","gold_sense_keys":["cell%1:03:00::"],"candidates":[{"index":1,"sense_key":"cell%1:06:03::","synset_id":"cell.n.01"},{"index":2,"sense_key":"cell%1:03:00::","synset_id":"cell.n.02"},{"index":3,"sense_key":"cell%1:06:00::","synset_id":"cell.n.03"},{"index":4,"sense_key":"cell%1:14:00::","synset_id":"cell.n.04"},{"index":5,"sense_key":"cell%1:06:04::","synset_id":"cellular_telephone.n.01"},{"index":6,"sense_key":"cell%1:06:02::","synset_id":"cell.n.06"},{"index":7,"sense_key":"cell%1:06:01::","synset_id":"cell.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cell%1:03:00::","call_ids":["senseval2.d001.s041.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cell%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":934,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0019879999999999997,"input_uncached_usd":0.0018679999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8790340420091525}
{"item_id":"senseval2.d001.s041.t012","gold_sense_keys":["contain%2:42:13::"],"candidates":[{"index":1,"sense_key":"contain%2:42:00::","synset_id":"incorporate.v.02"},{"index":2,"sense_key":"contain%2:42:13::","synset_id":"hold.v.11"},{"index":3,"sense_key":"contain%2:41:00::","synset_id":"control.v.02"},{"index":4,"sense_key":"contain%2:42:06::","synset_id":"contain.v.04"},{"index":5,"sense_key":"contain%2:42:14::","synset_id":"contain.v.05"},{"index":6,"sense_key":"contain%2:33:00::","synset_id":"check.v.18"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"contain%2:42:00::","call_ids":["senseval2.d001.s041.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"contain%2:42:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":958,"cached_input_tokens":0,"output_tokens":78,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002696,"input_uncached_usd":0.001916,"input_cached_usd":0.0,"output_usd":0.0007800000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.745983749977313}
{"item_id":"senseval2.d001.s042.t001","gold_sense_keys":["gross%5:00:00:visible:00"],"candidates":[{"index":1,"sense_key":"gross%3:00:00::","synset_id":"gross.a.01"},{"index":2,"sense_key":"gross%5:00:00:general:00","synset_id":"gross.s.01"},{"index":3,"sense_key":"gross%5:00:00:fat:01","synset_id":"gross.s.02"},{"index":4,"sense_key":"gross%5:00:00:visible:00","synset_id":"megascopic.s.01"},{"index":5,"sense_key":"gross%5:00:00:unmitigated:00","synset_id":"arrant.s.01"},{"index":6,"sense_key":"gross%5:00:00:indecent:00","synset_id":"crude.s.02"},{"index":7,"sense_key":"gross%5:00:00:conspicuous:00","synset_id":"crying.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"gross%5:00:00:visible:00","call_ids":["senseval2.d001.s042.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"gross%5:00:00:visible:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":973,"cached_input_tokens":0,"output_tokens":141,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0033559999999999996,"input_uncached_usd":0.0019459999999999998,"input_cached_usd":0.0,"output_usd":0.00141,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9117872919887304}
{"item_id":"senseval2.d001.s042.t003","gold_sense_keys":["damage%1:11:00::"],"candidates":[{"index":1,"sense_key":"damage%1:11:00::","synset_id":"damage.n.01"},{"index":2,"sense_key":"damage%1:11:01::","synset_id":"damage.n.02"},{"index":3,"sense_key":"damage%1:04:00::","synset_id":"damage.n.03"},{"index":4,"sense_key":"damage%1:21:00::","synset_id":"price.n.02"},{"index":5,"sense_key":"damage%1:04:01::","synset_id":"wrong.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"damage%1:11:00::","call_ids":["senseval2.d001.s042.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"damage%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":774,"cached_input_tokens":0,"output_tokens":69,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002238,"input_uncached_usd":0.001548,"input_cached_usd":0.0,"output_usd":0.0006900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9273896660888568}
{"item_id":"senseval2.d001.s042.t004","gold_sense_keys":["visible%3:00:00::"],"candidates":[{"index":1,"sense_key":"visible%3:00:00::","synset_id":"visible.a.01"},{"index":2,"sense_key":"visible%5:00:00:overt:00","synset_id":"visible.s.01"},{"index":3,"sense_key":"visible%5:00:00:available:00","synset_id":"visible.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"visible%3:00:00::","call_ids":["senseval2.d001.s042.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"visible%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":682,"cached_input_tokens":0,"output_tokens":138,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002744,"input_uncached_usd":0.001364,"input_cached_usd":0.0,"output_usd":0.0013800000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.359408042044379}
{"item_id":"senseval2.d001.s043.t000","gold_sense_keys":["find%2:31:09::","find%2:32:00::","find%2:36:00::","find%2:39:02::"],"candidates":[{"index":1,"sense_key":"find%2:40:02::","synset_id":"find.v.01"},{"index":2,"sense_key":"find%2:39:02::","synset_id":"detect.v.01"},{"index":3,"sense_key":"find%2:40:00::","synset_id":"find.v.03"},{"index":4,"sense_key":"find%2:32:00::","synset_id":"determine.v.01"},{"index":5,"sense_key":"find%2:31:10::","synset_id":"find.v.05"},{"index":6,"sense_key":"find%2:39:01::","synset_id":"witness.v.02"},{"index":7,"sense_key":"find%2:40:01::","synset_id":"line_up.v.02"},{"index":8,"sense_key":"find%2:36:00::","synset_id":"discover.v.03"},{"index":9,"sense_key":"find%2:31:09::","synset_id":"discover.v.04"},{"index":10,"sense_key":"find%2:40:03::","synset_id":"find.v.10"},{"index":11,"sense_key":"find%2:32:01::","synset_id":"rule.v.04"},{"index":12,"sense_key":"find%2:30:00::","synset_id":"receive.v.02"},{"index":13,"sense_key":"find%2:39:05::","synset_id":"find.v.13"},{"index":14,"sense_key":"find%2:40:15::","synset_id":"recover.v.01"},{"index":15,"sense_key":"find%2:38:10::","synset_id":"find.v.15"},{"index":16,"sense_key":"find%2:30:13::","synset_id":"find_oneself.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"find%2:32:00::","call_ids":["senseval2.d001.s043.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"find%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1624,"cached_input_tokens":0,"output_tokens":281,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.006058000000000001,"input_uncached_usd":0.003248,"input_cached_usd":0.0,"output_usd":0.0028100000000000004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.956529917079024}
{"item_id":"senseval2.d001.s043.t001","gold_sense_keys":["child%1:18:00::"],"candidates":[{"index":1,"sense_key":"child%1:18:00::","synset_id":"child.n.01"},{"index":2,"sense_key":"child%1:18:01::","synset_id":"child.n.02"},{"index":3,"sense_key":"child%1:18:06::","synset_id":"child.n.03"},{"index":4,"sense_key":"child%1:18:07::","synset_id":"child.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"child%1:18:01::","call_ids":["senseval2.d001.s043.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"child%1:18:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":711,"cached_input_tokens":0,"output_tokens":137,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002792,"input_uncached_usd":0.0014219999999999999,"input_cached_usd":0.0,"output_usd":0.0013700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9590050840051845}
{"item_id":"senseval2.d001.s043.t002","gold_sense_keys":["eye%1:08:00::"],"candidates":[{"index":1,"sense_key":"eye%1:08:00::","synset_id":"eye.n.01"},{"index":2,"sense_key":"eye%1:09:00::","synset_id":"eye.n.02"},{"index":3,"sense_key":"eye%1:09:01::","synset_id":"eye.n.03"},{"index":4,"sense_key":"eye%1:15:00::","synset_id":"center.n.01"},{"index":5,"sense_key":"eye%1:06:00::","synset_id":"eye.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"eye%1:08:00::","call_ids":["senseval2.d001.s043.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"eye%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":747,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001614,"input_uncached_usd":0.0014939999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9150426670676097}
{"item_id":"senseval2.d001.s043.t003","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s043.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":792,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001704,"input_uncached_usd":0.001584,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0896601249696687}
{"item_id":"senseval2.d001.s043.t004","gold_sense_keys":["inherit%2:40:01::"],"candidates":[{"index":1,"sense_key":"inherit%2:40:00::","synset_id":"inherit.v.01"},{"index":2,"sense_key":"inherit%2:40:02::","synset_id":"inherit.v.02"},{"index":3,"sense_key":"inherit%2:40:01::","synset_id":"inherit.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"inherit%2:40:01::","call_ids":["senseval2.d001.s043.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"inherit%2:40:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":629,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001378,"input_uncached_usd":0.001258,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0844045829726383}
{"item_id":"senseval2.d001.s043.t005","gold_sense_keys":["damaged%3:00:00::"],"candidates":[{"index":1,"sense_key":"damaged%3:00:00::","synset_id":"damaged.a.01"},{"index":2,"sense_key":"damaged%5:00:00:disreputable:00","synset_id":"discredited.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"damaged%3:00:00::","call_ids":["senseval2.d001.s043.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"damaged%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":610,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00134,"input_uncached_usd":0.00122,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5877787499921396}
{"item_id":"senseval2.d001.s043.t006","gold_sense_keys":["copy%1:06:00::"],"candidates":[{"index":1,"sense_key":"copy%1:10:00::","synset_id":"transcript.n.02"},{"index":2,"sense_key":"copy%1:06:00::","synset_id":"copy.n.02"},{"index":3,"sense_key":"copy%1:10:01::","synset_id":"copy.n.03"},{"index":4,"sense_key":"copy%1:10:02::","synset_id":"copy.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"copy%1:06:00::","call_ids":["senseval2.d001.s043.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"copy%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":688,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001496,"input_uncached_usd":0.0013759999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.046731667011045}
{"item_id":"senseval2.d001.s043.t009","gold_sense_keys":["parent%1:18:00::"],"candidates":[{"index":1,"sense_key":"parent%1:18:00::","synset_id":"parent.n.01"},{"index":2,"sense_key":"parent%1:03:00::","synset_id":"parent.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"parent%1:18:00::","call_ids":["senseval2.d001.s043.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"parent%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":580,"cached_input_tokens":0,"output_tokens":117,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00233,"input_uncached_usd":0.00116,"input_cached_usd":0.0,"output_usd":0.00117,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7163554170401767}
{"item_id":"senseval2.d001.s044.t001","gold_sense_keys":["actually%4:02:01::"],"candidates":[{"index":1,"sense_key":"actually%4:02:01::","synset_id":"actually.r.01"},{"index":2,"sense_key":"actually%4:02:00::","synset_id":"actually.r.02"},{"index":3,"sense_key":"actually%4:02:03::","synset_id":"actually.r.03"},{"index":4,"sense_key":"actually%4:02:02::","synset_id":"actually.r.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"actually%4:02:01::","call_ids":["senseval2.d001.s044.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"actually%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":730,"cached_input_tokens":0,"output_tokens":125,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0027099999999999997,"input_uncached_usd":0.00146,"input_cached_usd":0.0,"output_usd":0.00125,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6935897920047864}
{"item_id":"senseval2.d001.s044.t002","gold_sense_keys":["see%2:39:00::"],"candidates":[{"index":1,"sense_key":"see%2:39:00::","synset_id":"see.v.01"},{"index":2,"sense_key":"see%2:31:01::","synset_id":"understand.v.02"},{"index":3,"sense_key":"see%2:39:02::","synset_id":"witness.v.02"},{"index":4,"sense_key":"see%2:36:00::","synset_id":"visualize.v.01"},{"index":5,"sense_key":"see%2:31:00::","synset_id":"see.v.05"},{"index":6,"sense_key":"see%2:31:02::","synset_id":"learn.v.02"},{"index":7,"sense_key":"see%2:39:03::","synset_id":"watch.v.03"},{"index":8,"sense_key":"see%2:38:00::","synset_id":"meet.v.01"},{"index":9,"sense_key":"see%2:32:00::","synset_id":"determine.v.08"},{"index":10,"sense_key":"see%2:31:03::","synset_id":"see.v.10"},{"index":11,"sense_key":"see%2:41:11::","synset_id":"see.v.11"},{"index":12,"sense_key":"see%2:41:12::","synset_id":"see.v.12"},{"index":13,"sense_key":"see%2:41:10::","synset_id":"visit.v.01"},{"index":14,"sense_key":"see%2:41:14::","synset_id":"attend.v.02"},{"index":15,"sense_key":"see%2:35:09::","synset_id":"see.v.15"},{"index":16,"sense_key":"see%2:41:00::","synset_id":"go_steady.v.01"},{"index":17,"sense_key":"see%2:39:05::","synset_id":"see.v.17"},{"index":18,"sense_key":"see%2:31:08::","synset_id":"see.v.18"},{"index":19,"sense_key":"see%2:39:11::","synset_id":"see.v.19"},{"index":20,"sense_key":"see%2:39:13::","synset_id":"examine.v.02"},{"index":21,"sense_key":"see%2:39:12::","synset_id":"experience.v.01"},{"index":22,"sense_key":"see%2:38:03::","synset_id":"see.v.22"},{"index":23,"sense_key":"see%2:33:00::","synset_id":"see.v.23"},{"index":24,"sense_key":"see%2:31:13::","synset_id":"interpret.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"see%2:39:00::","call_ids":["senseval2.d001.s044.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"see%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2020,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00466,"input_uncached_usd":0.00404,"input_cached_usd":0.0,"output_usd":0.00062,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4208573749056086}
{"item_id":"senseval2.d001.s044.t003","gold_sense_keys":["bit%1:17:00::","bit%1:23:01::"],"candidates":[{"index":1,"sense_key":"bit%1:23:01::","synset_id":"spot.n.10"},{"index":2,"sense_key":"bit%1:17:00::","synset_id":"bit.n.02"},{"index":3,"sense_key":"bit%1:28:00::","synset_id":"moment.n.02"},{"index":4,"sense_key":"bit%1:11:00::","synset_id":"piece.n.05"},{"index":5,"sense_key":"bit%1:06:01::","synset_id":"bit.n.05"},{"index":6,"sense_key":"bit%1:23:00::","synset_id":"bit.n.06"},{"index":7,"sense_key":"bit%1:13:00::","synset_id":"morsel.n.02"},{"index":8,"sense_key":"bit%1:10:01::","synset_id":"snatch.n.01"},{"index":9,"sense_key":"bit%1:10:00::","synset_id":"act.n.04"},{"index":10,"sense_key":"bit%1:06:02::","synset_id":"bit.n.10"},{"index":11,"sense_key":"bit%1:06:00::","synset_id":"bit.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bit%1:23:01::","call_ids":["senseval2.d001.s044.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bit%1:23:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1158,"cached_input_tokens":0,"output_tokens":159,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0039059999999999997,"input_uncached_usd":0.002316,"input_cached_usd":0.0,"output_usd":0.00159,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.923198250005953}
{"item_id":"senseval2.d001.s045.t000","gold_sense_keys":["assume%2:31:00::"],"candidates":[{"index":1,"sense_key":"assume%2:31:00::","synset_id":"assume.v.01"},{"index":2,"sense_key":"assume%2:41:00::","synset_id":"assume.v.02"},{"index":3,"sense_key":"assume%2:30:00::","synset_id":"assume.v.03"},{"index":4,"sense_key":"assume%2:40:01::","synset_id":"bear.v.06"},{"index":5,"sense_key":"assume%2:38:00::","synset_id":"assume.v.05"},{"index":6,"sense_key":"assume%2:40:00::","synset_id":"assume.v.06"},{"index":7,"sense_key":"assume%2:36:00::","synset_id":"simulate.v.03"},{"index":8,"sense_key":"assume%2:35:00::","synset_id":"assume.v.08"},{"index":9,"sense_key":"assume%2:29:00::","synset_id":"wear.v.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"assume%2:31:00::","call_ids":["senseval2.d001.s045.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"assume%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1095,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00231,"input_uncached_usd":0.00219,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5117818330181763}
{"item_id":"senseval2.d001.s045.t002","gold_sense_keys":["piece%1:06:00::","piece%1:17:00::"],"candidates":[{"index":1,"sense_key":"piece%1:06:00::","synset_id":"piece.n.01"},{"index":2,"sense_key":"piece%1:06:05::","synset_id":"piece.n.02"},{"index":3,"sense_key":"piece%1:17:00::","synset_id":"part.n.03"},{"index":4,"sense_key":"piece%1:10:01::","synset_id":"musical_composition.n.01"},{"index":5,"sense_key":"piece%1:11:00::","synset_id":"piece.n.05"},{"index":6,"sense_key":"piece%1:10:02::","synset_id":"piece.n.06"},{"index":7,"sense_key":"piece%1:06:03::","synset_id":"firearm.n.01"},{"index":8,"sense_key":"piece%1:13:00::","synset_id":"piece.n.08"},{"index":9,"sense_key":"piece%1:07:00::","synset_id":"piece.n.09"},{"index":10,"sense_key":"piece%1:06:04::","synset_id":"objet_d'art.n.01"},{"index":11,"sense_key":"piece%1:28:00::","synset_id":"while.n.01"},{"index":12,"sense_key":"piece%1:21:00::","synset_id":"slice.n.01"},{"index":13,"sense_key":"piece%1:06:02::","synset_id":"man.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"piece%1:17:00::","call_ids":["senseval2.d001.s045.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"piece%1:17:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1251,"cached_input_tokens":0,"output_tokens":106,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003562,"input_uncached_usd":0.002502,"input_cached_usd":0.0,"output_usd":0.0010600000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.574068499961868}
{"item_id":"senseval2.d001.s045.t003","gold_sense_keys":["contain%2:42:00::"],"candidates":[{"index":1,"sense_key":"contain%2:42:00::","synset_id":"incorporate.v.02"},{"index":2,"sense_key":"contain%2:42:13::","synset_id":"hold.v.11"},{"index":3,"sense_key":"contain%2:41:00::","synset_id":"control.v.02"},{"index":4,"sense_key":"contain%2:42:06::","synset_id":"contain.v.04"},{"index":5,"sense_key":"contain%2:42:14::","synset_id":"contain.v.05"},{"index":6,"sense_key":"contain%2:33:00::","synset_id":"check.v.18"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"contain%2:42:00::","call_ids":["senseval2.d001.s045.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"contain%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":891,"cached_input_tokens":0,"output_tokens":76,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002542,"input_uncached_usd":0.001782,"input_cached_usd":0.0,"output_usd":0.00076,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.657092791981995}
{"item_id":"senseval2.d001.s045.t006","gold_sense_keys":["loss%1:07:00::","loss%1:21:01::"],"candidates":[{"index":1,"sense_key":"loss%1:21:01::","synset_id":"loss.n.01"},{"index":2,"sense_key":"loss%1:22:00::","synset_id":"loss.n.02"},{"index":3,"sense_key":"loss%1:04:00::","synset_id":"loss.n.03"},{"index":4,"sense_key":"loss%1:07:00::","synset_id":"loss.n.04"},{"index":5,"sense_key":"loss%1:11:01::","synset_id":"loss.n.05"},{"index":6,"sense_key":"loss%1:21:00::","synset_id":"loss.n.06"},{"index":7,"sense_key":"loss%1:11:00::","synset_id":"personnel_casualty.n.01"},{"index":8,"sense_key":"loss%1:11:02::","synset_id":"passing.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"loss%1:04:00::","call_ids":["senseval2.d001.s045.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"loss%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":924,"cached_input_tokens":0,"output_tokens":123,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003078,"input_uncached_usd":0.0018479999999999998,"input_cached_usd":0.0,"output_usd":0.0012300000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7125192090170458}
{"item_id":"senseval2.d001.s045.t007","gold_sense_keys":["critical%5:00:00:indispensable:00"],"candidates":[{"index":1,"sense_key":"critical%3:00:01::","synset_id":"critical.a.01"},{"index":2,"sense_key":"critical%3:00:04::","synset_id":"critical.a.02"},{"index":3,"sense_key":"critical%3:00:02::","synset_id":"critical.a.03"},{"index":4,"sense_key":"critical%5:00:00:indispensable:00","synset_id":"critical.s.01"},{"index":5,"sense_key":"critical%5:00:00:crucial:00","synset_id":"critical.s.02"},{"index":6,"sense_key":"critical%3:00:03::","synset_id":"critical.a.06"},{"index":7,"sense_key":"critical%3:01:00::","synset_id":"critical.a.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"critical%5:00:00:indispensable:00","call_ids":["senseval2.d001.s045.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"critical%5:00:00:indispensable:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":911,"cached_input_tokens":0,"output_tokens":142,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003242,"input_uncached_usd":0.0018219999999999998,"input_cached_usd":0.0,"output_usd":0.00142,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9145500409649685}
{"item_id":"senseval2.d001.s045.t008","gold_sense_keys":["role%1:07:00::"],"candidates":[{"index":1,"sense_key":"role%1:04:00::","synset_id":"function.n.03"},{"index":2,"sense_key":"role%1:09:00::","synset_id":"character.n.04"},{"index":3,"sense_key":"role%1:07:00::","synset_id":"function.n.02"},{"index":4,"sense_key":"role%1:04:01::","synset_id":"role.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"role%1:07:00::","call_ids":["senseval2.d001.s045.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"role%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":713,"cached_input_tokens":0,"output_tokens":175,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003176,"input_uncached_usd":0.001426,"input_cached_usd":0.0,"output_usd":0.00175,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2299254999961704}
{"item_id":"senseval2.d001.s045.t009","gold_sense_keys":["set_off%2:36:00::"],"candidates":[{"index":1,"sense_key":"set_off%2:36:00::","synset_id":"trip.v.04"},{"index":2,"sense_key":"set_off%2:38:00::","synset_id":"depart.v.03"},{"index":3,"sense_key":"set_off%2:30:02::","synset_id":"bring_out.v.04"},{"index":4,"sense_key":"set_off%2:30:00::","synset_id":"explode.v.01"},{"index":5,"sense_key":"set_off%2:42:00::","synset_id":"cancel.v.02"},{"index":6,"sense_key":"set_off%2:30:01::","synset_id":"set_off.v.06"},{"index":7,"sense_key":"set_off%2:32:00::","synset_id":"incite.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"set_off%2:36:00::","call_ids":["senseval2.d001.s045.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"set_off%2:36:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":906,"cached_input_tokens":0,"output_tokens":125,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003062,"input_uncached_usd":0.001812,"input_cached_usd":0.0,"output_usd":0.00125,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1116517080226913}
{"item_id":"senseval2.d001.s045.t010","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s045.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":775,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00167,"input_uncached_usd":0.00155,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.004741583019495}
{"item_id":"senseval2.d001.s046.t000","gold_sense_keys":["know%2:31:01::"],"candidates":[{"index":1,"sense_key":"know%2:31:01::","synset_id":"know.v.01"},{"index":2,"sense_key":"know%2:31:03::","synset_id":"know.v.02"},{"index":3,"sense_key":"know%2:31:02::","synset_id":"know.v.03"},{"index":4,"sense_key":"know%2:31:00::","synset_id":"know.v.04"},{"index":5,"sense_key":"know%2:31:04::","synset_id":"know.v.05"},{"index":6,"sense_key":"know%2:31:06::","synset_id":"acknowledge.v.06"},{"index":7,"sense_key":"know%2:31:15::","synset_id":"know.v.07"},{"index":8,"sense_key":"know%2:35:00::","synset_id":"sleep_together.v.01"},{"index":9,"sense_key":"know%2:31:11::","synset_id":"know.v.09"},{"index":10,"sense_key":"know%2:31:12::","synset_id":"know.v.10"},{"index":11,"sense_key":"know%2:31:14::","synset_id":"know.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"know%2:31:01::","call_ids":["senseval2.d001.s046.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"know%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1192,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003114,"input_uncached_usd":0.002384,"input_cached_usd":0.0,"output_usd":0.0007300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2992843330139294}
{"item_id":"senseval2.d001.s047.t000","gold_sense_keys":["then%4:02:00::"],"candidates":[{"index":1,"sense_key":"then%4:02:00::","synset_id":"then.r.01"},{"index":2,"sense_key":"then%4:02:02::","synset_id":"then.r.02"},{"index":3,"sense_key":"then%4:02:01::","synset_id":"then.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"then%4:02:00::","call_ids":["senseval2.d001.s047.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"then%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":622,"cached_input_tokens":0,"output_tokens":94,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002184,"input_uncached_usd":0.0012439999999999999,"input_cached_usd":0.0,"output_usd":0.0009400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4593833750113845}
{"item_id":"senseval2.d001.s047.t001","gold_sense_keys":["scientific%3:01:00::"],"candidates":[{"index":1,"sense_key":"scientific%3:01:00::","synset_id":"scientific.a.01"},{"index":2,"sense_key":"scientific%3:00:00::","synset_id":"scientific.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"scientific%3:01:00::","call_ids":["senseval2.d001.s047.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"scientific%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":549,"cached_input_tokens":0,"output_tokens":98,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002078,"input_uncached_usd":0.001098,"input_cached_usd":0.0,"output_usd":0.0009800000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.027159082936123}
{"item_id":"senseval2.d001.s047.t002","gold_sense_keys":["team%1:14:00::"],"candidates":[{"index":1,"sense_key":"team%1:14:00::","synset_id":"team.n.01"},{"index":2,"sense_key":"team%1:14:01::","synset_id":"team.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"team%1:14:00::","call_ids":["senseval2.d001.s047.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"team%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":535,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00119,"input_uncached_usd":0.00107,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4857869169209152}
{"item_id":"senseval2.d001.s047.t003","gold_sense_keys":["lead%2:41:00::"],"candidates":[{"index":1,"sense_key":"lead%2:38:01::","synset_id":"lead.v.01"},{"index":2,"sense_key":"lead%2:42:04::","synset_id":"leave.v.07"},{"index":3,"sense_key":"lead%2:42:12::","synset_id":"lead.v.03"},{"index":4,"sense_key":"lead%2:38:00::","synset_id":"lead.v.04"},{"index":5,"sense_key":"lead%2:32:00::","synset_id":"lead.v.05"},{"index":6,"sense_key":"lead%2:42:00::","synset_id":"run.v.03"},{"index":7,"sense_key":"lead%2:41:00::","synset_id":"head.v.02"},{"index":8,"sense_key":"lead%2:42:01::","synset_id":"lead.v.08"},{"index":9,"sense_key":"lead%2:41:12::","synset_id":"contribute.v.03"},{"index":10,"sense_key":"lead%2:36:00::","synset_id":"conduct.v.02"},{"index":11,"sense_key":"lead%2:42:03::","synset_id":"go.v.25"},{"index":12,"sense_key":"lead%2:38:02::","synset_id":"precede.v.04"},{"index":13,"sense_key":"lead%2:42:02::","synset_id":"run.v.23"},{"index":14,"sense_key":"lead%2:32:01::","synset_id":"moderate.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"lead%2:41:00::","call_ids":["senseval2.d001.s047.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"lead%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1283,"cached_input_tokens":0,"output_tokens":103,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003596,"input_uncached_usd":0.0025659999999999997,"input_cached_usd":0.0,"output_usd":0.00103,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.3450857079587877}
{"item_id":"senseval2.d001.s047.t006","gold_sense_keys":["then%4:02:01::"],"candidates":[{"index":1,"sense_key":"then%4:02:00::","synset_id":"then.r.01"},{"index":2,"sense_key":"then%4:02:02::","synset_id":"then.r.02"},{"index":3,"sense_key":"then%4:02:01::","synset_id":"then.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"then%4:02:01::","call_ids":["senseval2.d001.s047.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"then%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":621,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001892,"input_uncached_usd":0.001242,"input_cached_usd":0.0,"output_usd":0.0006500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3318773750215769}
{"item_id":"senseval2.d001.s047.t007","gold_sense_keys":["find%2:31:09::","find%2:32:00::","find%2:40:03::"],"candidates":[{"index":1,"sense_key":"find%2:40:02::","synset_id":"find.v.01"},{"index":2,"sense_key":"find%2:39:02::","synset_id":"detect.v.01"},{"index":3,"sense_key":"find%2:40:00::","synset_id":"find.v.03"},{"index":4,"sense_key":"find%2:32:00::","synset_id":"determine.v.01"},{"index":5,"sense_key":"find%2:31:10::","synset_id":"find.v.05"},{"index":6,"sense_key":"find%2:39:01::","synset_id":"witness.v.02"},{"index":7,"sense_key":"find%2:40:01::","synset_id":"line_up.v.02"},{"index":8,"sense_key":"find%2:36:00::","synset_id":"discover.v.03"},{"index":9,"sense_key":"find%2:31:09::","synset_id":"discover.v.04"},{"index":10,"sense_key":"find%2:40:03::","synset_id":"find.v.10"},{"index":11,"sense_key":"find%2:32:01::","synset_id":"rule.v.04"},{"index":12,"sense_key":"find%2:30:00::","synset_id":"receive.v.02"},{"index":13,"sense_key":"find%2:39:05::","synset_id":"find.v.13"},{"index":14,"sense_key":"find%2:40:15::","synset_id":"recover.v.01"},{"index":15,"sense_key":"find%2:38:10::","synset_id":"find.v.15"},{"index":16,"sense_key":"find%2:30:13::","synset_id":"find_oneself.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"find%2:32:00::","call_ids":["senseval2.d001.s047.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"find%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1608,"cached_input_tokens":0,"output_tokens":135,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004566,"input_uncached_usd":0.0032159999999999997,"input_cached_usd":0.0,"output_usd":0.00135,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.3744666250422597}
{"item_id":"senseval2.d001.s047.t008","gold_sense_keys":["answer%1:10:02::"],"candidates":[{"index":1,"sense_key":"answer%1:10:01::","synset_id":"answer.n.01"},{"index":2,"sense_key":"answer%1:10:02::","synset_id":"solution.n.02"},{"index":3,"sense_key":"answer%1:10:00::","synset_id":"answer.n.03"},{"index":4,"sense_key":"answer%1:10:03::","synset_id":"answer.n.04"},{"index":5,"sense_key":"answer%1:04:00::","synset_id":"answer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"answer%1:10:02::","call_ids":["senseval2.d001.s047.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"answer%1:10:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":863,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001846,"input_uncached_usd":0.0017259999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1757921250537038}
{"item_id":"senseval2.d001.s048.t000","gold_sense_keys":["team%1:14:00::"],"candidates":[{"index":1,"sense_key":"team%1:14:00::","synset_id":"team.n.01"},{"index":2,"sense_key":"team%1:14:01::","synset_id":"team.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"team%1:14:00::","call_ids":["senseval2.d001.s048.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"team%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":569,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001258,"input_uncached_usd":0.001138,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.088292249944061}
{"item_id":"senseval2.d001.s048.t001","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":2,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":3,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":4,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":5,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":6,"sense_key":"use%2:41:03::","synset_id":"use.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%2:34:01::","call_ids":["senseval2.d001.s048.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":878,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001876,"input_uncached_usd":0.001756,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8062756248982623}
{"item_id":"senseval2.d001.s048.t002","gold_sense_keys":["battery%1:14:01::"],"candidates":[{"index":1,"sense_key":"battery%1:14:02::","synset_id":"battery.n.01"},{"index":2,"sense_key":"battery%1:06:00::","synset_id":"battery.n.02"},{"index":3,"sense_key":"battery%1:14:01::","synset_id":"battery.n.03"},{"index":4,"sense_key":"battery%1:14:03::","synset_id":"battery.n.04"},{"index":5,"sense_key":"battery%1:06:03::","synset_id":"battery.n.05"},{"index":6,"sense_key":"battery%1:04:01::","synset_id":"barrage.n.02"},{"index":7,"sense_key":"battery%1:04:00::","synset_id":"battery.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"battery%1:14:01::","call_ids":["senseval2.d001.s048.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"battery%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":882,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001884,"input_uncached_usd":0.001764,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5233345420565456}
{"item_id":"senseval2.d001.s048.t004","gold_sense_keys":["develop%2:36:01::"],"candidates":[{"index":1,"sense_key":"develop%2:36:01::","synset_id":"develop.v.01"},{"index":2,"sense_key":"develop%2:36:00::","synset_id":"evolve.v.01"},{"index":3,"sense_key":"develop%2:30:06::","synset_id":"develop.v.03"},{"index":4,"sense_key":"develop%2:29:00::","synset_id":"grow.v.08"},{"index":5,"sense_key":"develop%2:42:00::","synset_id":"originate.v.01"},{"index":6,"sense_key":"develop%2:30:04::","synset_id":"build_up.v.05"},{"index":7,"sense_key":"develop%2:32:00::","synset_id":"explicate.v.02"},{"index":8,"sense_key":"develop%2:31:00::","synset_id":"train.v.01"},{"index":9,"sense_key":"develop%2:30:05::","synset_id":"develop.v.09"},{"index":10,"sense_key":"develop%2:30:00::","synset_id":"develop.v.10"},{"index":11,"sense_key":"develop%2:30:09::","synset_id":"modernize.v.02"},{"index":12,"sense_key":"develop%2:30:01::","synset_id":"develop.v.12"},{"index":13,"sense_key":"develop%2:36:09::","synset_id":"develop.v.13"},{"index":14,"sense_key":"develop%2:30:10::","synset_id":"develop.v.14"},{"index":15,"sense_key":"develop%2:39:00::","synset_id":"develop.v.15"},{"index":16,"sense_key":"develop%2:35:04::","synset_id":"develop.v.16"},{"index":17,"sense_key":"develop%2:33:03::","synset_id":"develop.v.17"},{"index":18,"sense_key":"develop%2:33:02::","synset_id":"develop.v.18"},{"index":19,"sense_key":"develop%2:30:13::","synset_id":"develop.v.19"},{"index":20,"sense_key":"develop%2:30:03::","synset_id":"break.v.53"},{"index":21,"sense_key":"develop%2:30:12::","synset_id":"develop.v.21"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"develop%2:36:01::","call_ids":["senseval2.d001.s048.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"develop%2:36:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1872,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004304,"input_uncached_usd":0.003744,"input_cached_usd":0.0,"output_usd":0.0005600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3744119169423357}
{"item_id":"senseval2.d001.s048.t007","gold_sense_keys":["genetic%3:01:02::"],"candidates":[{"index":1,"sense_key":"genetic%5:00:00:inheritable:00","synset_id":"familial.s.01"},{"index":2,"sense_key":"genetic%3:01:02::","synset_id":"genic.a.01"},{"index":3,"sense_key":"genetic%3:01:01::","synset_id":"genetic.a.03"},{"index":4,"sense_key":"genetic%3:01:00::","synset_id":"genetic.a.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"genetic%3:01:02::","call_ids":["senseval2.d001.s048.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"genetic%3:01:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":749,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002058,"input_uncached_usd":0.001498,"input_cached_usd":0.0,"output_usd":0.0005600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3573234579525888}
{"item_id":"senseval2.d001.s048.t008","gold_sense_keys":["material%1:27:00::"],"candidates":[{"index":1,"sense_key":"material%1:27:00::","synset_id":"material.n.01"},{"index":2,"sense_key":"material%1:10:00::","synset_id":"material.n.02"},{"index":3,"sense_key":"material%1:06:00::","synset_id":"fabric.n.01"},{"index":4,"sense_key":"material%1:06:01::","synset_id":"material.n.04"},{"index":5,"sense_key":"material%1:18:00::","synset_id":"material.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"material%1:27:00::","call_ids":["senseval2.d001.s048.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"material%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":840,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00229,"input_uncached_usd":0.0016799999999999999,"input_cached_usd":0.0,"output_usd":0.0006100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.5075972920749336}
{"item_id":"senseval2.d001.s048.t011","gold_sense_keys":["presence%1:26:00::"],"candidates":[{"index":1,"sense_key":"presence%1:26:00::","synset_id":"presence.n.01"},{"index":2,"sense_key":"presence%1:15:00::","synset_id":"presence.n.02"},{"index":3,"sense_key":"presence%1:18:00::","synset_id":"presence.n.03"},{"index":4,"sense_key":"presence%1:09:00::","synset_id":"presence.n.04"},{"index":5,"sense_key":"presence%1:07:00::","synset_id":"bearing.n.03"},{"index":6,"sense_key":"presence%1:04:00::","synset_id":"presence.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"presence%1:26:00::","call_ids":["senseval2.d001.s048.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"presence%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":758,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001636,"input_uncached_usd":0.001516,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.15339870902244}
{"item_id":"senseval2.d001.s048.t012","gold_sense_keys":["cell%1:03:00::"],"candidates":[{"index":1,"sense_key":"cell%1:06:03::","synset_id":"cell.n.01"},{"index":2,"sense_key":"cell%1:03:00::","synset_id":"cell.n.02"},{"index":3,"sense_key":"cell%1:06:00::","synset_id":"cell.n.03"},{"index":4,"sense_key":"cell%1:14:00::","synset_id":"cell.n.04"},{"index":5,"sense_key":"cell%1:06:04::","synset_id":"cellular_telephone.n.01"},{"index":6,"sense_key":"cell%1:06:02::","synset_id":"cell.n.06"},{"index":7,"sense_key":"cell%1:06:01::","synset_id":"cell.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cell%1:03:00::","call_ids":["senseval2.d001.s048.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cell%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":902,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001924,"input_uncached_usd":0.0018039999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9474300419678912}
{"item_id":"senseval2.d001.s049.t000","gold_sense_keys":["analyze%2:31:00::","analyze%2:31:01::"],"candidates":[{"index":1,"sense_key":"analyze%2:31:00::","synset_id":"analyze.v.01"},{"index":2,"sense_key":"analyze%2:31:01::","synset_id":"analyze.v.02"},{"index":3,"sense_key":"analyze%2:31:04::","synset_id":"analyze.v.03"},{"index":4,"sense_key":"analyze%2:31:03::","synset_id":"analyze.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"analyze%2:31:01::","call_ids":["senseval2.d001.s049.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"analyze%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":789,"cached_input_tokens":0,"output_tokens":75,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002328,"input_uncached_usd":0.001578,"input_cached_usd":0.0,"output_usd":0.00075,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.2952001249650493}
{"item_id":"senseval2.d001.s049.t001","gold_sense_keys":["cell%1:03:00::"],"candidates":[{"index":1,"sense_key":"cell%1:06:03::","synset_id":"cell.n.01"},{"index":2,"sense_key":"cell%1:03:00::","synset_id":"cell.n.02"},{"index":3,"sense_key":"cell%1:06:00::","synset_id":"cell.n.03"},{"index":4,"sense_key":"cell%1:14:00::","synset_id":"cell.n.04"},{"index":5,"sense_key":"cell%1:06:04::","synset_id":"cellular_telephone.n.01"},{"index":6,"sense_key":"cell%1:06:02::","synset_id":"cell.n.06"},{"index":7,"sense_key":"cell%1:06:01::","synset_id":"cell.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cell%1:03:00::","call_ids":["senseval2.d001.s049.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cell%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":862,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001844,"input_uncached_usd":0.0017239999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.05546470801346}
{"item_id":"senseval2.d001.s049.t002","gold_sense_keys":["extract%2:35:04::"],"candidates":[{"index":1,"sense_key":"extract%2:35:04::","synset_id":"extract.v.01"},{"index":2,"sense_key":"extract%2:40:00::","synset_id":"extract.v.02"},{"index":3,"sense_key":"extract%2:36:00::","synset_id":"educe.v.01"},{"index":4,"sense_key":"extract%2:30:00::","synset_id":"distill.v.03"},{"index":5,"sense_key":"extract%2:35:07::","synset_id":"extract.v.05"},{"index":6,"sense_key":"extract%2:35:05::","synset_id":"press_out.v.03"},{"index":7,"sense_key":"extract%2:31:00::","synset_id":"excerpt.v.01"},{"index":8,"sense_key":"extract%2:31:02::","synset_id":"extract.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"extract%2:35:04::","call_ids":["senseval2.d001.s049.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"extract%2:35:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":906,"cached_input_tokens":0,"output_tokens":118,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002992,"input_uncached_usd":0.001812,"input_cached_usd":0.0,"output_usd":0.00118,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7224692910676822}
{"item_id":"senseval2.d001.s049.t003","gold_sense_keys":["eye%1:08:00::"],"candidates":[{"index":1,"sense_key":"eye%1:08:00::","synset_id":"eye.n.01"},{"index":2,"sense_key":"eye%1:09:00::","synset_id":"eye.n.02"},{"index":3,"sense_key":"eye%1:09:01::","synset_id":"eye.n.03"},{"index":4,"sense_key":"eye%1:15:00::","synset_id":"center.n.01"},{"index":5,"sense_key":"eye%1:06:00::","synset_id":"eye.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"eye%1:08:00::","call_ids":["senseval2.d001.s049.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"eye%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":725,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00157,"input_uncached_usd":0.00145,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0174443339928985}
{"item_id":"senseval2.d001.s049.t005","gold_sense_keys":["find%2:31:09::","find%2:32:00::","find%2:36:00::","find%2:39:02::"],"candidates":[{"index":1,"sense_key":"find%2:40:02::","synset_id":"find.v.01"},{"index":2,"sense_key":"find%2:39:02::","synset_id":"detect.v.01"},{"index":3,"sense_key":"find%2:40:00::","synset_id":"find.v.03"},{"index":4,"sense_key":"find%2:32:00::","synset_id":"determine.v.01"},{"index":5,"sense_key":"find%2:31:10::","synset_id":"find.v.05"},{"index":6,"sense_key":"find%2:39:01::","synset_id":"witness.v.02"},{"index":7,"sense_key":"find%2:40:01::","synset_id":"line_up.v.02"},{"index":8,"sense_key":"find%2:36:00::","synset_id":"discover.v.03"},{"index":9,"sense_key":"find%2:31:09::","synset_id":"discover.v.04"},{"index":10,"sense_key":"find%2:40:03::","synset_id":"find.v.10"},{"index":11,"sense_key":"find%2:32:01::","synset_id":"rule.v.04"},{"index":12,"sense_key":"find%2:30:00::","synset_id":"receive.v.02"},{"index":13,"sense_key":"find%2:39:05::","synset_id":"find.v.13"},{"index":14,"sense_key":"find%2:40:15::","synset_id":"recover.v.01"},{"index":15,"sense_key":"find%2:38:10::","synset_id":"find.v.15"},{"index":16,"sense_key":"find%2:30:13::","synset_id":"find_oneself.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"find%2:39:02::","call_ids":["senseval2.d001.s049.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"find%2:39:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1602,"cached_input_tokens":0,"output_tokens":146,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004664,"input_uncached_usd":0.003204,"input_cached_usd":0.0,"output_usd":0.0014600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8838319170754403}
{"item_id":"senseval2.d001.s049.t006","gold_sense_keys":["defect%1:26:01::"],"candidates":[{"index":1,"sense_key":"defect%1:26:01::","synset_id":"defect.n.01"},{"index":2,"sense_key":"defect%1:07:01::","synset_id":"defect.n.02"},{"index":3,"sense_key":"defect%1:26:00::","synset_id":"defect.n.03"},{"index":4,"sense_key":"defect%1:07:00::","synset_id":"blemish.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"defect%1:26:01::","call_ids":["senseval2.d001.s049.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"defect%1:26:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":726,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002132,"input_uncached_usd":0.001452,"input_cached_usd":0.0,"output_usd":0.00068,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3583602500148118}
{"item_id":"senseval2.d001.s049.t007","gold_sense_keys":["second%5:00:00:ordinal:00"],"candidates":[{"index":1,"sense_key":"second%5:00:00:ordinal:00","synset_id":"second.s.01"},{"index":2,"sense_key":"second%3:00:00::","synset_id":"second.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"second%5:00:00:ordinal:00","call_ids":["senseval2.d001.s049.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"second%5:00:00:ordinal:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":574,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001268,"input_uncached_usd":0.001148,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0596532910130918}
{"item_id":"senseval2.d001.s049.t008","gold_sense_keys":["copy%1:06:00::"],"candidates":[{"index":1,"sense_key":"copy%1:10:00::","synset_id":"transcript.n.02"},{"index":2,"sense_key":"copy%1:06:00::","synset_id":"copy.n.02"},{"index":3,"sense_key":"copy%1:10:01::","synset_id":"copy.n.03"},{"index":4,"sense_key":"copy%1:10:02::","synset_id":"copy.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"copy%1:06:00::","call_ids":["senseval2.d001.s049.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"copy%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":666,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001452,"input_uncached_usd":0.0013319999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3818364170147106}
{"item_id":"senseval2.d001.s049.t010","gold_sense_keys":["exact%3:00:00::"],"candidates":[{"index":1,"sense_key":"exact%3:00:00::","synset_id":"exact.a.01"},{"index":2,"sense_key":"exact%5:00:00:correct:00","synset_id":"accurate.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"exact%3:00:00::","call_ids":["senseval2.d001.s049.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"exact%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":598,"cached_input_tokens":0,"output_tokens":122,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002416,"input_uncached_usd":0.001196,"input_cached_usd":0.0,"output_usd":0.0012200000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8366584170144051}
{"item_id":"senseval2.d001.s049.t012","gold_sense_keys":["first%5:00:00:ordinal:00"],"candidates":[{"index":1,"sense_key":"first%3:00:00::","synset_id":"first.a.01"},{"index":2,"sense_key":"first%5:00:00:ordinal:00","synset_id":"first.s.01"},{"index":3,"sense_key":"first%5:00:02:opening:00","synset_id":"inaugural.s.01"},{"index":4,"sense_key":"first%5:00:01:opening:00","synset_id":"beginning.s.01"},{"index":5,"sense_key":"first%5:00:00:best:00","synset_id":"first.s.04"},{"index":6,"sense_key":"first%3:00:03::","synset_id":"first.a.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"first%5:00:00:ordinal:00","call_ids":["senseval2.d001.s049.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"first%5:00:00:ordinal:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":814,"cached_input_tokens":0,"output_tokens":135,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0029779999999999997,"input_uncached_usd":0.0016279999999999999,"input_cached_usd":0.0,"output_usd":0.00135,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9585804169764742}
{"item_id":"senseval2.d001.s049.t013","gold_sense_keys":["copy%1:06:00::"],"candidates":[{"index":1,"sense_key":"copy%1:10:00::","synset_id":"transcript.n.02"},{"index":2,"sense_key":"copy%1:06:00::","synset_id":"copy.n.02"},{"index":3,"sense_key":"copy%1:10:01::","synset_id":"copy.n.03"},{"index":4,"sense_key":"copy%1:10:02::","synset_id":"copy.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"copy%1:06:00::","call_ids":["senseval2.d001.s049.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"copy%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":666,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001992,"input_uncached_usd":0.0013319999999999999,"input_cached_usd":0.0,"output_usd":0.0006600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2244789999676868}
{"item_id":"senseval2.d001.s050.t000","gold_sense_keys":["finding%1:17:00::"],"candidates":[{"index":1,"sense_key":"finding%1:04:00::","synset_id":"determination.n.01"},{"index":2,"sense_key":"finding%1:04:01::","synset_id":"finding.n.02"},{"index":3,"sense_key":"finding%1:17:00::","synset_id":"finding.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"finding%1:17:00::","call_ids":["senseval2.d001.s050.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"finding%1:17:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":631,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001382,"input_uncached_usd":0.0012619999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5028921670746058}
{"item_id":"senseval2.d001.s050.t001","gold_sense_keys":["medicine%1:04:00::"],"candidates":[{"index":1,"sense_key":"medicine%1:09:00::","synset_id":"medicine.n.01"},{"index":2,"sense_key":"medicine%1:06:00::","synset_id":"medicine.n.02"},{"index":3,"sense_key":"medicine%1:04:00::","synset_id":"medicine.n.03"},{"index":4,"sense_key":"medicine%1:04:02::","synset_id":"music.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"medicine%1:04:00::","call_ids":["senseval2.d001.s050.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"medicine%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":737,"cached_input_tokens":0,"output_tokens":46,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001934,"input_uncached_usd":0.0014739999999999998,"input_cached_usd":0.0,"output_usd":0.00046,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4617224580142647}
{"item_id":"senseval2.d001.s051.t000","gold_sense_keys":["first%3:00:00::"],"candidates":[{"index":1,"sense_key":"first%3:00:00::","synset_id":"first.a.01"},{"index":2,"sense_key":"first%5:00:00:ordinal:00","synset_id":"first.s.01"},{"index":3,"sense_key":"first%5:00:02:opening:00","synset_id":"inaugural.s.01"},{"index":4,"sense_key":"first%5:00:01:opening:00","synset_id":"beginning.s.01"},{"index":5,"sense_key":"first%5:00:00:best:00","synset_id":"first.s.04"},{"index":6,"sense_key":"first%3:00:03::","synset_id":"first.a.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"first%5:00:00:ordinal:00","call_ids":["senseval2.d001.s051.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"first%5:00:00:ordinal:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":839,"cached_input_tokens":0,"output_tokens":155,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003228,"input_uncached_usd":0.001678,"input_cached_usd":0.0,"output_usd":0.0015500000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.086911207996309}
{"item_id":"senseval2.d001.s051.t001","gold_sense_keys":["time%1:11:00::"],"candidates":[{"index":1,"sense_key":"time%1:11:00::","synset_id":"time.n.01"},{"index":2,"sense_key":"time%1:28:05::","synset_id":"time.n.02"},{"index":3,"sense_key":"time%1:28:00::","synset_id":"time.n.03"},{"index":4,"sense_key":"time%1:28:06::","synset_id":"time.n.04"},{"index":5,"sense_key":"time%1:03:00::","synset_id":"time.n.05"},{"index":6,"sense_key":"time%1:11:01::","synset_id":"time.n.06"},{"index":7,"sense_key":"time%1:28:03::","synset_id":"clock_time.n.01"},{"index":8,"sense_key":"time%1:28:04::","synset_id":"fourth_dimension.n.01"},{"index":9,"sense_key":"time%1:07:01::","synset_id":"meter.n.04"},{"index":10,"sense_key":"time%1:28:01::","synset_id":"prison_term.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"time%1:11:00::","call_ids":["senseval2.d001.s051.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"time%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1047,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002214,"input_uncached_usd":0.002094,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8407079580938444}
{"item_id":"senseval2.d001.s051.t002","gold_sense_keys":["show%2:31:00::","show%2:32:00::"],"candidates":[{"index":1,"sense_key":"show%2:39:02::","synset_id":"show.v.01"},{"index":2,"sense_key":"show%2:31:00::","synset_id":"prove.v.02"},{"index":3,"sense_key":"show%2:32:00::","synset_id":"testify.v.02"},{"index":4,"sense_key":"show%2:39:00::","synset_id":"show.v.04"},{"index":5,"sense_key":"show%2:36:00::","synset_id":"picture.v.02"},{"index":6,"sense_key":"show%2:32:02::","synset_id":"express.v.01"},{"index":7,"sense_key":"show%2:32:01::","synset_id":"indicate.v.02"},{"index":8,"sense_key":"show%2:39:01::","synset_id":"show.v.08"},{"index":9,"sense_key":"show%2:32:04::","synset_id":"read.v.08"},{"index":10,"sense_key":"show%2:32:03::","synset_id":"show.v.10"},{"index":11,"sense_key":"show%2:38:00::","synset_id":"usher.v.01"},{"index":12,"sense_key":"show%2:33:00::","synset_id":"show.v.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"show%2:31:00::","call_ids":["senseval2.d001.s051.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"show%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1291,"cached_input_tokens":0,"output_tokens":78,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0033620000000000004,"input_uncached_usd":0.002582,"input_cached_usd":0.0,"output_usd":0.0007800000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4377662080805749}
{"item_id":"senseval2.d001.s051.t003","gold_sense_keys":["loss%1:04:00::"],"candidates":[{"index":1,"sense_key":"loss%1:21:01::","synset_id":"loss.n.01"},{"index":2,"sense_key":"loss%1:22:00::","synset_id":"loss.n.02"},{"index":3,"sense_key":"loss%1:04:00::","synset_id":"loss.n.03"},{"index":4,"sense_key":"loss%1:07:00::","synset_id":"loss.n.04"},{"index":5,"sense_key":"loss%1:11:01::","synset_id":"loss.n.05"},{"index":6,"sense_key":"loss%1:21:00::","synset_id":"loss.n.06"},{"index":7,"sense_key":"loss%1:11:00::","synset_id":"personnel_casualty.n.01"},{"index":8,"sense_key":"loss%1:11:02::","synset_id":"passing.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"loss%1:04:00::","call_ids":["senseval2.d001.s051.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"loss%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":944,"cached_input_tokens":0,"output_tokens":125,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003138,"input_uncached_usd":0.001888,"input_cached_usd":0.0,"output_usd":0.00125,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.13728795805946}
{"item_id":"senseval2.d001.s051.t004","gold_sense_keys":["copy%1:06:00::"],"candidates":[{"index":1,"sense_key":"copy%1:10:00::","synset_id":"transcript.n.02"},{"index":2,"sense_key":"copy%1:06:00::","synset_id":"copy.n.02"},{"index":3,"sense_key":"copy%1:10:01::","synset_id":"copy.n.03"},{"index":4,"sense_key":"copy%1:10:02::","synset_id":"copy.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"copy%1:06:00::","call_ids":["senseval2.d001.s051.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"copy%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":691,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001502,"input_uncached_usd":0.001382,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4770342920674011}
{"item_id":"senseval2.d001.s051.t005","gold_sense_keys":["same%3:00:02::"],"candidates":[{"index":1,"sense_key":"same%3:00:02::","synset_id":"same.a.01"},{"index":2,"sense_key":"same%3:00:00::","synset_id":"same.a.02"},{"index":3,"sense_key":"same%3:00:04::","synset_id":"like.a.02"},{"index":4,"sense_key":"same%5:00:00:unchanged:00","synset_id":"same.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"same%3:00:02::","call_ids":["senseval2.d001.s051.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"same%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":698,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001966,"input_uncached_usd":0.0013959999999999999,"input_cached_usd":0.0,"output_usd":0.0005700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4709863329771906}
{"item_id":"senseval2.d001.s051.t007","gold_sense_keys":["lead%2:41:12::","lead%2:42:12::"],"candidates":[{"index":1,"sense_key":"lead%2:38:01::","synset_id":"lead.v.01"},{"index":2,"sense_key":"lead%2:42:04::","synset_id":"leave.v.07"},{"index":3,"sense_key":"lead%2:42:12::","synset_id":"lead.v.03"},{"index":4,"sense_key":"lead%2:38:00::","synset_id":"lead.v.04"},{"index":5,"sense_key":"lead%2:32:00::","synset_id":"lead.v.05"},{"index":6,"sense_key":"lead%2:42:00::","synset_id":"run.v.03"},{"index":7,"sense_key":"lead%2:41:00::","synset_id":"head.v.02"},{"index":8,"sense_key":"lead%2:42:01::","synset_id":"lead.v.08"},{"index":9,"sense_key":"lead%2:41:12::","synset_id":"contribute.v.03"},{"index":10,"sense_key":"lead%2:36:00::","synset_id":"conduct.v.02"},{"index":11,"sense_key":"lead%2:42:03::","synset_id":"go.v.25"},{"index":12,"sense_key":"lead%2:38:02::","synset_id":"precede.v.04"},{"index":13,"sense_key":"lead%2:42:02::","synset_id":"run.v.23"},{"index":14,"sense_key":"lead%2:32:01::","synset_id":"moderate.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"lead%2:42:12::","call_ids":["senseval2.d001.s051.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"lead%2:42:12::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1303,"cached_input_tokens":0,"output_tokens":133,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003936,"input_uncached_usd":0.002606,"input_cached_usd":0.0,"output_usd":0.00133,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.633565374999307}
{"item_id":"senseval2.d001.s051.t008","gold_sense_keys":["eruption%1:11:03::","eruption%1:26:02::"],"candidates":[{"index":1,"sense_key":"eruption%1:11:00::","synset_id":"volcanic_eruption.n.01"},{"index":2,"sense_key":"eruption%1:26:02::","synset_id":"eruption.n.02"},{"index":3,"sense_key":"eruption%1:26:01::","synset_id":"eruption.n.03"},{"index":4,"sense_key":"eruption%1:11:03::","synset_id":"outbreak.n.01"},{"index":5,"sense_key":"eruption%1:11:01::","synset_id":"bang.n.02"},{"index":6,"sense_key":"eruption%1:11:02::","synset_id":"eruption.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"eruption%1:11:03::","call_ids":["senseval2.d001.s051.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"eruption%1:11:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":813,"cached_input_tokens":0,"output_tokens":138,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003006,"input_uncached_usd":0.0016259999999999998,"input_cached_usd":0.0,"output_usd":0.0013800000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9793768749805167}
{"item_id":"senseval2.d001.s051.t009","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s051.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":795,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00171,"input_uncached_usd":0.0015899999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.648691916023381}
{"item_id":"senseval2.d001.s052.t001","gold_sense_keys":["satisfying%5:00:00:satisfactory:00"],"candidates":[{"index":1,"sense_key":"satisfying%5:00:00:wholesome:00","synset_id":"hearty.s.02"},{"index":2,"sense_key":"satisfying%5:00:00:satisfactory:00","synset_id":"comforting.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"satisfying%5:00:00:satisfactory:00","call_ids":["senseval2.d001.s052.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"satisfying%5:00:00:satisfactory:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":617,"cached_input_tokens":0,"output_tokens":90,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002134,"input_uncached_usd":0.0012339999999999999,"input_cached_usd":0.0,"output_usd":0.0009000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.834433124982752}
{"item_id":"senseval2.d001.s052.t002","gold_sense_keys":["say%2:32:00::","say%2:32:15::"],"candidates":[{"index":1,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":6,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":9,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":10,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":11,"sense_key":"say%2:32:09::","synset_id":"say.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"say%2:32:00::","call_ids":["senseval2.d001.s052.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"say%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1146,"cached_input_tokens":0,"output_tokens":139,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003682,"input_uncached_usd":0.0022919999999999998,"input_cached_usd":0.0,"output_usd":0.0013900000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9040465420112014}
{"item_id":"senseval2.d001.s052.t003","gold_sense_keys":["now%4:02:04::"],"candidates":[{"index":1,"sense_key":"now%4:02:05::","synset_id":"now.r.01"},{"index":2,"sense_key":"now%4:02:00::","synset_id":"nowadays.r.01"},{"index":3,"sense_key":"now%4:02:02::","synset_id":"now.r.03"},{"index":4,"sense_key":"now%4:02:04::","synset_id":"now.r.04"},{"index":5,"sense_key":"now%4:02:01::","synset_id":"immediately.r.01"},{"index":6,"sense_key":"now%4:02:07::","synset_id":"now.r.06"},{"index":7,"sense_key":"now%4:02:06::","synset_id":"now.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"now%4:02:04::","call_ids":["senseval2.d001.s052.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"now%4:02:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":927,"cached_input_tokens":0,"output_tokens":85,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002704,"input_uncached_usd":0.001854,"input_cached_usd":0.0,"output_usd":0.0008500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":6.4436588330427185}
{"item_id":"senseval2.d001.s053.t000","gold_sense_keys":["true%3:00:00::"],"candidates":[{"index":1,"sense_key":"true%3:00:00::","synset_id":"true.a.01"},{"index":2,"sense_key":"true%5:00:00:accurate:00","synset_id":"true.s.01"},{"index":3,"sense_key":"true%5:00:02:faithful:00","synset_id":"true.s.02"},{"index":4,"sense_key":"true%3:00:04::","synset_id":"truthful.a.01"},{"index":5,"sense_key":"true%5:00:00:typical:00","synset_id":"true.s.03"},{"index":6,"sense_key":"true%5:00:00:trustworthy:00","synset_id":"dependable.s.01"},{"index":7,"sense_key":"true%5:00:00:sincere:00","synset_id":"genuine.s.01"},{"index":8,"sense_key":"true%5:00:00:real:02","synset_id":"true.s.06"},{"index":9,"sense_key":"true%5:00:00:geographic:00","synset_id":"true.s.07"},{"index":10,"sense_key":"true%5:00:00:legitimate:00","synset_id":"true.s.08"},{"index":11,"sense_key":"true%5:00:00:harmonious:00","synset_id":"on-key.s.01"},{"index":12,"sense_key":"true%5:00:00:even:01","synset_id":"true.s.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"true%3:00:00::","call_ids":["senseval2.d001.s053.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"true%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1221,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0025619999999999996,"input_uncached_usd":0.0024419999999999997,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1134443749906495}
{"item_id":"senseval2.d001.s053.t002","gold_sense_keys":["true%3:00:00::"],"candidates":[{"index":1,"sense_key":"true%3:00:00::","synset_id":"true.a.01"},{"index":2,"sense_key":"true%5:00:00:accurate:00","synset_id":"true.s.01"},{"index":3,"sense_key":"true%5:00:02:faithful:00","synset_id":"true.s.02"},{"index":4,"sense_key":"true%3:00:04::","synset_id":"truthful.a.01"},{"index":5,"sense_key":"true%5:00:00:typical:00","synset_id":"true.s.03"},{"index":6,"sense_key":"true%5:00:00:trustworthy:00","synset_id":"dependable.s.01"},{"index":7,"sense_key":"true%5:00:00:sincere:00","synset_id":"genuine.s.01"},{"index":8,"sense_key":"true%5:00:00:real:02","synset_id":"true.s.06"},{"index":9,"sense_key":"true%5:00:00:geographic:00","synset_id":"true.s.07"},{"index":10,"sense_key":"true%5:00:00:legitimate:00","synset_id":"true.s.08"},{"index":11,"sense_key":"true%5:00:00:harmonious:00","synset_id":"on-key.s.01"},{"index":12,"sense_key":"true%5:00:00:even:01","synset_id":"true.s.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"true%3:00:00::","call_ids":["senseval2.d001.s053.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"true%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1221,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003052,"input_uncached_usd":0.0024419999999999997,"input_cached_usd":0.0,"output_usd":0.0006100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3271010830067098}
{"item_id":"senseval2.d001.s053.t003","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s053.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":775,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00167,"input_uncached_usd":0.00155,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.141011624946259}
{"item_id":"senseval2.d001.s054.t000","gold_sense_keys":["audacious%5:00:00:adventurous:00"],"candidates":[{"index":1,"sense_key":"audacious%5:00:00:bold:00","synset_id":"audacious.s.01"},{"index":2,"sense_key":"audacious%5:00:00:unashamed:00","synset_id":"audacious.s.02"},{"index":3,"sense_key":"audacious%5:00:00:adventurous:00","synset_id":"audacious.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"audacious%5:00:00:adventurous:00","call_ids":["senseval2.d001.s054.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"audacious%5:00:00:adventurous:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":811,"cached_input_tokens":0,"output_tokens":93,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002552,"input_uncached_usd":0.001622,"input_cached_usd":0.0,"output_usd":0.00093,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5915069580078125}
{"item_id":"senseval2.d001.s054.t001","gold_sense_keys":["claim%1:10:02::"],"candidates":[{"index":1,"sense_key":"claim%1:10:00::","synset_id":"claim.n.01"},{"index":2,"sense_key":"claim%1:10:02::","synset_id":"claim.n.02"},{"index":3,"sense_key":"claim%1:04:00::","synset_id":"claim.n.03"},{"index":4,"sense_key":"claim%1:07:02::","synset_id":"claim.n.04"},{"index":5,"sense_key":"claim%1:07:01::","synset_id":"title.n.07"},{"index":6,"sense_key":"claim%1:10:01::","synset_id":"call.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"claim%1:10:02::","call_ids":["senseval2.d001.s054.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"claim%1:10:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":823,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001766,"input_uncached_usd":0.001646,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2177213339600712}
{"item_id":"senseval2.d001.s055.t000","gold_sense_keys":["young%3:00:00::"],"candidates":[{"index":1,"sense_key":"young%3:00:00::","synset_id":"young.a.01"},{"index":2,"sense_key":"young%5:00:00:early:02","synset_id":"new.s.08"},{"index":3,"sense_key":"young%5:00:01:young:00","synset_id":"youthful.s.01"},{"index":4,"sense_key":"young%5:00:00:new:00","synset_id":"young.s.03"},{"index":5,"sense_key":"young%5:00:00:inexperienced:00","synset_id":"unseasoned.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"young%3:00:00::","call_ids":["senseval2.d001.s055.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"young%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":796,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001712,"input_uncached_usd":0.0015919999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3872418330283836}
{"item_id":"senseval2.d001.s055.t003","gold_sense_keys":["believe%2:31:04::"],"candidates":[{"index":1,"sense_key":"believe%2:31:00::","synset_id":"believe.v.01"},{"index":2,"sense_key":"believe%2:31:04::","synset_id":"think.v.01"},{"index":3,"sense_key":"believe%2:31:03::","synset_id":"believe.v.03"},{"index":4,"sense_key":"believe%2:31:01::","synset_id":"believe.v.04"},{"index":5,"sense_key":"believe%2:31:02::","synset_id":"believe.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"believe%2:31:00::","call_ids":["senseval2.d001.s055.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"believe%2:31:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":781,"cached_input_tokens":0,"output_tokens":166,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003222,"input_uncached_usd":0.001562,"input_cached_usd":0.0,"output_usd":0.0016600000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1619590419577435}
{"item_id":"senseval2.d001.s055.t004","gold_sense_keys":["right%3:00:04::"],"candidates":[{"index":1,"sense_key":"right%3:00:00::","synset_id":"right.a.01"},{"index":2,"sense_key":"right%3:00:02::","synset_id":"correct.a.01"},{"index":3,"sense_key":"right%5:00:01:proper:00","synset_id":"correct.s.01"},{"index":4,"sense_key":"right%3:00:01::","synset_id":"right.a.04"},{"index":5,"sense_key":"right%3:00:04::","synset_id":"right.a.05"},{"index":6,"sense_key":"right%5:00:00:appropriate:00","synset_id":"proper.s.03"},{"index":7,"sense_key":"right%3:00:03::","synset_id":"right.a.07"},{"index":8,"sense_key":"right%5:00:00:satisfactory:00","synset_id":"right.s.03"},{"index":9,"sense_key":"right%5:00:00:right-handed:00","synset_id":"right.s.04"},{"index":10,"sense_key":"right%5:00:02:proper:00","synset_id":"correct.s.02"},{"index":11,"sense_key":"right%5:00:00:perpendicular:00","synset_id":"right.s.06"},{"index":12,"sense_key":"right%5:00:00:outside:00","synset_id":"right.s.07"},{"index":13,"sense_key":"right%5:00:00:opportune:00","synset_id":"good.s.10"},{"index":14,"sense_key":"right%5:00:00:accurate:00","synset_id":"veracious.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"right%3:00:04::","call_ids":["senseval2.d001.s055.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"right%3:00:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1296,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0027119999999999996,"input_uncached_usd":0.0025919999999999997,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.15660999994725}
{"item_id":"senseval2.d001.s055.t005","gold_sense_keys":["set_out%2:30:00::"],"candidates":[{"index":1,"sense_key":"set_out%2:30:00::","synset_id":"get_down.v.07"},{"index":2,"sense_key":"set_out%2:35:00::","synset_id":"range.v.05"},{"index":3,"sense_key":"set_out%2:38:00::","synset_id":"depart.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"set_out%2:30:00::","call_ids":["senseval2.d001.s055.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"set_out%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":700,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00152,"input_uncached_usd":0.0014,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9132066250313073}
{"item_id":"senseval2.d001.s055.t006","gold_sense_keys":["repeat%2:36:00::"],"candidates":[{"index":1,"sense_key":"repeat%2:32:00::","synset_id":"repeat.v.01"},{"index":2,"sense_key":"repeat%2:36:00::","synset_id":"duplicate.v.01"},{"index":3,"sense_key":"repeat%2:30:00::","synset_id":"recur.v.01"},{"index":4,"sense_key":"repeat%2:32:01::","synset_id":"repeat.v.04"},{"index":5,"sense_key":"repeat%2:41:00::","synset_id":"repeat.v.05"},{"index":6,"sense_key":"repeat%2:36:01::","synset_id":"reprise.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"repeat%2:36:00::","call_ids":["senseval2.d001.s055.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"repeat%2:36:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":843,"cached_input_tokens":0,"output_tokens":125,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0029360000000000002,"input_uncached_usd":0.001686,"input_cached_usd":0.0,"output_usd":0.00125,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.591600334038958}
{"item_id":"senseval2.d001.s055.t007","gold_sense_keys":["experiment%1:04:00::"],"candidates":[{"index":1,"sense_key":"experiment%1:04:00::","synset_id":"experiment.n.01"},{"index":2,"sense_key":"experiment%1:09:00::","synset_id":"experiment.n.02"},{"index":3,"sense_key":"experiment%1:04:01::","synset_id":"experiment.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"experiment%1:04:00::","call_ids":["senseval2.d001.s055.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"experiment%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":657,"cached_input_tokens":0,"output_tokens":118,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0024939999999999997,"input_uncached_usd":0.0013139999999999998,"input_cached_usd":0.0,"output_usd":0.00118,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6430914999218658}
{"item_id":"senseval2.d001.s055.t008","gold_sense_keys":["cell%1:03:00::"],"candidates":[{"index":1,"sense_key":"cell%1:06:03::","synset_id":"cell.n.01"},{"index":2,"sense_key":"cell%1:03:00::","synset_id":"cell.n.02"},{"index":3,"sense_key":"cell%1:06:00::","synset_id":"cell.n.03"},{"index":4,"sense_key":"cell%1:14:00::","synset_id":"cell.n.04"},{"index":5,"sense_key":"cell%1:06:04::","synset_id":"cellular_telephone.n.01"},{"index":6,"sense_key":"cell%1:06:02::","synset_id":"cell.n.06"},{"index":7,"sense_key":"cell%1:06:01::","synset_id":"cell.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cell%1:03:00::","call_ids":["senseval2.d001.s055.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cell%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":901,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001922,"input_uncached_usd":0.001802,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1431339170085266}
{"item_id":"senseval2.d001.s055.t009","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":2,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":3,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"other%3:00:00::","call_ids":["senseval2.d001.s055.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":725,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00157,"input_uncached_usd":0.00145,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0556980409892276}
{"item_id":"senseval2.d001.s055.t010","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s055.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":809,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001738,"input_uncached_usd":0.0016179999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1106688749277964}
{"item_id":"senseval2.d001.s056.t001","gold_sense_keys":["team%1:14:00::"],"candidates":[{"index":1,"sense_key":"team%1:14:00::","synset_id":"team.n.01"},{"index":2,"sense_key":"team%1:14:01::","synset_id":"team.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"team%1:14:00::","call_ids":["senseval2.d001.s056.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"team%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":604,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001328,"input_uncached_usd":0.0012079999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4904856249922886}
{"item_id":"senseval2.d001.s056.t002","gold_sense_keys":["report%2:32:04::"],"candidates":[{"index":1,"sense_key":"report%2:32:00::","synset_id":"report.v.01"},{"index":2,"sense_key":"report%2:32:04::","synset_id":"report.v.02"},{"index":3,"sense_key":"report%2:32:03::","synset_id":"report.v.03"},{"index":4,"sense_key":"report%2:32:01::","synset_id":"report.v.04"},{"index":5,"sense_key":"report%2:32:13::","synset_id":"report.v.05"},{"index":6,"sense_key":"report%2:32:02::","synset_id":"report.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"report%2:32:04::","call_ids":["senseval2.d001.s056.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"report%2:32:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":869,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001858,"input_uncached_usd":0.001738,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6884302080143243}
{"item_id":"senseval2.d001.s056.t003","gold_sense_keys":["dual%5:00:00:multiple:00"],"candidates":[{"index":1,"sense_key":"dual%5:00:00:multiple:00","synset_id":"double.s.02"},{"index":2,"sense_key":"dual%5:00:02:multiple:00","synset_id":"double.s.01"},{"index":3,"sense_key":"dual%5:00:00:plural:00","synset_id":"dual.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"dual%5:00:00:multiple:00","call_ids":["senseval2.d001.s056.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"dual%5:00:00:multiple:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":819,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001758,"input_uncached_usd":0.001638,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0724171670153737}
{"item_id":"senseval2.d001.s056.t005","gold_sense_keys":["loss%1:04:00::"],"candidates":[{"index":1,"sense_key":"loss%1:21:01::","synset_id":"loss.n.01"},{"index":2,"sense_key":"loss%1:22:00::","synset_id":"loss.n.02"},{"index":3,"sense_key":"loss%1:04:00::","synset_id":"loss.n.03"},{"index":4,"sense_key":"loss%1:07:00::","synset_id":"loss.n.04"},{"index":5,"sense_key":"loss%1:11:01::","synset_id":"loss.n.05"},{"index":6,"sense_key":"loss%1:21:00::","synset_id":"loss.n.06"},{"index":7,"sense_key":"loss%1:11:00::","synset_id":"personnel_casualty.n.01"},{"index":8,"sense_key":"loss%1:11:02::","synset_id":"passing.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"loss%1:04:00::","call_ids":["senseval2.d001.s056.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"loss%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":993,"cached_input_tokens":0,"output_tokens":144,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0034260000000000002,"input_uncached_usd":0.001986,"input_cached_usd":0.0,"output_usd":0.00144,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7209415829274803}
{"item_id":"senseval2.d001.s056.t007","gold_sense_keys":["childhood%1:28:00::"],"candidates":[{"index":1,"sense_key":"childhood%1:28:00::","synset_id":"childhood.n.01"},{"index":2,"sense_key":"childhood%1:26:00::","synset_id":"childhood.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"childhood%1:28:00::","call_ids":["senseval2.d001.s056.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"childhood%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":609,"cached_input_tokens":0,"output_tokens":71,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001928,"input_uncached_usd":0.001218,"input_cached_usd":0.0,"output_usd":0.00071,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5179389159893617}
{"item_id":"senseval2.d001.s056.t008","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s056.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":844,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001808,"input_uncached_usd":0.0016879999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4674765829695389}
{"item_id":"senseval2.d001.s056.t010","gold_sense_keys":["call%2:32:02::"],"candidates":[{"index":1,"sense_key":"call%2:32:02::","synset_id":"name.v.01"},{"index":2,"sense_key":"call%2:32:00::","synset_id":"call.v.02"},{"index":3,"sense_key":"call%2:32:01::","synset_id":"call.v.03"},{"index":4,"sense_key":"call%2:32:09::","synset_id":"shout.v.02"},{"index":5,"sense_key":"call%2:32:05::","synset_id":"call.v.05"},{"index":6,"sense_key":"call%2:41:00::","synset_id":"visit.v.03"},{"index":7,"sense_key":"call%2:41:04::","synset_id":"call.v.07"},{"index":8,"sense_key":"call%2:32:13::","synset_id":"call.v.08"},{"index":9,"sense_key":"call%2:32:04::","synset_id":"call.v.09"},{"index":10,"sense_key":"call%2:32:03::","synset_id":"call.v.10"},{"index":11,"sense_key":"call%2:42:09::","synset_id":"call.v.11"},{"index":12,"sense_key":"call%2:41:14::","synset_id":"address.v.06"},{"index":13,"sense_key":"call%2:41:12::","synset_id":"call.v.13"},{"index":14,"sense_key":"call%2:40:09::","synset_id":"call.v.14"},{"index":15,"sense_key":"call%2:40:05::","synset_id":"bid.v.04"},{"index":16,"sense_key":"call%2:38:00::","synset_id":"call.v.16"},{"index":17,"sense_key":"call%2:32:06::","synset_id":"call.v.17"},{"index":18,"sense_key":"call%2:32:10::","synset_id":"predict.v.01"},{"index":19,"sense_key":"call%2:32:12::","synset_id":"call.v.19"},{"index":20,"sense_key":"call%2:32:07::","synset_id":"call.v.20"},{"index":21,"sense_key":"call%2:32:08::","synset_id":"call.v.21"},{"index":22,"sense_key":"call%2:32:11::","synset_id":"call.v.22"},{"index":23,"sense_key":"call%2:32:15::","synset_id":"call.v.23"},{"index":24,"sense_key":"call%2:32:14::","synset_id":"call.v.24"},{"index":25,"sense_key":"call%2:31:13::","synset_id":"call.v.25"},{"index":26,"sense_key":"call%2:31:00::","synset_id":"call.v.26"},{"index":27,"sense_key":"call%2:31:05::","synset_id":"call.v.27"},{"index":28,"sense_key":"call%2:29:05::","synset_id":"call.v.28"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"call%2:32:02::","call_ids":["senseval2.d001.s056.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"call%2:32:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2148,"cached_input_tokens":0,"output_tokens":121,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.005506,"input_uncached_usd":0.0042959999999999995,"input_cached_usd":0.0,"output_usd":0.0012100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5120187080465257}
{"item_id":"senseval2.d001.s057.t001","gold_sense_keys":["turn_to%2:30:00::"],"candidates":[{"index":1,"sense_key":"turn_to%2:32:01::","synset_id":"address.v.01"},{"index":2,"sense_key":"turn_to%2:30:00::","synset_id":"turn_to.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"turn_to%2:30:00::","call_ids":["senseval2.d001.s057.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"turn_to%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":660,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00144,"input_uncached_usd":0.00132,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.065491000073962}
{"item_id":"senseval2.d001.s057.t005","gold_sense_keys":["second%5:00:00:ordinal:00"],"candidates":[{"index":1,"sense_key":"second%5:00:00:ordinal:00","synset_id":"second.s.01"},{"index":2,"sense_key":"second%3:00:00::","synset_id":"second.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"second%5:00:00:ordinal:00","call_ids":["senseval2.d001.s057.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"second%5:00:00:ordinal:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":653,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001426,"input_uncached_usd":0.0013059999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":11.391406667069532}
{"item_id":"senseval2.d001.s057.t006","gold_sense_keys":["big%3:00:01::","big%5:00:00:conspicuous:00"],"candidates":[{"index":1,"sense_key":"big%3:00:01::","synset_id":"large.a.01"},{"index":2,"sense_key":"big%5:00:00:important:00","synset_id":"big.s.01"},{"index":3,"sense_key":"big%5:00:00:intense:00","synset_id":"bad.s.01"},{"index":4,"sense_key":"big%5:00:00:loud:00","synset_id":"big.s.03"},{"index":5,"sense_key":"big%5:00:00:conspicuous:00","synset_id":"big.s.04"},{"index":6,"sense_key":"big%5:00:00:intemperate:00","synset_id":"big.s.05"},{"index":7,"sense_key":"big%5:00:02:proud:00","synset_id":"boastful.s.01"},{"index":8,"sense_key":"big%5:00:00:proud:00","synset_id":"big.s.07"},{"index":9,"sense_key":"big%5:00:00:mature:01","synset_id":"adult.s.01"},{"index":10,"sense_key":"big%5:00:00:heavy:08","synset_id":"big.s.09"},{"index":11,"sense_key":"big%5:00:00:generous:02","synset_id":"big.s.10"},{"index":12,"sense_key":"big%5:00:00:generous:01","synset_id":"big.s.11"},{"index":13,"sense_key":"big%5:00:00:pregnant:00","synset_id":"big.s.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"big%3:00:01::","call_ids":["senseval2.d001.s057.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"big%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1346,"cached_input_tokens":0,"output_tokens":122,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0039120000000000005,"input_uncached_usd":0.002692,"input_cached_usd":0.0,"output_usd":0.0012200000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.086027584038675}
{"item_id":"senseval2.d001.s057.t007","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s057.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":849,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001818,"input_uncached_usd":0.0016979999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8234006250277162}
{"item_id":"senseval2.d001.s057.t008","gold_sense_keys":["killer%1:26:00::"],"candidates":[{"index":1,"sense_key":"killer%1:18:00::","synset_id":"killer.n.01"},{"index":2,"sense_key":"killer%1:26:00::","synset_id":"cause_of_death.n.01"},{"index":3,"sense_key":"killer%1:09:00::","synset_id":"killer.n.03"},{"index":4,"sense_key":"killer%1:05:00::","synset_id":"killer_whale.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"killer%1:26:00::","call_ids":["senseval2.d001.s057.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"killer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":765,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00165,"input_uncached_usd":0.00153,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2364960419945419}
{"item_id":"senseval2.d001.s057.t009","gold_sense_keys":["u.s.%1:15:00::"],"candidates":[{"index":1,"sense_key":"u.s.%1:14:00::","synset_id":"united_states_government.n.01"},{"index":2,"sense_key":"us%1:15:00::","synset_id":"united_states.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"us%1:15:00::","call_ids":["senseval2.d001.s057.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"us%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":708,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002026,"input_uncached_usd":0.001416,"input_cached_usd":0.0,"output_usd":0.0006100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3240337920142338}
{"item_id":"senseval2.d001.s058.t000","gold_sense_keys":["believe%2:31:03::"],"candidates":[{"index":1,"sense_key":"believe%2:31:00::","synset_id":"believe.v.01"},{"index":2,"sense_key":"believe%2:31:04::","synset_id":"think.v.01"},{"index":3,"sense_key":"believe%2:31:03::","synset_id":"believe.v.03"},{"index":4,"sense_key":"believe%2:31:01::","synset_id":"believe.v.04"},{"index":5,"sense_key":"believe%2:31:02::","synset_id":"believe.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"believe%2:31:04::","call_ids":["senseval2.d001.s058.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"believe%2:31:04::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":844,"cached_input_tokens":0,"output_tokens":254,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004228,"input_uncached_usd":0.0016879999999999998,"input_cached_usd":0.0,"output_usd":0.00254,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.7691048750421032}
{"item_id":"senseval2.d001.s058.t004","gold_sense_keys":["arise%2:42:00::"],"candidates":[{"index":1,"sense_key":"arise%2:42:00::","synset_id":"originate.v.01"},{"index":2,"sense_key":"arise%2:42:01::","synset_id":"arise.v.02"},{"index":3,"sense_key":"arise%2:38:00::","synset_id":"arise.v.03"},{"index":4,"sense_key":"arise%2:30:00::","synset_id":"arise.v.04"},{"index":5,"sense_key":"arise%2:38:04::","synset_id":"rise.v.01"},{"index":6,"sense_key":"arise%2:41:00::","synset_id":"rebel.v.01"},{"index":7,"sense_key":"arise%2:29:00::","synset_id":"get_up.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"arise%2:42:00::","call_ids":["senseval2.d001.s058.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"arise%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1001,"cached_input_tokens":0,"output_tokens":152,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003522,"input_uncached_usd":0.002002,"input_cached_usd":0.0,"output_usd":0.00152,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9963830839842558}
{"item_id":"senseval2.d001.s058.t006","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s058.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":872,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001864,"input_uncached_usd":0.0017439999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.218844834016636}
{"item_id":"senseval2.d001.s058.t008","gold_sense_keys":["often%4:02:00::","often%4:02:02::"],"candidates":[{"index":1,"sense_key":"often%4:02:00::","synset_id":"frequently.r.01"},{"index":2,"sense_key":"often%4:02:01::","synset_id":"much.r.05"},{"index":3,"sense_key":"often%4:02:02::","synset_id":"often.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"often%4:02:02::","call_ids":["senseval2.d001.s058.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"often%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":707,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0020239999999999998,"input_uncached_usd":0.001414,"input_cached_usd":0.0,"output_usd":0.0006100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.285471708048135}
{"item_id":"senseval2.d001.s058.t009","gold_sense_keys":["seem%2:39:01::"],"candidates":[{"index":1,"sense_key":"seem%2:39:00::","synset_id":"look.v.02"},{"index":2,"sense_key":"seem%2:39:01::","synset_id":"appear.v.04"},{"index":3,"sense_key":"seem%2:42:00::","synset_id":"seem.v.03"},{"index":4,"sense_key":"seem%2:39:02::","synset_id":"seem.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"seem%2:39:00::","call_ids":["senseval2.d001.s058.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"seem%2:39:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":814,"cached_input_tokens":0,"output_tokens":240,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004028,"input_uncached_usd":0.0016279999999999999,"input_cached_usd":0.0,"output_usd":0.0024000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.1420330830151215}
{"item_id":"senseval2.d001.s058.t010","gold_sense_keys":["develop%2:30:00::"],"candidates":[{"index":1,"sense_key":"develop%2:36:01::","synset_id":"develop.v.01"},{"index":2,"sense_key":"develop%2:36:00::","synset_id":"evolve.v.01"},{"index":3,"sense_key":"develop%2:30:06::","synset_id":"develop.v.03"},{"index":4,"sense_key":"develop%2:29:00::","synset_id":"grow.v.08"},{"index":5,"sense_key":"develop%2:42:00::","synset_id":"originate.v.01"},{"index":6,"sense_key":"develop%2:30:04::","synset_id":"build_up.v.05"},{"index":7,"sense_key":"develop%2:32:00::","synset_id":"explicate.v.02"},{"index":8,"sense_key":"develop%2:31:00::","synset_id":"train.v.01"},{"index":9,"sense_key":"develop%2:30:05::","synset_id":"develop.v.09"},{"index":10,"sense_key":"develop%2:30:00::","synset_id":"develop.v.10"},{"index":11,"sense_key":"develop%2:30:09::","synset_id":"modernize.v.02"},{"index":12,"sense_key":"develop%2:30:01::","synset_id":"develop.v.12"},{"index":13,"sense_key":"develop%2:36:09::","synset_id":"develop.v.13"},{"index":14,"sense_key":"develop%2:30:10::","synset_id":"develop.v.14"},{"index":15,"sense_key":"develop%2:39:00::","synset_id":"develop.v.15"},{"index":16,"sense_key":"develop%2:35:04::","synset_id":"develop.v.16"},{"index":17,"sense_key":"develop%2:33:03::","synset_id":"develop.v.17"},{"index":18,"sense_key":"develop%2:33:02::","synset_id":"develop.v.18"},{"index":19,"sense_key":"develop%2:30:13::","synset_id":"develop.v.19"},{"index":20,"sense_key":"develop%2:30:03::","synset_id":"break.v.53"},{"index":21,"sense_key":"develop%2:30:12::","synset_id":"develop.v.21"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"develop%2:30:00::","call_ids":["senseval2.d001.s058.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"develop%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1934,"cached_input_tokens":0,"output_tokens":186,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0057280000000000005,"input_uncached_usd":0.003868,"input_cached_usd":0.0,"output_usd":0.00186,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.5629199590766802}
{"item_id":"senseval2.d001.s058.t011","gold_sense_keys":["stage%1:26:00::","stage%1:28:00::"],"candidates":[{"index":1,"sense_key":"stage%1:28:00::","synset_id":"phase.n.01"},{"index":2,"sense_key":"stage%1:26:00::","synset_id":"degree.n.02"},{"index":3,"sense_key":"stage%1:06:00::","synset_id":"stage.n.03"},{"index":4,"sense_key":"stage%1:10:00::","synset_id":"stage.n.04"},{"index":5,"sense_key":"stage%1:06:01::","synset_id":"stagecoach.n.01"},{"index":6,"sense_key":"stage%1:04:00::","synset_id":"stage.n.06"},{"index":7,"sense_key":"stage%1:15:00::","synset_id":"stage.n.07"},{"index":8,"sense_key":"stage%1:06:02::","synset_id":"stage.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"stage%1:28:00::","call_ids":["senseval2.d001.s058.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"stage%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1119,"cached_input_tokens":0,"output_tokens":118,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003418,"input_uncached_usd":0.002238,"input_cached_usd":0.0,"output_usd":0.00118,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.498924999963492}
{"item_id":"senseval2.d001.s059.t000","gold_sense_keys":["often%4:02:00::","often%4:02:02::"],"candidates":[{"index":1,"sense_key":"often%4:02:00::","synset_id":"frequently.r.01"},{"index":2,"sense_key":"often%4:02:01::","synset_id":"much.r.05"},{"index":3,"sense_key":"often%4:02:02::","synset_id":"often.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"often%4:02:02::","call_ids":["senseval2.d001.s059.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"often%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":738,"cached_input_tokens":0,"output_tokens":94,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0024159999999999997,"input_uncached_usd":0.0014759999999999999,"input_cached_usd":0.0,"output_usd":0.0009400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.52355395804625}
{"item_id":"senseval2.d001.s059.t001","gold_sense_keys":["precede%2:42:00::"],"candidates":[{"index":1,"sense_key":"precede%2:42:00::","synset_id":"predate.v.01"},{"index":2,"sense_key":"precede%2:42:01::","synset_id":"precede.v.02"},{"index":3,"sense_key":"precede%2:41:00::","synset_id":"precede.v.03"},{"index":4,"sense_key":"precede%2:38:00::","synset_id":"precede.v.04"},{"index":5,"sense_key":"precede%2:32:03::","synset_id":"precede.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"precede%2:42:00::","call_ids":["senseval2.d001.s059.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"precede%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":896,"cached_input_tokens":0,"output_tokens":243,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004222,"input_uncached_usd":0.001792,"input_cached_usd":0.0,"output_usd":0.0024300000000000003,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.7846772089833394}
{"item_id":"senseval2.d001.s059.t002","gold_sense_keys":["development%1:22:01::"],"candidates":[{"index":1,"sense_key":"development%1:04:01::","synset_id":"development.n.01"},{"index":2,"sense_key":"development%1:22:02::","synset_id":"development.n.02"},{"index":3,"sense_key":"development%1:22:01::","synset_id":"growth.n.01"},{"index":4,"sense_key":"development%1:11:01::","synset_id":"development.n.04"},{"index":5,"sense_key":"development%1:04:02::","synset_id":"exploitation.n.01"},{"index":6,"sense_key":"development%1:15:00::","synset_id":"development.n.06"},{"index":7,"sense_key":"development%1:26:00::","synset_id":"development.n.07"},{"index":8,"sense_key":"development%1:22:00::","synset_id":"development.n.08"},{"index":9,"sense_key":"development%1:10:00::","synset_id":"development.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"development%1:22:01::","call_ids":["senseval2.d001.s059.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"development%1:22:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1249,"cached_input_tokens":0,"output_tokens":112,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0036179999999999997,"input_uncached_usd":0.002498,"input_cached_usd":0.0,"output_usd":0.0011200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7651210830081254}
{"item_id":"senseval2.d001.s059.t003","gold_sense_keys":["polyp%1:26:00::"],"candidates":[{"index":1,"sense_key":"polyp%1:26:00::","synset_id":"polyp.n.01"},{"index":2,"sense_key":"polyp%1:05:00::","synset_id":"polyp.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"polyp%1:26:00::","call_ids":["senseval2.d001.s059.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"polyp%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":759,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0016380000000000001,"input_uncached_usd":0.001518,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4162700000451878}
{"item_id":"senseval2.d001.s059.t005","gold_sense_keys":["case%1:11:00::"],"candidates":[{"index":1,"sense_key":"case%1:11:00::","synset_id":"case.n.01"},{"index":2,"sense_key":"case%1:26:00::","synset_id":"event.n.02"},{"index":3,"sense_key":"case%1:04:00::","synset_id":"lawsuit.n.01"},{"index":4,"sense_key":"case%1:09:00::","synset_id":"case.n.04"},{"index":5,"sense_key":"case%1:06:00::","synset_id":"case.n.05"},{"index":6,"sense_key":"case%1:18:03::","synset_id":"case.n.06"},{"index":7,"sense_key":"case%1:18:02::","synset_id":"subject.n.06"},{"index":8,"sense_key":"case%1:10:02::","synset_id":"case.n.08"},{"index":9,"sense_key":"case%1:10:01::","synset_id":"case.n.09"},{"index":10,"sense_key":"case%1:23:00::","synset_id":"case.n.10"},{"index":11,"sense_key":"case%1:10:00::","synset_id":"case.n.11"},{"index":12,"sense_key":"case%1:26:02::","synset_id":"case.n.12"},{"index":13,"sense_key":"case%1:18:04::","synset_id":"character.n.05"},{"index":14,"sense_key":"case%1:10:03::","synset_id":"font.n.01"},{"index":15,"sense_key":"case%1:08:00::","synset_id":"sheath.n.02"},{"index":16,"sense_key":"case%1:06:06::","synset_id":"shell.n.08"},{"index":17,"sense_key":"case%1:06:07::","synset_id":"casing.n.03"},{"index":18,"sense_key":"case%1:06:03::","synset_id":"case.n.18"},{"index":19,"sense_key":"case%1:06:02::","synset_id":"case.n.19"},{"index":20,"sense_key":"case%1:06:01::","synset_id":"case.n.20"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"case%1:11:00::","call_ids":["senseval2.d001.s059.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"case%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1839,"cached_input_tokens":0,"output_tokens":97,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004648,"input_uncached_usd":0.003678,"input_cached_usd":0.0,"output_usd":0.00097,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9512030000332743}
{"item_id":"senseval2.d001.s059.t006","gold_sense_keys":["become%2:30:00::"],"candidates":[{"index":1,"sense_key":"become%2:30:00::","synset_id":"become.v.01"},{"index":2,"sense_key":"become%2:42:01::","synset_id":"become.v.02"},{"index":3,"sense_key":"become%2:42:00::","synset_id":"become.v.03"},{"index":4,"sense_key":"become%2:42:03::","synset_id":"become.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"become%2:30:00::","call_ids":["senseval2.d001.s059.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"become%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":813,"cached_input_tokens":0,"output_tokens":174,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003366,"input_uncached_usd":0.0016259999999999998,"input_cached_usd":0.0,"output_usd":0.0017400000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.486637624911964}
{"item_id":"senseval2.d001.s059.t010","gold_sense_keys":["stage%1:26:00::","stage%1:28:00::"],"candidates":[{"index":1,"sense_key":"stage%1:28:00::","synset_id":"phase.n.01"},{"index":2,"sense_key":"stage%1:26:00::","synset_id":"degree.n.02"},{"index":3,"sense_key":"stage%1:06:00::","synset_id":"stage.n.03"},{"index":4,"sense_key":"stage%1:10:00::","synset_id":"stage.n.04"},{"index":5,"sense_key":"stage%1:06:01::","synset_id":"stagecoach.n.01"},{"index":6,"sense_key":"stage%1:04:00::","synset_id":"stage.n.06"},{"index":7,"sense_key":"stage%1:15:00::","synset_id":"stage.n.07"},{"index":8,"sense_key":"stage%1:06:02::","synset_id":"stage.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"stage%1:26:00::","call_ids":["senseval2.d001.s059.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"stage%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1150,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00438,"input_uncached_usd":0.0023,"input_cached_usd":0.0,"output_usd":0.0020800000000000003,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.388259792001918}
{"item_id":"senseval2.d001.s059.t011","gold_sense_keys":["progress%2:38:00::"],"candidates":[{"index":1,"sense_key":"progress%2:30:00::","synset_id":"progress.v.01"},{"index":2,"sense_key":"progress%2:38:00::","synset_id":"advance.v.01"},{"index":3,"sense_key":"progress%2:30:01::","synset_id":"build_up.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"progress%2:38:00::","call_ids":["senseval2.d001.s059.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"progress%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":809,"cached_input_tokens":0,"output_tokens":95,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002568,"input_uncached_usd":0.0016179999999999999,"input_cached_usd":0.0,"output_usd":0.0009500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2223759999033064}
{"item_id":"senseval2.d001.s059.t012","gold_sense_keys":["less%4:02:00::"],"candidates":[{"index":1,"sense_key":"less%4:02:00::","synset_id":"less.r.01"},{"index":2,"sense_key":"less%4:02:01::","synset_id":"less.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"less%4:02:00::","call_ids":["senseval2.d001.s059.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"less%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":692,"cached_input_tokens":0,"output_tokens":88,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002264,"input_uncached_usd":0.001384,"input_cached_usd":0.0,"output_usd":0.00088,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3910858749877661}
{"item_id":"senseval2.d001.s059.t013","gold_sense_keys":["severe%5:00:00:critical:03","severe%5:00:00:intense:00","severe%5:00:01:bad:00"],"candidates":[{"index":1,"sense_key":"severe%5:00:00:intense:00","synset_id":"severe.s.01"},{"index":2,"sense_key":"severe%5:00:00:strong:00","synset_id":"hard.s.01"},{"index":3,"sense_key":"severe%5:00:00:plain:01","synset_id":"austere.s.01"},{"index":4,"sense_key":"severe%5:00:02:nonindulgent:00","synset_id":"severe.s.04"},{"index":5,"sense_key":"severe%5:00:00:critical:03","synset_id":"dangerous.s.01"},{"index":6,"sense_key":"severe%5:00:01:bad:00","synset_id":"severe.s.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"severe%5:00:00:intense:00","call_ids":["senseval2.d001.s059.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"severe%5:00:00:intense:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1022,"cached_input_tokens":0,"output_tokens":182,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0038640000000000002,"input_uncached_usd":0.002044,"input_cached_usd":0.0,"output_usd":0.0018200000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.454272542032413}
{"item_id":"senseval2.d001.s059.t014","gold_sense_keys":["deadly%5:00:01:fatal:00"],"candidates":[{"index":1,"sense_key":"deadly%5:00:01:fatal:00","synset_id":"deadly.s.01"},{"index":2,"sense_key":"deadly%5:00:02:fatal:00","synset_id":"deadly.s.02"},{"index":3,"sense_key":"deadly%5:00:00:toxic:00","synset_id":"deadly.s.03"},{"index":4,"sense_key":"deadly%5:00:00:unpardonable:00","synset_id":"deadly.s.04"},{"index":5,"sense_key":"deadly%5:00:00:noxious:00","synset_id":"baneful.s.01"},{"index":6,"sense_key":"deadly%5:00:00:virulent:00","synset_id":"deadly.s.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"deadly%5:00:01:fatal:00","call_ids":["senseval2.d001.s059.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"deadly%5:00:01:fatal:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":961,"cached_input_tokens":0,"output_tokens":131,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003232,"input_uncached_usd":0.0019219999999999999,"input_cached_usd":0.0,"output_usd":0.0013100000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8434107080101967}
{"item_id":"senseval2.d001.s059.t015","gold_sense_keys":["genetic%3:01:02::"],"candidates":[{"index":1,"sense_key":"genetic%5:00:00:inheritable:00","synset_id":"familial.s.01"},{"index":2,"sense_key":"genetic%3:01:02::","synset_id":"genic.a.01"},{"index":3,"sense_key":"genetic%3:01:01::","synset_id":"genetic.a.03"},{"index":4,"sense_key":"genetic%3:01:00::","synset_id":"genetic.a.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"genetic%3:01:02::","call_ids":["senseval2.d001.s059.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"genetic%3:01:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":842,"cached_input_tokens":0,"output_tokens":118,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0028640000000000002,"input_uncached_usd":0.001684,"input_cached_usd":0.0,"output_usd":0.00118,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4209460000274703}
{"item_id":"senseval2.d001.s059.t016","gold_sense_keys":["damage%1:11:00::"],"candidates":[{"index":1,"sense_key":"damage%1:11:00::","synset_id":"damage.n.01"},{"index":2,"sense_key":"damage%1:11:01::","synset_id":"damage.n.02"},{"index":3,"sense_key":"damage%1:04:00::","synset_id":"damage.n.03"},{"index":4,"sense_key":"damage%1:21:00::","synset_id":"price.n.02"},{"index":5,"sense_key":"damage%1:04:01::","synset_id":"wrong.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"damage%1:11:00::","call_ids":["senseval2.d001.s059.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"damage%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":842,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001804,"input_uncached_usd":0.001684,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0570627499837428}
{"item_id":"senseval2.d001.s059.t017","gold_sense_keys":["occur%2:30:00::"],"candidates":[{"index":1,"sense_key":"occur%2:30:00::","synset_id":"happen.v.01"},{"index":2,"sense_key":"occur%2:31:00::","synset_id":"occur.v.02"},{"index":3,"sense_key":"occur%2:42:03::","synset_id":"occur.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"occur%2:30:00::","call_ids":["senseval2.d001.s059.t017__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"occur%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":789,"cached_input_tokens":0,"output_tokens":132,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0028980000000000004,"input_uncached_usd":0.001578,"input_cached_usd":0.0,"output_usd":0.0013200000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.016980457934551}
{"item_id":"senseval2.d001.s060.t001","gold_sense_keys":["student%1:18:00::"],"candidates":[{"index":1,"sense_key":"student%1:18:00::","synset_id":"student.n.01"},{"index":2,"sense_key":"student%1:18:01::","synset_id":"scholar.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"student%1:18:00::","call_ids":["senseval2.d001.s060.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"student%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":749,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001618,"input_uncached_usd":0.001498,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.954983374918811}
{"item_id":"senseval2.d001.s060.t002","gold_sense_keys":["begin%2:30:00::"],"candidates":[{"index":1,"sense_key":"begin%2:30:00::","synset_id":"get_down.v.07"},{"index":2,"sense_key":"begin%2:42:00::","synset_id":"begin.v.02"},{"index":3,"sense_key":"begin%2:30:01::","synset_id":"begin.v.03"},{"index":4,"sense_key":"begin%2:32:03::","synset_id":"begin.v.04"},{"index":5,"sense_key":"begin%2:42:02::","synset_id":"begin.v.05"},{"index":6,"sense_key":"begin%2:42:04::","synset_id":"begin.v.06"},{"index":7,"sense_key":"begin%2:42:03::","synset_id":"begin.v.07"},{"index":8,"sense_key":"begin%2:41:04::","synset_id":"begin.v.08"},{"index":9,"sense_key":"begin%2:41:00::","synset_id":"begin.v.09"},{"index":10,"sense_key":"begin%2:32:04::","synset_id":"begin.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"begin%2:30:00::","call_ids":["senseval2.d001.s060.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"begin%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1341,"cached_input_tokens":0,"output_tokens":167,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004352,"input_uncached_usd":0.002682,"input_cached_usd":0.0,"output_usd":0.00167,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.396434499998577}
{"item_id":"senseval2.d001.s060.t003","gold_sense_keys":["month%1:28:00::"],"candidates":[{"index":1,"sense_key":"month%1:28:01::","synset_id":"calendar_month.n.01"},{"index":2,"sense_key":"month%1:28:00::","synset_id":"month.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"month%1:28:00::","call_ids":["senseval2.d001.s060.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"month%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":733,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002046,"input_uncached_usd":0.0014659999999999999,"input_cached_usd":0.0,"output_usd":0.00058,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5724706249311566}
{"item_id":"senseval2.d001.s060.t004","gold_sense_keys":["tedious%5:00:00:uninteresting:00"],"candidates":[{"index":1,"sense_key":"tedious%5:00:00:uninteresting:00","synset_id":"boring.s.01"},{"index":2,"sense_key":"tedious%5:00:00:prolix:00","synset_id":"long-winded.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"tedious%5:00:00:uninteresting:00","call_ids":["senseval2.d001.s060.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"tedious%5:00:00:uninteresting:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":850,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00182,"input_uncached_usd":0.0017,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0328452499816194}
{"item_id":"senseval2.d001.s060.t005","gold_sense_keys":["often%4:02:02::"],"candidates":[{"index":1,"sense_key":"often%4:02:00::","synset_id":"frequently.r.01"},{"index":2,"sense_key":"often%4:02:01::","synset_id":"much.r.05"},{"index":3,"sense_key":"often%4:02:02::","synset_id":"often.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"often%4:02:00::","call_ids":["senseval2.d001.s060.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"often%4:02:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":781,"cached_input_tokens":0,"output_tokens":106,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002622,"input_uncached_usd":0.001562,"input_cached_usd":0.0,"output_usd":0.0010600000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6582697500707582}
{"item_id":"senseval2.d001.s060.t009","gold_sense_keys":["search%2:35:00::","search%2:39:00::"],"candidates":[{"index":1,"sense_key":"search%2:35:00::","synset_id":"search.v.01"},{"index":2,"sense_key":"search%2:39:00::","synset_id":"search.v.02"},{"index":3,"sense_key":"search%2:31:00::","synset_id":"research.v.02"},{"index":4,"sense_key":"search%2:35:01::","synset_id":"search.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"search%2:35:00::","call_ids":["senseval2.d001.s060.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"search%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":893,"cached_input_tokens":0,"output_tokens":157,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003356,"input_uncached_usd":0.0017859999999999998,"input_cached_usd":0.0,"output_usd":0.0015700000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.36403741594404}
{"item_id":"senseval2.d001.s060.t010","gold_sense_keys":["sign%1:26:00::"],"candidates":[{"index":1,"sense_key":"sign%1:10:05::","synset_id":"sign.n.01"},{"index":2,"sense_key":"sign%1:10:00::","synset_id":"sign.n.02"},{"index":3,"sense_key":"sign%1:10:03::","synset_id":"signal.n.01"},{"index":4,"sense_key":"sign%1:06:01::","synset_id":"signboard.n.01"},{"index":5,"sense_key":"sign%1:15:00::","synset_id":"sign_of_the_zodiac.n.01"},{"index":6,"sense_key":"sign%1:26:00::","synset_id":"sign.n.06"},{"index":7,"sense_key":"sign%1:24:00::","synset_id":"polarity.n.02"},{"index":8,"sense_key":"sign%1:11:00::","synset_id":"augury.n.01"},{"index":9,"sense_key":"sign%1:10:01::","synset_id":"sign.n.09"},{"index":10,"sense_key":"sign%1:10:06::","synset_id":"sign.n.10"},{"index":11,"sense_key":"sign%1:10:04::","synset_id":"sign.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"sign%1:10:05::","call_ids":["senseval2.d001.s060.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"sign%1:10:05::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1333,"cached_input_tokens":0,"output_tokens":119,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003856,"input_uncached_usd":0.002666,"input_cached_usd":0.0,"output_usd":0.00119,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5168815829092637}
{"item_id":"senseval2.d001.s060.t011","gold_sense_keys":["genetic%3:01:02::"],"candidates":[{"index":1,"sense_key":"genetic%5:00:00:inheritable:00","synset_id":"familial.s.01"},{"index":2,"sense_key":"genetic%3:01:02::","synset_id":"genic.a.01"},{"index":3,"sense_key":"genetic%3:01:01::","synset_id":"genetic.a.03"},{"index":4,"sense_key":"genetic%3:01:00::","synset_id":"genetic.a.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"genetic%3:01:02::","call_ids":["senseval2.d001.s060.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"genetic%3:01:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":885,"cached_input_tokens":0,"output_tokens":136,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00313,"input_uncached_usd":0.0017699999999999999,"input_cached_usd":0.0,"output_usd":0.00136,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.258008167031221}
{"item_id":"senseval2.d001.s060.t012","gold_sense_keys":["damage%1:11:00::"],"candidates":[{"index":1,"sense_key":"damage%1:11:00::","synset_id":"damage.n.01"},{"index":2,"sense_key":"damage%1:11:01::","synset_id":"damage.n.02"},{"index":3,"sense_key":"damage%1:04:00::","synset_id":"damage.n.03"},{"index":4,"sense_key":"damage%1:21:00::","synset_id":"price.n.02"},{"index":5,"sense_key":"damage%1:04:01::","synset_id":"wrong.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"damage%1:11:00::","call_ids":["senseval2.d001.s060.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"damage%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":885,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00189,"input_uncached_usd":0.0017699999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.7931120420107618}
{"item_id":"senseval2.d001.s061.t000","gold_sense_keys":["begin%2:30:00::"],"candidates":[{"index":1,"sense_key":"begin%2:30:00::","synset_id":"get_down.v.07"},{"index":2,"sense_key":"begin%2:42:00::","synset_id":"begin.v.02"},{"index":3,"sense_key":"begin%2:30:01::","synset_id":"begin.v.03"},{"index":4,"sense_key":"begin%2:32:03::","synset_id":"begin.v.04"},{"index":5,"sense_key":"begin%2:42:02::","synset_id":"begin.v.05"},{"index":6,"sense_key":"begin%2:42:04::","synset_id":"begin.v.06"},{"index":7,"sense_key":"begin%2:42:03::","synset_id":"begin.v.07"},{"index":8,"sense_key":"begin%2:41:04::","synset_id":"begin.v.08"},{"index":9,"sense_key":"begin%2:41:00::","synset_id":"begin.v.09"},{"index":10,"sense_key":"begin%2:32:04::","synset_id":"begin.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"begin%2:30:00::","call_ids":["senseval2.d001.s061.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"begin%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1282,"cached_input_tokens":0,"output_tokens":81,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003374,"input_uncached_usd":0.002564,"input_cached_usd":0.0,"output_usd":0.0008100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9169656250160187}
{"item_id":"senseval2.d001.s061.t001","gold_sense_keys":["confusing%5:00:00:unclear:00"],"candidates":[{"index":1,"sense_key":"confusing%5:00:00:disorienting:00","synset_id":"confusing.s.01"},{"index":2,"sense_key":"confusing%5:00:00:unclear:00","synset_id":"confusing.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"confusing%5:00:00:unclear:00","call_ids":["senseval2.d001.s061.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"confusing%5:00:00:unclear:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":742,"cached_input_tokens":0,"output_tokens":91,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002394,"input_uncached_usd":0.0014839999999999999,"input_cached_usd":0.0,"output_usd":0.0009100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1708818749757484}
{"item_id":"senseval2.d001.s061.t002","gold_sense_keys":["variety%1:07:00::","variety%1:14:01::"],"candidates":[{"index":1,"sense_key":"variety%1:14:01::","synset_id":"assortment.n.01"},{"index":2,"sense_key":"variety%1:07:00::","synset_id":"diverseness.n.01"},{"index":3,"sense_key":"variety%1:14:00::","synset_id":"variety.n.03"},{"index":4,"sense_key":"variety%1:10:00::","synset_id":"variety_show.n.01"},{"index":5,"sense_key":"variety%1:09:01::","synset_id":"kind.n.01"},{"index":6,"sense_key":"variety%1:07:01::","synset_id":"variety.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"variety%1:14:01::","call_ids":["senseval2.d001.s061.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"variety%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1019,"cached_input_tokens":0,"output_tokens":157,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003608,"input_uncached_usd":0.002038,"input_cached_usd":0.0,"output_usd":0.0015700000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.530498709063977}
{"item_id":"senseval2.d001.s061.t003","gold_sense_keys":["genetic%3:01:02::"],"candidates":[{"index":1,"sense_key":"genetic%5:00:00:inheritable:00","synset_id":"familial.s.01"},{"index":2,"sense_key":"genetic%3:01:02::","synset_id":"genic.a.01"},{"index":3,"sense_key":"genetic%3:01:01::","synset_id":"genetic.a.03"},{"index":4,"sense_key":"genetic%3:01:00::","synset_id":"genetic.a.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"genetic%3:01:02::","call_ids":["senseval2.d001.s061.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"genetic%3:01:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":826,"cached_input_tokens":0,"output_tokens":119,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002842,"input_uncached_usd":0.0016519999999999998,"input_cached_usd":0.0,"output_usd":0.00119,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8570916670141742}
{"item_id":"senseval2.d001.s061.t004","gold_sense_keys":["exist%2:42:00::"],"candidates":[{"index":1,"sense_key":"exist%2:42:00::","synset_id":"exist.v.01"},{"index":2,"sense_key":"exist%2:42:01::","synset_id":"exist.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"exist%2:42:00::","call_ids":["senseval2.d001.s061.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"exist%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":690,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0015,"input_uncached_usd":0.00138,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0312474999809638}
{"item_id":"senseval2.d001.s061.t005","gold_sense_keys":["only%4:02:01::"],"candidates":[{"index":1,"sense_key":"only%4:02:02::","synset_id":"merely.r.01"},{"index":2,"sense_key":"only%4:02:01::","synset_id":"entirely.r.02"},{"index":3,"sense_key":"only%4:02:00::","synset_id":"only.r.03"},{"index":4,"sense_key":"only%4:02:04::","synset_id":"only.r.04"},{"index":5,"sense_key":"only%4:02:06::","synset_id":"only.r.05"},{"index":6,"sense_key":"only%4:02:03::","synset_id":"only.r.06"},{"index":7,"sense_key":"only%4:02:05::","synset_id":"only.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"only%4:02:01::","call_ids":["senseval2.d001.s061.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"only%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":935,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00255,"input_uncached_usd":0.00187,"input_cached_usd":0.0,"output_usd":0.00068,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5119407919701189}
{"item_id":"senseval2.d001.s061.t006","gold_sense_keys":["polyp%1:26:00::"],"candidates":[{"index":1,"sense_key":"polyp%1:26:00::","synset_id":"polyp.n.01"},{"index":2,"sense_key":"polyp%1:05:00::","synset_id":"polyp.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"polyp%1:26:00::","call_ids":["senseval2.d001.s061.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"polyp%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":743,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001606,"input_uncached_usd":0.001486,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9969636250752956}
{"item_id":"senseval2.d001.s061.t008","gold_sense_keys":["cell%1:03:00::"],"candidates":[{"index":1,"sense_key":"cell%1:06:03::","synset_id":"cell.n.01"},{"index":2,"sense_key":"cell%1:03:00::","synset_id":"cell.n.02"},{"index":3,"sense_key":"cell%1:06:00::","synset_id":"cell.n.03"},{"index":4,"sense_key":"cell%1:14:00::","synset_id":"cell.n.04"},{"index":5,"sense_key":"cell%1:06:04::","synset_id":"cellular_telephone.n.01"},{"index":6,"sense_key":"cell%1:06:02::","synset_id":"cell.n.06"},{"index":7,"sense_key":"cell%1:06:01::","synset_id":"cell.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cell%1:03:00::","call_ids":["senseval2.d001.s061.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cell%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":979,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002078,"input_uncached_usd":0.001958,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9945557500468567}
{"item_id":"senseval2.d001.s061.t010","gold_sense_keys":["polyp%1:26:00::"],"candidates":[{"index":1,"sense_key":"polyp%1:26:00::","synset_id":"polyp.n.01"},{"index":2,"sense_key":"polyp%1:05:00::","synset_id":"polyp.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"polyp%1:26:00::","call_ids":["senseval2.d001.s061.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"polyp%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":743,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001606,"input_uncached_usd":0.001486,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8499269169988111}
{"item_id":"senseval2.d001.s061.t012","gold_sense_keys":["cell%1:03:00::"],"candidates":[{"index":1,"sense_key":"cell%1:06:03::","synset_id":"cell.n.01"},{"index":2,"sense_key":"cell%1:03:00::","synset_id":"cell.n.02"},{"index":3,"sense_key":"cell%1:06:00::","synset_id":"cell.n.03"},{"index":4,"sense_key":"cell%1:14:00::","synset_id":"cell.n.04"},{"index":5,"sense_key":"cell%1:06:04::","synset_id":"cellular_telephone.n.01"},{"index":6,"sense_key":"cell%1:06:02::","synset_id":"cell.n.06"},{"index":7,"sense_key":"cell%1:06:01::","synset_id":"cell.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cell%1:03:00::","call_ids":["senseval2.d001.s061.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cell%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":979,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002078,"input_uncached_usd":0.001958,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2928253330755979}
{"item_id":"senseval2.d001.s062.t001","gold_sense_keys":["coherent%3:00:00::"],"candidates":[{"index":1,"sense_key":"coherent%3:00:00::","synset_id":"coherent.a.01"},{"index":2,"sense_key":"coherent%5:00:00:rational:00","synset_id":"coherent.s.01"},{"index":3,"sense_key":"coherent%3:00:01::","synset_id":"coherent.a.03"},{"index":4,"sense_key":"coherent%5:00:00:adhesive:00","synset_id":"coherent.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"coherent%3:00:00::","call_ids":["senseval2.d001.s062.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"coherent%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":812,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001744,"input_uncached_usd":0.001624,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9340009170118719}
{"item_id":"senseval2.d001.s062.t002","gold_sense_keys":["picture%1:26:00::"],"candidates":[{"index":1,"sense_key":"picture%1:06:00::","synset_id":"picture.n.01"},{"index":2,"sense_key":"picture%1:06:01::","synset_id":"painting.n.01"},{"index":3,"sense_key":"picture%1:09:00::","synset_id":"mental_picture.n.01"},{"index":4,"sense_key":"picture%1:26:00::","synset_id":"picture.n.04"},{"index":5,"sense_key":"picture%1:10:00::","synset_id":"picture.n.05"},{"index":6,"sense_key":"picture%1:10:02::","synset_id":"movie.n.01"},{"index":7,"sense_key":"picture%1:10:01::","synset_id":"video.n.01"},{"index":8,"sense_key":"picture%1:10:03::","synset_id":"word_picture.n.01"},{"index":9,"sense_key":"picture%1:09:01::","synset_id":"picture.n.09"},{"index":10,"sense_key":"picture%1:06:02::","synset_id":"photograph.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"picture%1:09:00::","call_ids":["senseval2.d001.s062.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"picture%1:09:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1339,"cached_input_tokens":0,"output_tokens":198,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004658,"input_uncached_usd":0.002678,"input_cached_usd":0.0,"output_usd":0.00198,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1641704170033336}
{"item_id":"senseval2.d001.s062.t003","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s062.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":864,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001848,"input_uncached_usd":0.001728,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8470385840628296}
{"item_id":"senseval2.d001.s062.t004","gold_sense_keys":["development%1:22:02::"],"candidates":[{"index":1,"sense_key":"development%1:04:01::","synset_id":"development.n.01"},{"index":2,"sense_key":"development%1:22:02::","synset_id":"development.n.02"},{"index":3,"sense_key":"development%1:22:01::","synset_id":"growth.n.01"},{"index":4,"sense_key":"development%1:11:01::","synset_id":"development.n.04"},{"index":5,"sense_key":"development%1:04:02::","synset_id":"exploitation.n.01"},{"index":6,"sense_key":"development%1:15:00::","synset_id":"development.n.06"},{"index":7,"sense_key":"development%1:26:00::","synset_id":"development.n.07"},{"index":8,"sense_key":"development%1:22:00::","synset_id":"development.n.08"},{"index":9,"sense_key":"development%1:10:00::","synset_id":"development.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"development%1:22:02::","call_ids":["senseval2.d001.s062.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"development%1:22:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1210,"cached_input_tokens":0,"output_tokens":94,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00336,"input_uncached_usd":0.00242,"input_cached_usd":0.0,"output_usd":0.0009400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7485827920027077}
{"item_id":"senseval2.d001.s062.t005","gold_sense_keys":["emerge%2:30:01::"],"candidates":[{"index":1,"sense_key":"emerge%2:30:00::","synset_id":"emerge.v.01"},{"index":2,"sense_key":"emerge%2:30:02::","synset_id":"issue.v.04"},{"index":3,"sense_key":"emerge%2:30:01::","synset_id":"emerge.v.03"},{"index":4,"sense_key":"emerge%2:38:00::","synset_id":"emerge.v.04"},{"index":5,"sense_key":"emerge%2:42:00::","synset_id":"come_forth.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"emerge%2:30:01::","call_ids":["senseval2.d001.s062.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"emerge%2:30:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":813,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001746,"input_uncached_usd":0.0016259999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9700029999949038}
{"item_id":"senseval2.d001.s063.t000","gold_sense_keys":["copy%1:06:00::"],"candidates":[{"index":1,"sense_key":"copy%1:10:00::","synset_id":"transcript.n.02"},{"index":2,"sense_key":"copy%1:06:00::","synset_id":"copy.n.02"},{"index":3,"sense_key":"copy%1:10:01::","synset_id":"copy.n.03"},{"index":4,"sense_key":"copy%1:10:02::","synset_id":"copy.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"copy%1:06:00::","call_ids":["senseval2.d001.s063.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"copy%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":743,"cached_input_tokens":0,"output_tokens":102,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002506,"input_uncached_usd":0.001486,"input_cached_usd":0.0,"output_usd":0.00102,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.152846041950397}
{"item_id":"senseval2.d001.s063.t001","gold_sense_keys":["certain%5:00:00:definite:00"],"candidates":[{"index":1,"sense_key":"certain%5:00:00:definite:00","synset_id":"certain.s.01"},{"index":2,"sense_key":"certain%3:00:02::","synset_id":"certain.a.02"},{"index":3,"sense_key":"certain%3:00:01::","synset_id":"certain.a.03"},{"index":4,"sense_key":"certain%3:00:03::","synset_id":"certain.a.04"},{"index":5,"sense_key":"certain%3:00:05::","synset_id":"sealed.a.01"},{"index":6,"sense_key":"certain%5:00:00:reliable:00","synset_id":"certain.s.02"},{"index":7,"sense_key":"certain%5:00:00:careful:00","synset_id":"certain.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"certain%5:00:00:definite:00","call_ids":["senseval2.d001.s063.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"certain%5:00:00:definite:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":999,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0021179999999999997,"input_uncached_usd":0.0019979999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0562516660429537}
{"item_id":"senseval2.d001.s063.t003","gold_sense_keys":["knock_out%2:30:01::"],"candidates":[{"index":1,"sense_key":"knock_out%2:30:01::","synset_id":"knock_out.v.01"},{"index":2,"sense_key":"knock_out%2:35:00::","synset_id":"knock_cold.v.01"},{"index":3,"sense_key":"knock_out%2:30:00::","synset_id":"knock_out.v.03"},{"index":4,"sense_key":"knock_out%2:37:00::","synset_id":"knock_out.v.04"},{"index":5,"sense_key":"knock_out%2:30:02::","synset_id":"knock_out.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"knock_out%2:30:01::","call_ids":["senseval2.d001.s063.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"knock_out%2:30:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":789,"cached_input_tokens":0,"output_tokens":101,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002588,"input_uncached_usd":0.001578,"input_cached_usd":0.0,"output_usd":0.00101,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.667844958952628}
{"item_id":"senseval2.d001.s063.t004","gold_sense_keys":["polyp%1:26:00::"],"candidates":[{"index":1,"sense_key":"polyp%1:26:00::","synset_id":"polyp.n.01"},{"index":2,"sense_key":"polyp%1:05:00::","synset_id":"polyp.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"polyp%1:26:00::","call_ids":["senseval2.d001.s063.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"polyp%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":703,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001526,"input_uncached_usd":0.001406,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0560376659268513}
{"item_id":"senseval2.d001.s063.t005","gold_sense_keys":["develop%2:30:00::","develop%2:42:00::"],"candidates":[{"index":1,"sense_key":"develop%2:36:01::","synset_id":"develop.v.01"},{"index":2,"sense_key":"develop%2:36:00::","synset_id":"evolve.v.01"},{"index":3,"sense_key":"develop%2:30:06::","synset_id":"develop.v.03"},{"index":4,"sense_key":"develop%2:29:00::","synset_id":"grow.v.08"},{"index":5,"sense_key":"develop%2:42:00::","synset_id":"originate.v.01"},{"index":6,"sense_key":"develop%2:30:04::","synset_id":"build_up.v.05"},{"index":7,"sense_key":"develop%2:32:00::","synset_id":"explicate.v.02"},{"index":8,"sense_key":"develop%2:31:00::","synset_id":"train.v.01"},{"index":9,"sense_key":"develop%2:30:05::","synset_id":"develop.v.09"},{"index":10,"sense_key":"develop%2:30:00::","synset_id":"develop.v.10"},{"index":11,"sense_key":"develop%2:30:09::","synset_id":"modernize.v.02"},{"index":12,"sense_key":"develop%2:30:01::","synset_id":"develop.v.12"},{"index":13,"sense_key":"develop%2:36:09::","synset_id":"develop.v.13"},{"index":14,"sense_key":"develop%2:30:10::","synset_id":"develop.v.14"},{"index":15,"sense_key":"develop%2:39:00::","synset_id":"develop.v.15"},{"index":16,"sense_key":"develop%2:35:04::","synset_id":"develop.v.16"},{"index":17,"sense_key":"develop%2:33:03::","synset_id":"develop.v.17"},{"index":18,"sense_key":"develop%2:33:02::","synset_id":"develop.v.18"},{"index":19,"sense_key":"develop%2:30:13::","synset_id":"develop.v.19"},{"index":20,"sense_key":"develop%2:30:03::","synset_id":"break.v.53"},{"index":21,"sense_key":"develop%2:30:12::","synset_id":"develop.v.21"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"develop%2:42:00::","call_ids":["senseval2.d001.s063.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"develop%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1909,"cached_input_tokens":0,"output_tokens":202,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0058379999999999994,"input_uncached_usd":0.0038179999999999998,"input_cached_usd":0.0,"output_usd":0.00202,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.3607205419102684}
{"item_id":"senseval2.d001.s064.t000","gold_sense_keys":["copy%1:06:00::"],"candidates":[{"index":1,"sense_key":"copy%1:10:00::","synset_id":"transcript.n.02"},{"index":2,"sense_key":"copy%1:06:00::","synset_id":"copy.n.02"},{"index":3,"sense_key":"copy%1:10:01::","synset_id":"copy.n.03"},{"index":4,"sense_key":"copy%1:10:02::","synset_id":"copy.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"copy%1:06:00::","call_ids":["senseval2.d001.s064.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"copy%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":726,"cached_input_tokens":0,"output_tokens":86,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002312,"input_uncached_usd":0.001452,"input_cached_usd":0.0,"output_usd":0.0008600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6104957920033485}
{"item_id":"senseval2.d001.s064.t001","gold_sense_keys":["second%5:00:00:ordinal:00"],"candidates":[{"index":1,"sense_key":"second%5:00:00:ordinal:00","synset_id":"second.s.01"},{"index":2,"sense_key":"second%3:00:00::","synset_id":"second.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"second%5:00:00:ordinal:00","call_ids":["senseval2.d001.s064.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"second%5:00:00:ordinal:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":634,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0013880000000000001,"input_uncached_usd":0.001268,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0135785000165924}
{"item_id":"senseval2.d001.s064.t003","gold_sense_keys":["then%4:02:00::"],"candidates":[{"index":1,"sense_key":"then%4:02:00::","synset_id":"then.r.01"},{"index":2,"sense_key":"then%4:02:02::","synset_id":"then.r.02"},{"index":3,"sense_key":"then%4:02:01::","synset_id":"then.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"then%4:02:00::","call_ids":["senseval2.d001.s064.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"then%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":675,"cached_input_tokens":0,"output_tokens":85,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0021999999999999997,"input_uncached_usd":0.0013499999999999999,"input_cached_usd":0.0,"output_usd":0.0008500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1772148329764605}
{"item_id":"senseval2.d001.s064.t004","gold_sense_keys":["delete%2:35:00::"],"candidates":[{"index":1,"sense_key":"delete%2:35:00::","synset_id":"delete.v.01"},{"index":2,"sense_key":"delete%2:32:00::","synset_id":"erase.v.03"},{"index":3,"sense_key":"delete%2:30:00::","synset_id":"edit.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"delete%2:35:00::","call_ids":["senseval2.d001.s064.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"delete%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":687,"cached_input_tokens":0,"output_tokens":104,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0024140000000000003,"input_uncached_usd":0.001374,"input_cached_usd":0.0,"output_usd":0.0010400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7571421669563279}
{"item_id":"senseval2.d001.s064.t005","gold_sense_keys":["polyp%1:26:00::"],"candidates":[{"index":1,"sense_key":"polyp%1:26:00::","synset_id":"polyp.n.01"},{"index":2,"sense_key":"polyp%1:05:00::","synset_id":"polyp.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"polyp%1:26:00::","call_ids":["senseval2.d001.s064.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"polyp%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":686,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001492,"input_uncached_usd":0.001372,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4919776249444112}
{"item_id":"senseval2.d001.s064.t007","gold_sense_keys":["malignancy%1:26:00::"],"candidates":[{"index":1,"sense_key":"malignancy%1:26:00::","synset_id":"malignancy.n.01"},{"index":2,"sense_key":"malignancy%1:07:00::","synset_id":"malignity.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"malignancy%1:26:00::","call_ids":["senseval2.d001.s064.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"malignancy%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":632,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001384,"input_uncached_usd":0.001264,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8786168749211356}
{"item_id":"senseval2.d001.s065.t000","gold_sense_keys":["clear%3:00:00::"],"candidates":[{"index":1,"sense_key":"clear%3:00:00::","synset_id":"clear.a.01"},{"index":2,"sense_key":"clear%5:00:00:clearheaded:00","synset_id":"clear.s.01"},{"index":3,"sense_key":"clear%5:00:00:unobstructed:00","synset_id":"clear.s.02"},{"index":4,"sense_key":"clear%3:00:02::","synset_id":"clear.a.04"},{"index":5,"sense_key":"clear%5:00:02:free:00","synset_id":"clear.s.03"},{"index":6,"sense_key":"clear%5:00:00:untroubled:00","synset_id":"clear.s.04"},{"index":7,"sense_key":"clear%5:00:01:pure:02","synset_id":"clean.s.02"},{"index":8,"sense_key":"clear%5:00:00:unencumbered:00","synset_id":"clear.s.06"},{"index":9,"sense_key":"clear%5:00:00:distinct:00","synset_id":"clear.s.07"},{"index":10,"sense_key":"clear%3:00:04::","synset_id":"well-defined.a.02"},{"index":11,"sense_key":"clear%3:00:03::","synset_id":"clear.a.11"},{"index":12,"sense_key":"clear%5:00:00:unqualified:02","synset_id":"clean.s.01"},{"index":13,"sense_key":"clear%5:00:00:perfect:00","synset_id":"clear.s.09"},{"index":14,"sense_key":"clear%5:00:00:net:00","synset_id":"clear.s.10"},{"index":15,"sense_key":"clear%5:00:00:legible:00","synset_id":"clear.s.11"},{"index":16,"sense_key":"clear%5:00:00:innocent:00","synset_id":"absolved.s.01"},{"index":17,"sense_key":"clear%5:00:00:discerning:00","synset_id":"clear.s.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"clear%3:00:00::","call_ids":["senseval2.d001.s065.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"clear%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1539,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003198,"input_uncached_usd":0.003078,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0263274169992656}
{"item_id":"senseval2.d001.s065.t003","gold_sense_keys":["damage%2:30:00::"],"candidates":[{"index":1,"sense_key":"damage%2:30:00::","synset_id":"damage.v.01"},{"index":2,"sense_key":"damage%2:30:01::","synset_id":"damage.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"damage%2:30:00::","call_ids":["senseval2.d001.s065.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"damage%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":561,"cached_input_tokens":0,"output_tokens":130,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002422,"input_uncached_usd":0.001122,"input_cached_usd":0.0,"output_usd":0.0013000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6521613749209791}
{"item_id":"senseval2.d001.s065.t006","gold_sense_keys":["develop%2:30:00::","develop%2:42:00::"],"candidates":[{"index":1,"sense_key":"develop%2:36:01::","synset_id":"develop.v.01"},{"index":2,"sense_key":"develop%2:36:00::","synset_id":"evolve.v.01"},{"index":3,"sense_key":"develop%2:30:06::","synset_id":"develop.v.03"},{"index":4,"sense_key":"develop%2:29:00::","synset_id":"grow.v.08"},{"index":5,"sense_key":"develop%2:42:00::","synset_id":"originate.v.01"},{"index":6,"sense_key":"develop%2:30:04::","synset_id":"build_up.v.05"},{"index":7,"sense_key":"develop%2:32:00::","synset_id":"explicate.v.02"},{"index":8,"sense_key":"develop%2:31:00::","synset_id":"train.v.01"},{"index":9,"sense_key":"develop%2:30:05::","synset_id":"develop.v.09"},{"index":10,"sense_key":"develop%2:30:00::","synset_id":"develop.v.10"},{"index":11,"sense_key":"develop%2:30:09::","synset_id":"modernize.v.02"},{"index":12,"sense_key":"develop%2:30:01::","synset_id":"develop.v.12"},{"index":13,"sense_key":"develop%2:36:09::","synset_id":"develop.v.13"},{"index":14,"sense_key":"develop%2:30:10::","synset_id":"develop.v.14"},{"index":15,"sense_key":"develop%2:39:00::","synset_id":"develop.v.15"},{"index":16,"sense_key":"develop%2:35:04::","synset_id":"develop.v.16"},{"index":17,"sense_key":"develop%2:33:03::","synset_id":"develop.v.17"},{"index":18,"sense_key":"develop%2:33:02::","synset_id":"develop.v.18"},{"index":19,"sense_key":"develop%2:30:13::","synset_id":"develop.v.19"},{"index":20,"sense_key":"develop%2:30:03::","synset_id":"break.v.53"},{"index":21,"sense_key":"develop%2:30:12::","synset_id":"develop.v.21"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"develop%2:42:00::","call_ids":["senseval2.d001.s065.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"develop%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1835,"cached_input_tokens":0,"output_tokens":153,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0052,"input_uncached_usd":0.0036699999999999997,"input_cached_usd":0.0,"output_usd":0.0015300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.538334082928486}
{"item_id":"senseval2.d001.s066.t000","gold_sense_keys":["report%1:10:03::"],"candidates":[{"index":1,"sense_key":"report%1:10:03::","synset_id":"report.n.01"},{"index":2,"sense_key":"report%1:10:01::","synset_id":"report.n.02"},{"index":3,"sense_key":"report%1:10:00::","synset_id":"report.n.03"},{"index":4,"sense_key":"report%1:11:00::","synset_id":"report.n.04"},{"index":5,"sense_key":"report%1:10:04::","synset_id":"report_card.n.01"},{"index":6,"sense_key":"report%1:10:02::","synset_id":"composition.n.08"},{"index":7,"sense_key":"report%1:09:00::","synset_id":"reputation.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"report%1:10:03::","call_ids":["senseval2.d001.s066.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"report%1:10:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":905,"cached_input_tokens":0,"output_tokens":71,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00252,"input_uncached_usd":0.00181,"input_cached_usd":0.0,"output_usd":0.00071,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.234617874957621}
{"item_id":"senseval2.d001.s066.t001","gold_sense_keys":["galvanize%2:37:00::"],"candidates":[{"index":1,"sense_key":"galvanize%2:37:00::","synset_id":"startle.v.01"},{"index":2,"sense_key":"galvanize%2:35:00::","synset_id":"galvanize.v.02"},{"index":3,"sense_key":"galvanize%2:29:02::","synset_id":"galvanize.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"galvanize%2:37:00::","call_ids":["senseval2.d001.s066.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"galvanize%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":622,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001364,"input_uncached_usd":0.0012439999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.466310124960728}
{"item_id":"senseval2.d001.s066.t002","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":2,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":3,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"other%3:00:00::","call_ids":["senseval2.d001.s066.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":691,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001502,"input_uncached_usd":0.001382,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3121406669961289}
{"item_id":"senseval2.d001.s067.t000","gold_sense_keys":["confirming%5:00:00:supportive:00"],"candidates":[{"index":1,"sense_key":"confirming%3:00:02::","synset_id":"positive.a.04"},{"index":2,"sense_key":"confirming%5:00:00:supportive:00","synset_id":"collateral.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"confirming%5:00:00:supportive:00","call_ids":["senseval2.d001.s067.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"confirming%5:00:00:supportive:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":604,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001328,"input_uncached_usd":0.0012079999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2907962079625577}
{"item_id":"senseval2.d001.s067.t001","gold_sense_keys":["evidence%1:09:00::","evidence%1:10:01::"],"candidates":[{"index":1,"sense_key":"evidence%1:09:00::","synset_id":"evidence.n.01"},{"index":2,"sense_key":"evidence%1:10:01::","synset_id":"evidence.n.02"},{"index":3,"sense_key":"evidence%1:10:00::","synset_id":"evidence.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"evidence%1:09:00::","call_ids":["senseval2.d001.s067.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"evidence%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":643,"cached_input_tokens":0,"output_tokens":162,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002906,"input_uncached_usd":0.001286,"input_cached_usd":0.0,"output_usd":0.0016200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.979304958949797}
{"item_id":"senseval2.d001.s067.t002","gold_sense_keys":["need%2:42:00::"],"candidates":[{"index":1,"sense_key":"need%2:42:00::","synset_id":"necessitate.v.01"},{"index":2,"sense_key":"need%2:34:00::","synset_id":"want.v.02"},{"index":3,"sense_key":"need%2:34:01::","synset_id":"need.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"need%2:34:01::","call_ids":["senseval2.d001.s067.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"need%2:34:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":624,"cached_input_tokens":0,"output_tokens":143,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002678,"input_uncached_usd":0.001248,"input_cached_usd":0.0,"output_usd":0.00143,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.3308731670258567}
{"item_id":"senseval2.d001.s067.t004","gold_sense_keys":["loss%1:04:00::"],"candidates":[{"index":1,"sense_key":"loss%1:21:01::","synset_id":"loss.n.01"},{"index":2,"sense_key":"loss%1:22:00::","synset_id":"loss.n.02"},{"index":3,"sense_key":"loss%1:04:00::","synset_id":"loss.n.03"},{"index":4,"sense_key":"loss%1:07:00::","synset_id":"loss.n.04"},{"index":5,"sense_key":"loss%1:11:01::","synset_id":"loss.n.05"},{"index":6,"sense_key":"loss%1:21:00::","synset_id":"loss.n.06"},{"index":7,"sense_key":"loss%1:11:00::","synset_id":"personnel_casualty.n.01"},{"index":8,"sense_key":"loss%1:11:02::","synset_id":"passing.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"loss%1:04:00::","call_ids":["senseval2.d001.s067.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"loss%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":921,"cached_input_tokens":0,"output_tokens":104,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002882,"input_uncached_usd":0.0018419999999999999,"input_cached_usd":0.0,"output_usd":0.0010400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.3490983749506995}
{"item_id":"senseval2.d001.s067.t005","gold_sense_keys":["critical%5:00:00:indispensable:00"],"candidates":[{"index":1,"sense_key":"critical%3:00:01::","synset_id":"critical.a.01"},{"index":2,"sense_key":"critical%3:00:04::","synset_id":"critical.a.02"},{"index":3,"sense_key":"critical%3:00:02::","synset_id":"critical.a.03"},{"index":4,"sense_key":"critical%5:00:00:indispensable:00","synset_id":"critical.s.01"},{"index":5,"sense_key":"critical%5:00:00:crucial:00","synset_id":"critical.s.02"},{"index":6,"sense_key":"critical%3:00:03::","synset_id":"critical.a.06"},{"index":7,"sense_key":"critical%3:01:00::","synset_id":"critical.a.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"critical%5:00:00:indispensable:00","call_ids":["senseval2.d001.s067.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"critical%5:00:00:indispensable:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":908,"cached_input_tokens":0,"output_tokens":126,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0030759999999999997,"input_uncached_usd":0.001816,"input_cached_usd":0.0,"output_usd":0.00126,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6471942499047145}
{"item_id":"senseval2.d001.s067.t006","gold_sense_keys":["development%1:22:02::"],"candidates":[{"index":1,"sense_key":"development%1:04:01::","synset_id":"development.n.01"},{"index":2,"sense_key":"development%1:22:02::","synset_id":"development.n.02"},{"index":3,"sense_key":"development%1:22:01::","synset_id":"growth.n.01"},{"index":4,"sense_key":"development%1:11:01::","synset_id":"development.n.04"},{"index":5,"sense_key":"development%1:04:02::","synset_id":"exploitation.n.01"},{"index":6,"sense_key":"development%1:15:00::","synset_id":"development.n.06"},{"index":7,"sense_key":"development%1:26:00::","synset_id":"development.n.07"},{"index":8,"sense_key":"development%1:22:00::","synset_id":"development.n.08"},{"index":9,"sense_key":"development%1:10:00::","synset_id":"development.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"development%1:22:02::","call_ids":["senseval2.d001.s067.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"development%1:22:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1118,"cached_input_tokens":0,"output_tokens":163,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0038659999999999996,"input_uncached_usd":0.0022359999999999997,"input_cached_usd":0.0,"output_usd":0.0016300000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9956845419947058}
{"item_id":"senseval2.d001.s067.t007","gold_sense_keys":["common%3:00:01::","common%5:00:00:familiar:02"],"candidates":[{"index":1,"sense_key":"common%3:00:02::","synset_id":"common.a.01"},{"index":2,"sense_key":"common%3:00:01::","synset_id":"common.a.02"},{"index":3,"sense_key":"common%5:00:00:shared:00","synset_id":"common.s.01"},{"index":4,"sense_key":"common%5:00:00:familiar:02","synset_id":"common.s.02"},{"index":5,"sense_key":"common%5:00:00:informal:02","synset_id":"common.s.03"},{"index":6,"sense_key":"common%5:00:00:lowborn:00","synset_id":"common.s.04"},{"index":7,"sense_key":"common%5:00:00:inferior:02","synset_id":"coarse.s.02"},{"index":8,"sense_key":"common%5:00:00:unrefined:01","synset_id":"coarse.s.01"},{"index":9,"sense_key":"common%5:00:00:ordinary:00","synset_id":"common.s.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"common%5:00:00:familiar:02","call_ids":["senseval2.d001.s067.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"common%5:00:00:familiar:02","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1083,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002746,"input_uncached_usd":0.002166,"input_cached_usd":0.0,"output_usd":0.00058,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4926826669834554}
{"item_id":"senseval2.d001.s067.t009","gold_sense_keys":["say%2:32:00::","say%2:32:15::"],"candidates":[{"index":1,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":6,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":9,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":10,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":11,"sense_key":"say%2:32:09::","synset_id":"say.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"say%2:32:00::","call_ids":["senseval2.d001.s067.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"say%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1112,"cached_input_tokens":0,"output_tokens":232,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004543999999999999,"input_uncached_usd":0.002224,"input_cached_usd":0.0,"output_usd":0.00232,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.7439906670479104}
{"item_id":"senseval2.d001.s068.t000","gold_sense_keys":["yet%4:02:02::"],"candidates":[{"index":1,"sense_key":"yet%4:02:02::","synset_id":"yet.r.01"},{"index":2,"sense_key":"yet%4:02:04::","synset_id":"so_far.r.01"},{"index":3,"sense_key":"yet%4:02:03::","synset_id":"even.r.03"},{"index":4,"sense_key":"yet%4:02:00::","synset_id":"yet.r.04"},{"index":5,"sense_key":"yet%4:02:05::","synset_id":"so_far.r.03"},{"index":6,"sense_key":"yet%4:02:06::","synset_id":"however.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"yet%4:02:02::","call_ids":["senseval2.d001.s068.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"yet%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":910,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0024200000000000003,"input_uncached_usd":0.00182,"input_cached_usd":0.0,"output_usd":0.0006000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4067162499995902}
{"item_id":"senseval2.d001.s068.t001","gold_sense_keys":["nail%2:40:00::"],"candidates":[{"index":1,"sense_key":"nail%2:35:00::","synset_id":"nail.v.01"},{"index":2,"sense_key":"nail%2:35:01::","synset_id":"collar.v.01"},{"index":3,"sense_key":"nail%2:35:02::","synset_id":"smash.v.01"},{"index":4,"sense_key":"nail%2:41:01::","synset_id":"nail_down.v.01"},{"index":5,"sense_key":"nail%2:41:00::","synset_id":"breeze_through.v.01"},{"index":6,"sense_key":"nail%2:40:00::","synset_id":"pinpoint.v.01"},{"index":7,"sense_key":"nail%2:33:00::","synset_id":"complete.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"nail%2:40:00::","call_ids":["senseval2.d001.s068.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"nail%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":869,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001858,"input_uncached_usd":0.001738,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8955384580185637}
{"item_id":"senseval2.d001.s068.t002","gold_sense_keys":["identity%1:09:00::"],"candidates":[{"index":1,"sense_key":"identity%1:07:00::","synset_id":"identity.n.01"},{"index":2,"sense_key":"identity%1:09:00::","synset_id":"identity.n.02"},{"index":3,"sense_key":"identity%1:24:01::","synset_id":"identity.n.03"},{"index":4,"sense_key":"identity%1:07:02::","synset_id":"identity.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"identity%1:09:00::","call_ids":["senseval2.d001.s068.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"identity%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":726,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001572,"input_uncached_usd":0.001452,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1434371249051765}
{"item_id":"senseval2.d001.s068.t004","gold_sense_keys":["damage%2:30:00::"],"candidates":[{"index":1,"sense_key":"damage%2:30:00::","synset_id":"damage.v.01"},{"index":2,"sense_key":"damage%2:30:01::","synset_id":"damage.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"damage%2:30:00::","call_ids":["senseval2.d001.s068.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"damage%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":551,"cached_input_tokens":0,"output_tokens":161,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002712,"input_uncached_usd":0.001102,"input_cached_usd":0.0,"output_usd":0.00161,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9798743329010904}
{"item_id":"senseval2.d001.s068.t005","gold_sense_keys":["flip%2:30:00::"],"candidates":[{"index":1,"sense_key":"flip%2:38:00::","synset_id":"flip.v.01"},{"index":2,"sense_key":"flip%2:35:01::","synset_id":"throw.v.06"},{"index":3,"sense_key":"flip%2:39:10::","synset_id":"flick.v.02"},{"index":4,"sense_key":"flip%2:38:02::","synset_id":"flip.v.04"},{"index":5,"sense_key":"flip%2:38:01::","synset_id":"flip.v.05"},{"index":6,"sense_key":"flip%2:35:00::","synset_id":"flip.v.06"},{"index":7,"sense_key":"flip%2:35:05::","synset_id":"flip.v.07"},{"index":8,"sense_key":"flip%2:35:03::","synset_id":"flip.v.08"},{"index":9,"sense_key":"flip%2:31:00::","synset_id":"flip.v.09"},{"index":10,"sense_key":"flip%2:30:13::","synset_id":"flip.v.10"},{"index":11,"sense_key":"flip%2:30:00::","synset_id":"interchange.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"flip%2:35:01::","call_ids":["senseval2.d001.s068.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"flip%2:35:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1111,"cached_input_tokens":0,"output_tokens":179,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004012,"input_uncached_usd":0.002222,"input_cached_usd":0.0,"output_usd":0.0017900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":8.738047874998301}
{"item_id":"senseval2.d001.s068.t006","gold_sense_keys":["colon%1:08:00::"],"candidates":[{"index":1,"sense_key":"colon%1:08:00::","synset_id":"colon.n.01"},{"index":2,"sense_key":"colon%1:23:01::","synset_id":"colon.n.02"},{"index":3,"sense_key":"colon%1:23:00::","synset_id":"colon.n.03"},{"index":4,"sense_key":"colon%1:15:00::","synset_id":"colon.n.04"},{"index":5,"sense_key":"colon%1:10:00::","synset_id":"colon.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"colon%1:08:00::","call_ids":["senseval2.d001.s068.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"colon%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":741,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0016020000000000001,"input_uncached_usd":0.001482,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5073728749994189}
{"item_id":"senseval2.d001.s068.t007","gold_sense_keys":["cell%1:03:00::"],"candidates":[{"index":1,"sense_key":"cell%1:06:03::","synset_id":"cell.n.01"},{"index":2,"sense_key":"cell%1:03:00::","synset_id":"cell.n.02"},{"index":3,"sense_key":"cell%1:06:00::","synset_id":"cell.n.03"},{"index":4,"sense_key":"cell%1:14:00::","synset_id":"cell.n.04"},{"index":5,"sense_key":"cell%1:06:04::","synset_id":"cellular_telephone.n.01"},{"index":6,"sense_key":"cell%1:06:02::","synset_id":"cell.n.06"},{"index":7,"sense_key":"cell%1:06:01::","synset_id":"cell.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cell%1:03:00::","call_ids":["senseval2.d001.s068.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cell%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":855,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00183,"input_uncached_usd":0.00171,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0111039580078796}
{"item_id":"senseval2.d001.s068.t008","gold_sense_keys":["full-blown%5:00:00:complete:00"],"candidates":[{"index":1,"sense_key":"full-blown%5:00:00:mature:01","synset_id":"full-blown.s.01"},{"index":2,"sense_key":"full-blown%5:00:00:complete:00","synset_id":"full-blown.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"full-blown%5:00:00:complete:00","call_ids":["senseval2.d001.s068.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"full-blown%5:00:00:complete:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":570,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00126,"input_uncached_usd":0.00114,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5421167090535164}
{"item_id":"senseval2.d001.s068.t009","gold_sense_keys":["malignancy%1:26:00::"],"candidates":[{"index":1,"sense_key":"malignancy%1:26:00::","synset_id":"malignancy.n.01"},{"index":2,"sense_key":"malignancy%1:07:00::","synset_id":"malignity.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"malignancy%1:26:00::","call_ids":["senseval2.d001.s068.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"malignancy%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":565,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00125,"input_uncached_usd":0.00113,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5574032498989254}
{"item_id":"senseval2.d001.s070.t000","gold_sense_keys":["month%1:28:00::"],"candidates":[{"index":1,"sense_key":"month%1:28:01::","synset_id":"calendar_month.n.01"},{"index":2,"sense_key":"month%1:28:00::","synset_id":"month.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"month%1:28:00::","call_ids":["senseval2.d001.s070.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"month%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":580,"cached_input_tokens":0,"output_tokens":51,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00167,"input_uncached_usd":0.00116,"input_cached_usd":0.0,"output_usd":0.00051,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":4.33266758301761}
{"item_id":"senseval2.d001.s070.t002","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":2,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":3,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":4,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":5,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":6,"sense_key":"use%2:41:03::","synset_id":"use.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%2:34:01::","call_ids":["senseval2.d001.s070.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":861,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001842,"input_uncached_usd":0.001722,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4615401669871062}
{"item_id":"senseval2.d001.s070.t005","gold_sense_keys":["down%4:02:00::"],"candidates":[{"index":1,"sense_key":"down%4:02:00::","synset_id":"down.r.01"},{"index":2,"sense_key":"down%4:02:02::","synset_id":"down.r.02"},{"index":3,"sense_key":"down%4:02:03::","synset_id":"down.r.03"},{"index":4,"sense_key":"down%4:02:06::","synset_id":"down.r.04"},{"index":5,"sense_key":"down%4:02:05::","synset_id":"down.r.05"},{"index":6,"sense_key":"down%4:02:04::","synset_id":"down.r.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"down%4:02:00::","call_ids":["senseval2.d001.s070.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"down%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":798,"cached_input_tokens":0,"output_tokens":126,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002856,"input_uncached_usd":0.001596,"input_cached_usd":0.0,"output_usd":0.00126,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.932242916082032}
{"item_id":"senseval2.d001.s070.t006","gold_sense_keys":["length%1:07:00::"],"candidates":[{"index":1,"sense_key":"length%1:07:00::","synset_id":"length.n.01"},{"index":2,"sense_key":"length%1:07:01::","synset_id":"duration.n.03"},{"index":3,"sense_key":"length%1:07:02::","synset_id":"length.n.03"},{"index":4,"sense_key":"length%1:07:03::","synset_id":"distance.n.03"},{"index":5,"sense_key":"length%1:06:00::","synset_id":"length.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"length%1:07:00::","call_ids":["senseval2.d001.s070.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"length%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":771,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002312,"input_uncached_usd":0.001542,"input_cached_usd":0.0,"output_usd":0.0007700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.75703345797956}
{"item_id":"senseval2.d001.s070.t008","gold_sense_keys":["look_for%2:35:00::"],"candidates":[{"index":1,"sense_key":"look_for%2:35:00::","synset_id":"search.v.01"},{"index":2,"sense_key":"look_for%2:37:00::","synset_id":"anticipate.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"look_for%2:35:00::","call_ids":["senseval2.d001.s070.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"look_for%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":599,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001318,"input_uncached_usd":0.0011979999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0998846249422058}
{"item_id":"senseval2.d001.s070.t009","gold_sense_keys":["common%5:00:00:shared:00"],"candidates":[{"index":1,"sense_key":"common%3:00:02::","synset_id":"common.a.01"},{"index":2,"sense_key":"common%3:00:01::","synset_id":"common.a.02"},{"index":3,"sense_key":"common%5:00:00:shared:00","synset_id":"common.s.01"},{"index":4,"sense_key":"common%5:00:00:familiar:02","synset_id":"common.s.02"},{"index":5,"sense_key":"common%5:00:00:informal:02","synset_id":"common.s.03"},{"index":6,"sense_key":"common%5:00:00:lowborn:00","synset_id":"common.s.04"},{"index":7,"sense_key":"common%5:00:00:inferior:02","synset_id":"coarse.s.02"},{"index":8,"sense_key":"common%5:00:00:unrefined:01","synset_id":"coarse.s.01"},{"index":9,"sense_key":"common%5:00:00:ordinary:00","synset_id":"common.s.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"common%5:00:00:shared:00","call_ids":["senseval2.d001.s070.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"common%5:00:00:shared:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1104,"cached_input_tokens":0,"output_tokens":76,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002968,"input_uncached_usd":0.002208,"input_cached_usd":0.0,"output_usd":0.00076,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8126081250375137}
{"item_id":"senseval2.d001.s070.t010","gold_sense_keys":["bit%1:17:00::","bit%1:23:01::"],"candidates":[{"index":1,"sense_key":"bit%1:23:01::","synset_id":"spot.n.10"},{"index":2,"sense_key":"bit%1:17:00::","synset_id":"bit.n.02"},{"index":3,"sense_key":"bit%1:28:00::","synset_id":"moment.n.02"},{"index":4,"sense_key":"bit%1:11:00::","synset_id":"piece.n.05"},{"index":5,"sense_key":"bit%1:06:01::","synset_id":"bit.n.05"},{"index":6,"sense_key":"bit%1:23:00::","synset_id":"bit.n.06"},{"index":7,"sense_key":"bit%1:13:00::","synset_id":"morsel.n.02"},{"index":8,"sense_key":"bit%1:10:01::","synset_id":"snatch.n.01"},{"index":9,"sense_key":"bit%1:10:00::","synset_id":"act.n.04"},{"index":10,"sense_key":"bit%1:06:02::","synset_id":"bit.n.10"},{"index":11,"sense_key":"bit%1:06:00::","synset_id":"bit.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bit%1:23:01::","call_ids":["senseval2.d001.s070.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bit%1:23:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1167,"cached_input_tokens":0,"output_tokens":149,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003824,"input_uncached_usd":0.0023339999999999997,"input_cached_usd":0.0,"output_usd":0.0014900000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7890403750352561}
{"item_id":"senseval2.d001.s070.t011","gold_sense_keys":["genetic%3:01:02::"],"candidates":[{"index":1,"sense_key":"genetic%5:00:00:inheritable:00","synset_id":"familial.s.01"},{"index":2,"sense_key":"genetic%3:01:02::","synset_id":"genic.a.01"},{"index":3,"sense_key":"genetic%3:01:01::","synset_id":"genetic.a.03"},{"index":4,"sense_key":"genetic%3:01:00::","synset_id":"genetic.a.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"genetic%3:01:02::","call_ids":["senseval2.d001.s070.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"genetic%3:01:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":732,"cached_input_tokens":0,"output_tokens":136,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002824,"input_uncached_usd":0.001464,"input_cached_usd":0.0,"output_usd":0.00136,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0640204169321805}
{"item_id":"senseval2.d001.s070.t012","gold_sense_keys":["material%1:27:00::"],"candidates":[{"index":1,"sense_key":"material%1:27:00::","synset_id":"material.n.01"},{"index":2,"sense_key":"material%1:10:00::","synset_id":"material.n.02"},{"index":3,"sense_key":"material%1:06:00::","synset_id":"fabric.n.01"},{"index":4,"sense_key":"material%1:06:01::","synset_id":"material.n.04"},{"index":5,"sense_key":"material%1:18:00::","synset_id":"material.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"material%1:27:00::","call_ids":["senseval2.d001.s070.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"material%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":823,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002236,"input_uncached_usd":0.001646,"input_cached_usd":0.0,"output_usd":0.00059,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3050246250350028}
{"item_id":"senseval2.d001.s070.t013","gold_sense_keys":["lose%2:40:02::"],"candidates":[{"index":1,"sense_key":"lose%2:40:02::","synset_id":"lose.v.01"},{"index":2,"sense_key":"lose%2:33:00::","synset_id":"lose.v.02"},{"index":3,"sense_key":"lose%2:37:00::","synset_id":"lose.v.03"},{"index":4,"sense_key":"lose%2:35:00::","synset_id":"misplace.v.01"},{"index":5,"sense_key":"lose%2:40:01::","synset_id":"lose.v.05"},{"index":6,"sense_key":"lose%2:39:00::","synset_id":"lose.v.06"},{"index":7,"sense_key":"lose%2:40:06::","synset_id":"lose.v.07"},{"index":8,"sense_key":"lose%2:40:00::","synset_id":"lose.v.08"},{"index":9,"sense_key":"lose%2:33:01::","synset_id":"fall_back.v.04"},{"index":10,"sense_key":"lose%2:39:01::","synset_id":"miss.v.01"},{"index":11,"sense_key":"lose%2:30:03::","synset_id":"suffer.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"lose%2:40:02::","call_ids":["senseval2.d001.s070.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"lose%2:40:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1131,"cached_input_tokens":0,"output_tokens":136,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003622,"input_uncached_usd":0.0022619999999999997,"input_cached_usd":0.0,"output_usd":0.00136,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9909028339898214}
{"item_id":"senseval2.d001.s070.t015","gold_sense_keys":["cell%1:03:00::"],"candidates":[{"index":1,"sense_key":"cell%1:06:03::","synset_id":"cell.n.01"},{"index":2,"sense_key":"cell%1:03:00::","synset_id":"cell.n.02"},{"index":3,"sense_key":"cell%1:06:00::","synset_id":"cell.n.03"},{"index":4,"sense_key":"cell%1:14:00::","synset_id":"cell.n.04"},{"index":5,"sense_key":"cell%1:06:04::","synset_id":"cellular_telephone.n.01"},{"index":6,"sense_key":"cell%1:06:02::","synset_id":"cell.n.06"},{"index":7,"sense_key":"cell%1:06:01::","synset_id":"cell.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cell%1:03:00::","call_ids":["senseval2.d001.s070.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cell%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":885,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00189,"input_uncached_usd":0.0017699999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4271482080221176}
{"item_id":"senseval2.d001.s071.t000","gold_sense_keys":["piece%1:06:00::","piece%1:17:00::"],"candidates":[{"index":1,"sense_key":"piece%1:06:00::","synset_id":"piece.n.01"},{"index":2,"sense_key":"piece%1:06:05::","synset_id":"piece.n.02"},{"index":3,"sense_key":"piece%1:17:00::","synset_id":"part.n.03"},{"index":4,"sense_key":"piece%1:10:01::","synset_id":"musical_composition.n.01"},{"index":5,"sense_key":"piece%1:11:00::","synset_id":"piece.n.05"},{"index":6,"sense_key":"piece%1:10:02::","synset_id":"piece.n.06"},{"index":7,"sense_key":"piece%1:06:03::","synset_id":"firearm.n.01"},{"index":8,"sense_key":"piece%1:13:00::","synset_id":"piece.n.08"},{"index":9,"sense_key":"piece%1:07:00::","synset_id":"piece.n.09"},{"index":10,"sense_key":"piece%1:06:04::","synset_id":"objet_d'art.n.01"},{"index":11,"sense_key":"piece%1:28:00::","synset_id":"while.n.01"},{"index":12,"sense_key":"piece%1:21:00::","synset_id":"slice.n.01"},{"index":13,"sense_key":"piece%1:06:02::","synset_id":"man.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"piece%1:17:00::","call_ids":["senseval2.d001.s071.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"piece%1:17:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1271,"cached_input_tokens":0,"output_tokens":127,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0038120000000000003,"input_uncached_usd":0.002542,"input_cached_usd":0.0,"output_usd":0.00127,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8444230000022799}
{"item_id":"senseval2.d001.s071.t002","gold_sense_keys":["probably%4:02:00::"],"candidates":[{"index":1,"sense_key":"probably%4:02:00::","synset_id":"probably.r.01"},{"index":2,"sense_key":"probably%4:02:01::","synset_id":"credibly.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"probably%4:02:00::","call_ids":["senseval2.d001.s071.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"probably%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":637,"cached_input_tokens":0,"output_tokens":96,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002234,"input_uncached_usd":0.001274,"input_cached_usd":0.0,"output_usd":0.0009600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5632568340515718}
{"item_id":"senseval2.d001.s071.t003","gold_sense_keys":["constitute%2:42:00::"],"candidates":[{"index":1,"sense_key":"constitute%2:42:00::","synset_id":"constitute.v.01"},{"index":2,"sense_key":"constitute%2:41:00::","synset_id":"appoint.v.01"},{"index":3,"sense_key":"constitute%2:42:03::","synset_id":"form.v.02"},{"index":4,"sense_key":"constitute%2:36:00::","synset_id":"establish.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"constitute%2:42:00::","call_ids":["senseval2.d001.s071.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"constitute%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":730,"cached_input_tokens":0,"output_tokens":154,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003,"input_uncached_usd":0.00146,"input_cached_usd":0.0,"output_usd":0.0015400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.4628744169604033}
{"item_id":"senseval2.d001.s072.t000","gold_sense_keys":["find%2:39:02::"],"candidates":[{"index":1,"sense_key":"find%2:40:02::","synset_id":"find.v.01"},{"index":2,"sense_key":"find%2:39:02::","synset_id":"detect.v.01"},{"index":3,"sense_key":"find%2:40:00::","synset_id":"find.v.03"},{"index":4,"sense_key":"find%2:32:00::","synset_id":"determine.v.01"},{"index":5,"sense_key":"find%2:31:10::","synset_id":"find.v.05"},{"index":6,"sense_key":"find%2:39:01::","synset_id":"witness.v.02"},{"index":7,"sense_key":"find%2:40:01::","synset_id":"line_up.v.02"},{"index":8,"sense_key":"find%2:36:00::","synset_id":"discover.v.03"},{"index":9,"sense_key":"find%2:31:09::","synset_id":"discover.v.04"},{"index":10,"sense_key":"find%2:40:03::","synset_id":"find.v.10"},{"index":11,"sense_key":"find%2:32:01::","synset_id":"rule.v.04"},{"index":12,"sense_key":"find%2:30:00::","synset_id":"receive.v.02"},{"index":13,"sense_key":"find%2:39:05::","synset_id":"find.v.13"},{"index":14,"sense_key":"find%2:40:15::","synset_id":"recover.v.01"},{"index":15,"sense_key":"find%2:38:10::","synset_id":"find.v.15"},{"index":16,"sense_key":"find%2:30:13::","synset_id":"find_oneself.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"find%2:36:00::","call_ids":["senseval2.d001.s072.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"find%2:36:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1680,"cached_input_tokens":0,"output_tokens":290,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00626,"input_uncached_usd":0.0033599999999999997,"input_cached_usd":0.0,"output_usd":0.0029000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.852393375011161}
{"item_id":"senseval2.d001.s072.t001","gold_sense_keys":["last%5:00:00:past:00"],"candidates":[{"index":1,"sense_key":"last%5:00:00:past:00","synset_id":"last.s.01"},{"index":2,"sense_key":"last%3:00:00::","synset_id":"last.a.02"},{"index":3,"sense_key":"last%5:00:00:closing:00","synset_id":"concluding.s.01"},{"index":4,"sense_key":"last%5:00:00:unlikely:00","synset_id":"last.s.03"},{"index":5,"sense_key":"last%5:00:00:dying:00","synset_id":"last.s.04"},{"index":6,"sense_key":"last%5:00:00:ultimate:00","synset_id":"final.s.02"},{"index":7,"sense_key":"last%5:00:00:high:02","synset_id":"last.s.06"},{"index":8,"sense_key":"last%5:00:00:unalterable:00","synset_id":"final.s.03"},{"index":9,"sense_key":"last%5:00:00:worst:00","synset_id":"last.s.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"last%5:00:00:past:00","call_ids":["senseval2.d001.s072.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"last%5:00:00:past:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1082,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002724,"input_uncached_usd":0.0021639999999999997,"input_cached_usd":0.0,"output_usd":0.0005600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8446318749338388}
{"item_id":"senseval2.d001.s072.t003","gold_sense_keys":["dubious%5:00:00:uncertain:02","dubious%5:00:00:unconvinced:00"],"candidates":[{"index":1,"sense_key":"dubious%5:00:00:uncertain:02","synset_id":"doubtful.s.02"},{"index":2,"sense_key":"dubious%5:00:00:questionable:00","synset_id":"doubtful.s.01"},{"index":3,"sense_key":"dubious%5:00:00:unconvinced:00","synset_id":"dubious.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"dubious%5:00:00:unconvinced:00","call_ids":["senseval2.d001.s072.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"dubious%5:00:00:unconvinced:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":766,"cached_input_tokens":0,"output_tokens":148,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003012,"input_uncached_usd":0.001532,"input_cached_usd":0.0,"output_usd":0.0014800000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8326717499876395}
{"item_id":"senseval2.d001.s072.t004","gold_sense_keys":["search%1:04:00::","search%1:04:02::"],"candidates":[{"index":1,"sense_key":"search%1:04:00::","synset_id":"search.n.01"},{"index":2,"sense_key":"search%1:04:02::","synset_id":"search.n.02"},{"index":3,"sense_key":"search%1:22:00::","synset_id":"search.n.03"},{"index":4,"sense_key":"search%1:09:00::","synset_id":"search.n.04"},{"index":5,"sense_key":"search%1:04:01::","synset_id":"search.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"search%1:04:00::","call_ids":["senseval2.d001.s072.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"search%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":812,"cached_input_tokens":0,"output_tokens":144,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003064,"input_uncached_usd":0.001624,"input_cached_usd":0.0,"output_usd":0.00144,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1140804160386324}
{"item_id":"senseval2.d001.s073.t000","gold_sense_keys":["doubt%1:07:00::"],"candidates":[{"index":1,"sense_key":"doubt%1:09:00::","synset_id":"doubt.n.01"},{"index":2,"sense_key":"doubt%1:07:00::","synset_id":"doubt.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"doubt%1:07:00::","call_ids":["senseval2.d001.s073.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"doubt%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":656,"cached_input_tokens":0,"output_tokens":142,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002732,"input_uncached_usd":0.001312,"input_cached_usd":0.0,"output_usd":0.00142,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6195968340616673}
{"item_id":"senseval2.d001.s073.t001","gold_sense_keys":["stem%2:42:00::"],"candidates":[{"index":1,"sense_key":"stem%2:42:00::","synset_id":"stem.v.01"},{"index":2,"sense_key":"stem%2:42:03::","synset_id":"stem.v.02"},{"index":3,"sense_key":"stem%2:30:00::","synset_id":"stem.v.03"},{"index":4,"sense_key":"stem%2:30:01::","synset_id":"stem.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"stem%2:42:00::","call_ids":["senseval2.d001.s073.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"stem%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":706,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001532,"input_uncached_usd":0.0014119999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1257319999858737}
{"item_id":"senseval2.d001.s073.t002","gold_sense_keys":["fact%1:09:01::"],"candidates":[{"index":1,"sense_key":"fact%1:09:01::","synset_id":"fact.n.01"},{"index":2,"sense_key":"fact%1:10:01::","synset_id":"fact.n.02"},{"index":3,"sense_key":"fact%1:26:01::","synset_id":"fact.n.03"},{"index":4,"sense_key":"fact%1:09:02::","synset_id":"fact.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"fact%1:09:01::","call_ids":["senseval2.d001.s073.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"fact%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":738,"cached_input_tokens":0,"output_tokens":129,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0027660000000000002,"input_uncached_usd":0.0014759999999999999,"input_cached_usd":0.0,"output_usd":0.0012900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1193353340495378}
{"item_id":"senseval2.d001.s073.t003","gold_sense_keys":["several%5:00:00:some:00"],"candidates":[{"index":1,"sense_key":"several%5:00:00:some:00","synset_id":"several.s.01"},{"index":2,"sense_key":"several%5:00:00:individual:00","synset_id":"respective.s.01"},{"index":3,"sense_key":"several%5:00:00:different:00","synset_id":"several.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"several%5:00:00:some:00","call_ids":["senseval2.d001.s073.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"several%5:00:00:some:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":663,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001446,"input_uncached_usd":0.001326,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0507950000464916}
{"item_id":"senseval2.d001.s073.t004","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["senseval2.d001.s073.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":756,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001632,"input_uncached_usd":0.0015119999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0499091249657795}
{"item_id":"senseval2.d001.s073.t005","gold_sense_keys":["earlier%4:02:00::"],"candidates":[{"index":1,"sense_key":"earlier%4:02:00::","synset_id":"earlier.r.01"},{"index":2,"sense_key":"earlier%4:02:02::","synset_id":"sooner.r.01"},{"index":3,"sense_key":"earlier%4:02:04::","synset_id":"in_the_first_place.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"earlier%4:02:00::","call_ids":["senseval2.d001.s073.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"earlier%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":687,"cached_input_tokens":0,"output_tokens":144,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002814,"input_uncached_usd":0.001374,"input_cached_usd":0.0,"output_usd":0.00144,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.3366933331126347}
{"item_id":"senseval2.d001.s073.t007","gold_sense_keys":["find%2:31:09::","find%2:32:00::","find%2:36:00::","find%2:39:02::"],"candidates":[{"index":1,"sense_key":"find%2:40:02::","synset_id":"find.v.01"},{"index":2,"sense_key":"find%2:39:02::","synset_id":"detect.v.01"},{"index":3,"sense_key":"find%2:40:00::","synset_id":"find.v.03"},{"index":4,"sense_key":"find%2:32:00::","synset_id":"determine.v.01"},{"index":5,"sense_key":"find%2:31:10::","synset_id":"find.v.05"},{"index":6,"sense_key":"find%2:39:01::","synset_id":"witness.v.02"},{"index":7,"sense_key":"find%2:40:01::","synset_id":"line_up.v.02"},{"index":8,"sense_key":"find%2:36:00::","synset_id":"discover.v.03"},{"index":9,"sense_key":"find%2:31:09::","synset_id":"discover.v.04"},{"index":10,"sense_key":"find%2:40:03::","synset_id":"find.v.10"},{"index":11,"sense_key":"find%2:32:01::","synset_id":"rule.v.04"},{"index":12,"sense_key":"find%2:30:00::","synset_id":"receive.v.02"},{"index":13,"sense_key":"find%2:39:05::","synset_id":"find.v.13"},{"index":14,"sense_key":"find%2:40:15::","synset_id":"recover.v.01"},{"index":15,"sense_key":"find%2:38:10::","synset_id":"find.v.15"},{"index":16,"sense_key":"find%2:30:13::","synset_id":"find_oneself.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"find%2:31:09::","call_ids":["senseval2.d001.s073.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"find%2:31:09::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1647,"cached_input_tokens":0,"output_tokens":257,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.005864,"input_uncached_usd":0.003294,"input_cached_usd":0.0,"output_usd":0.0025700000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.187929709092714}
{"item_id":"senseval2.d001.s073.t008","gold_sense_keys":["experiment%1:04:00::"],"candidates":[{"index":1,"sense_key":"experiment%1:04:00::","synset_id":"experiment.n.01"},{"index":2,"sense_key":"experiment%1:09:00::","synset_id":"experiment.n.02"},{"index":3,"sense_key":"experiment%1:04:01::","synset_id":"experiment.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"experiment%1:04:00::","call_ids":["senseval2.d001.s073.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"experiment%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":663,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001446,"input_uncached_usd":0.001326,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.007373416912742}
{"item_id":"senseval2.d001.s073.t009","gold_sense_keys":["mouse%1:05:00::"],"candidates":[{"index":1,"sense_key":"mouse%1:05:00::","synset_id":"mouse.n.01"},{"index":2,"sense_key":"mouse%1:26:00::","synset_id":"shiner.n.01"},{"index":3,"sense_key":"mouse%1:18:00::","synset_id":"mouse.n.03"},{"index":4,"sense_key":"mouse%1:06:00::","synset_id":"mouse.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"mouse%1:05:00::","call_ids":["senseval2.d001.s073.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"mouse%1:05:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":770,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00166,"input_uncached_usd":0.00154,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4992425829404965}
{"item_id":"senseval2.d001.s073.t011","gold_sense_keys":["call%2:32:02::"],"candidates":[{"index":1,"sense_key":"call%2:32:02::","synset_id":"name.v.01"},{"index":2,"sense_key":"call%2:32:00::","synset_id":"call.v.02"},{"index":3,"sense_key":"call%2:32:01::","synset_id":"call.v.03"},{"index":4,"sense_key":"call%2:32:09::","synset_id":"shout.v.02"},{"index":5,"sense_key":"call%2:32:05::","synset_id":"call.v.05"},{"index":6,"sense_key":"call%2:41:00::","synset_id":"visit.v.03"},{"index":7,"sense_key":"call%2:41:04::","synset_id":"call.v.07"},{"index":8,"sense_key":"call%2:32:13::","synset_id":"call.v.08"},{"index":9,"sense_key":"call%2:32:04::","synset_id":"call.v.09"},{"index":10,"sense_key":"call%2:32:03::","synset_id":"call.v.10"},{"index":11,"sense_key":"call%2:42:09::","synset_id":"call.v.11"},{"index":12,"sense_key":"call%2:41:14::","synset_id":"address.v.06"},{"index":13,"sense_key":"call%2:41:12::","synset_id":"call.v.13"},{"index":14,"sense_key":"call%2:40:09::","synset_id":"call.v.14"},{"index":15,"sense_key":"call%2:40:05::","synset_id":"bid.v.04"},{"index":16,"sense_key":"call%2:38:00::","synset_id":"call.v.16"},{"index":17,"sense_key":"call%2:32:06::","synset_id":"call.v.17"},{"index":18,"sense_key":"call%2:32:10::","synset_id":"predict.v.01"},{"index":19,"sense_key":"call%2:32:12::","synset_id":"call.v.19"},{"index":20,"sense_key":"call%2:32:07::","synset_id":"call.v.20"},{"index":21,"sense_key":"call%2:32:08::","synset_id":"call.v.21"},{"index":22,"sense_key":"call%2:32:11::","synset_id":"call.v.22"},{"index":23,"sense_key":"call%2:32:15::","synset_id":"call.v.23"},{"index":24,"sense_key":"call%2:32:14::","synset_id":"call.v.24"},{"index":25,"sense_key":"call%2:31:13::","synset_id":"call.v.25"},{"index":26,"sense_key":"call%2:31:00::","synset_id":"call.v.26"},{"index":27,"sense_key":"call%2:31:05::","synset_id":"call.v.27"},{"index":28,"sense_key":"call%2:29:05::","synset_id":"call.v.28"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"call%2:32:02::","call_ids":["senseval2.d001.s073.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"call%2:32:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2119,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004358,"input_uncached_usd":0.004238,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9840590419480577}
{"item_id":"senseval2.d001.s073.t012","gold_sense_keys":["transform%2:30:00::"],"candidates":[{"index":1,"sense_key":"transform%2:30:02::","synset_id":"transform.v.01"},{"index":2,"sense_key":"transform%2:30:00::","synset_id":"transform.v.02"},{"index":3,"sense_key":"transform%2:30:03::","synset_id":"transform.v.03"},{"index":4,"sense_key":"transform%2:30:01::","synset_id":"translate.v.02"},{"index":5,"sense_key":"transform%2:30:06::","synset_id":"transform.v.05"},{"index":6,"sense_key":"transform%2:30:05::","synset_id":"transform.v.06"},{"index":7,"sense_key":"transform%2:30:04::","synset_id":"transform.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"transform%2:30:00::","call_ids":["senseval2.d001.s073.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"transform%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":889,"cached_input_tokens":0,"output_tokens":151,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003288,"input_uncached_usd":0.0017779999999999998,"input_cached_usd":0.0,"output_usd":0.00151,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8392130000283942}
{"item_id":"senseval2.d001.s073.t013","gold_sense_keys":["normal%3:00:01::"],"candidates":[{"index":1,"sense_key":"normal%3:00:01::","synset_id":"normal.a.01"},{"index":2,"sense_key":"normal%3:00:02::","synset_id":"normal.a.02"},{"index":3,"sense_key":"normal%3:00:03::","synset_id":"normal.a.03"},{"index":4,"sense_key":"normal%5:00:00:perpendicular:00","synset_id":"normal.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"normal%3:00:01::","call_ids":["senseval2.d001.s073.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"normal%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":704,"cached_input_tokens":0,"output_tokens":131,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002718,"input_uncached_usd":0.001408,"input_cached_usd":0.0,"output_usd":0.0013100000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7823222080478445}
{"item_id":"senseval2.d001.s073.t014","gold_sense_keys":["cell%1:03:00::"],"candidates":[{"index":1,"sense_key":"cell%1:06:03::","synset_id":"cell.n.01"},{"index":2,"sense_key":"cell%1:03:00::","synset_id":"cell.n.02"},{"index":3,"sense_key":"cell%1:06:00::","synset_id":"cell.n.03"},{"index":4,"sense_key":"cell%1:14:00::","synset_id":"cell.n.04"},{"index":5,"sense_key":"cell%1:06:04::","synset_id":"cellular_telephone.n.01"},{"index":6,"sense_key":"cell%1:06:02::","synset_id":"cell.n.06"},{"index":7,"sense_key":"cell%1:06:01::","synset_id":"cell.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cell%1:03:00::","call_ids":["senseval2.d001.s073.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cell%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":907,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001934,"input_uncached_usd":0.0018139999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6237651670817286}
{"item_id":"senseval2.d001.s073.t015","gold_sense_keys":["cancerous%5:00:00:malignant:02"],"candidates":[{"index":1,"sense_key":"cancerous%5:00:00:malignant:02","synset_id":"cancerous.s.01"},{"index":2,"sense_key":"cancerous%5:00:00:malign:00","synset_id":"cancerous.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancerous%5:00:00:malignant:02","call_ids":["senseval2.d001.s073.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancerous%5:00:00:malignant:02","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":627,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001374,"input_uncached_usd":0.001254,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5208867919864133}
{"item_id":"senseval2.d001.s073.t016","gold_sense_keys":["one%1:09:00::"],"candidates":[{"index":1,"sense_key":"one%1:23:00::","synset_id":"one.n.01"},{"index":2,"sense_key":"one%1:09:00::","synset_id":"one.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"one%1:09:00::","call_ids":["senseval2.d001.s073.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"one%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":623,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001366,"input_uncached_usd":0.001246,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.192267292062752}
{"item_id":"senseval2.d001.s074.t000","gold_sense_keys":["find%2:31:09::","find%2:32:00::","find%2:36:00::","find%2:39:02::"],"candidates":[{"index":1,"sense_key":"find%2:40:02::","synset_id":"find.v.01"},{"index":2,"sense_key":"find%2:39:02::","synset_id":"detect.v.01"},{"index":3,"sense_key":"find%2:40:00::","synset_id":"find.v.03"},{"index":4,"sense_key":"find%2:32:00::","synset_id":"determine.v.01"},{"index":5,"sense_key":"find%2:31:10::","synset_id":"find.v.05"},{"index":6,"sense_key":"find%2:39:01::","synset_id":"witness.v.02"},{"index":7,"sense_key":"find%2:40:01::","synset_id":"line_up.v.02"},{"index":8,"sense_key":"find%2:36:00::","synset_id":"discover.v.03"},{"index":9,"sense_key":"find%2:31:09::","synset_id":"discover.v.04"},{"index":10,"sense_key":"find%2:40:03::","synset_id":"find.v.10"},{"index":11,"sense_key":"find%2:32:01::","synset_id":"rule.v.04"},{"index":12,"sense_key":"find%2:30:00::","synset_id":"receive.v.02"},{"index":13,"sense_key":"find%2:39:05::","synset_id":"find.v.13"},{"index":14,"sense_key":"find%2:40:15::","synset_id":"recover.v.01"},{"index":15,"sense_key":"find%2:38:10::","synset_id":"find.v.15"},{"index":16,"sense_key":"find%2:30:13::","synset_id":"find_oneself.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"find%2:39:02::","call_ids":["senseval2.d001.s074.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"find%2:39:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1656,"cached_input_tokens":0,"output_tokens":251,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.005822,"input_uncached_usd":0.003312,"input_cached_usd":0.0,"output_usd":0.00251,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.432523165945895}
{"item_id":"senseval2.d001.s074.t001","gold_sense_keys":["exactly%4:02:01::"],"candidates":[{"index":1,"sense_key":"exactly%4:02:01::","synset_id":"precisely.r.01"},{"index":2,"sense_key":"exactly%4:02:04::","synset_id":"precisely.r.03"},{"index":3,"sense_key":"exactly%4:02:02::","synset_id":"precisely.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"exactly%4:02:01::","call_ids":["senseval2.d001.s074.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"exactly%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":717,"cached_input_tokens":0,"output_tokens":139,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002824,"input_uncached_usd":0.001434,"input_cached_usd":0.0,"output_usd":0.0013900000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.006358041893691}
{"item_id":"senseval2.d001.s074.t002","gold_sense_keys":["same%3:00:00::"],"candidates":[{"index":1,"sense_key":"same%3:00:02::","synset_id":"same.a.01"},{"index":2,"sense_key":"same%3:00:00::","synset_id":"same.a.02"},{"index":3,"sense_key":"same%3:00:04::","synset_id":"like.a.02"},{"index":4,"sense_key":"same%5:00:00:unchanged:00","synset_id":"same.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"same%3:00:02::","call_ids":["senseval2.d001.s074.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"same%3:00:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":727,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002124,"input_uncached_usd":0.001454,"input_cached_usd":0.0,"output_usd":0.00067,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.541427250020206}
{"item_id":"senseval2.d001.s074.t003","gold_sense_keys":["spot%1:15:01::"],"candidates":[{"index":1,"sense_key":"spot%1:15:01::","synset_id":"topographic_point.n.01"},{"index":2,"sense_key":"spot%1:10:01::","synset_id":"spot.n.02"},{"index":3,"sense_key":"spot%1:07:02::","synset_id":"point.n.14"},{"index":4,"sense_key":"spot%1:07:00::","synset_id":"smudge.n.02"},{"index":5,"sense_key":"spot%1:07:01::","synset_id":"spot.n.05"},{"index":6,"sense_key":"spot%1:10:00::","synset_id":"spot.n.06"},{"index":7,"sense_key":"spot%1:06:00::","synset_id":"spot.n.07"},{"index":8,"sense_key":"spot%1:04:01::","synset_id":"position.n.06"},{"index":9,"sense_key":"spot%1:26:00::","synset_id":"touch.n.08"},{"index":10,"sense_key":"spot%1:23:00::","synset_id":"spot.n.10"},{"index":11,"sense_key":"spot%1:10:02::","synset_id":"spot.n.11"},{"index":12,"sense_key":"spot%1:06:01::","synset_id":"spotlight.n.02"},{"index":13,"sense_key":"spot%1:06:02::","synset_id":"spot.n.13"},{"index":14,"sense_key":"spot%1:04:00::","synset_id":"blot.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"spot%1:15:01::","call_ids":["senseval2.d001.s074.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"spot%1:15:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1342,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0028039999999999996,"input_uncached_usd":0.0026839999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0319898340385407}
{"item_id":"senseval2.d001.s075.t000","gold_sense_keys":["say%2:32:01::"],"candidates":[{"index":1,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":6,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":9,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":10,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":11,"sense_key":"say%2:32:09::","synset_id":"say.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"say%2:32:01::","call_ids":["senseval2.d001.s075.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"say%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1263,"cached_input_tokens":0,"output_tokens":115,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0036760000000000004,"input_uncached_usd":0.002526,"input_cached_usd":0.0,"output_usd":0.0011500000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.3451779159950092}
{"item_id":"senseval2.d001.s075.t002","gold_sense_keys":["cause%2:36:00::"],"candidates":[{"index":1,"sense_key":"cause%2:36:00::","synset_id":"cause.v.01"},{"index":2,"sense_key":"cause%2:32:00::","synset_id":"induce.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cause%2:36:00::","call_ids":["senseval2.d001.s075.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cause%2:36:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":758,"cached_input_tokens":0,"output_tokens":96,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002476,"input_uncached_usd":0.001516,"input_cached_usd":0.0,"output_usd":0.0009600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.744800915941596}
{"item_id":"senseval2.d001.s075.t003","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s075.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":923,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001966,"input_uncached_usd":0.001846,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0409789170371369}
{"item_id":"senseval2.d001.s075.t004","gold_sense_keys":["promote%2:41:01::"],"candidates":[{"index":1,"sense_key":"promote%2:41:01::","synset_id":"promote.v.01"},{"index":2,"sense_key":"promote%2:41:00::","synset_id":"promote.v.02"},{"index":3,"sense_key":"promote%2:32:00::","synset_id":"advertise.v.02"},{"index":4,"sense_key":"promote%2:33:01::","synset_id":"promote.v.04"},{"index":5,"sense_key":"promote%2:33:00::","synset_id":"promote.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"promote%2:41:01::","call_ids":["senseval2.d001.s075.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"promote%2:41:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":972,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002064,"input_uncached_usd":0.001944,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6723469579592347}
{"item_id":"senseval2.d001.s075.t005","gold_sense_keys":["growth%1:22:02::"],"candidates":[{"index":1,"sense_key":"growth%1:22:00::","synset_id":"growth.n.01"},{"index":2,"sense_key":"growth%1:22:01::","synset_id":"growth.n.02"},{"index":3,"sense_key":"growth%1:22:02::","synset_id":"increase.n.03"},{"index":4,"sense_key":"growth%1:14:00::","synset_id":"growth.n.04"},{"index":5,"sense_key":"growth%1:11:01::","synset_id":"emergence.n.01"},{"index":6,"sense_key":"growth%1:26:00::","synset_id":"growth.n.06"},{"index":7,"sense_key":"growth%1:17:00::","synset_id":"growth.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"growth%1:22:02::","call_ids":["senseval2.d001.s075.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"growth%1:22:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1046,"cached_input_tokens":0,"output_tokens":163,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0037220000000000005,"input_uncached_usd":0.002092,"input_cached_usd":0.0,"output_usd":0.0016300000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.706409583101049}
{"item_id":"senseval2.d001.s075.t007","gold_sense_keys":["look_for%2:35:00::"],"candidates":[{"index":1,"sense_key":"look_for%2:35:00::","synset_id":"search.v.01"},{"index":2,"sense_key":"look_for%2:37:00::","synset_id":"anticipate.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"look_for%2:35:00::","call_ids":["senseval2.d001.s075.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"look_for%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":729,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001578,"input_uncached_usd":0.0014579999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.15829558391124}
{"item_id":"senseval2.d001.s075.t009","gold_sense_keys":["suppress%2:30:12::"],"candidates":[{"index":1,"sense_key":"suppress%2:30:00::","synset_id":"suppress.v.01"},{"index":2,"sense_key":"suppress%2:41:01::","synset_id":"oppress.v.01"},{"index":3,"sense_key":"suppress%2:41:00::","synset_id":"inhibit.v.04"},{"index":4,"sense_key":"suppress%2:31:00::","synset_id":"suppress.v.04"},{"index":5,"sense_key":"suppress%2:30:12::","synset_id":"suppress.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"suppress%2:30:12::","call_ids":["senseval2.d001.s075.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"suppress%2:30:12::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":919,"cached_input_tokens":0,"output_tokens":103,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002868,"input_uncached_usd":0.001838,"input_cached_usd":0.0,"output_usd":0.00103,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.783954167040065}
{"item_id":"senseval2.d001.s075.t010","gold_sense_keys":["growth%1:22:02::"],"candidates":[{"index":1,"sense_key":"growth%1:22:00::","synset_id":"growth.n.01"},{"index":2,"sense_key":"growth%1:22:01::","synset_id":"growth.n.02"},{"index":3,"sense_key":"growth%1:22:02::","synset_id":"increase.n.03"},{"index":4,"sense_key":"growth%1:14:00::","synset_id":"growth.n.04"},{"index":5,"sense_key":"growth%1:11:01::","synset_id":"emergence.n.01"},{"index":6,"sense_key":"growth%1:26:00::","synset_id":"growth.n.06"},{"index":7,"sense_key":"growth%1:17:00::","synset_id":"growth.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"growth%1:22:02::","call_ids":["senseval2.d001.s075.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"growth%1:22:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1046,"cached_input_tokens":0,"output_tokens":154,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0036320000000000002,"input_uncached_usd":0.002092,"input_cached_usd":0.0,"output_usd":0.0015400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.728251292021014}
{"item_id":"senseval2.d001.s076.t001","gold_sense_keys":["compare%2:31:00::"],"candidates":[{"index":1,"sense_key":"compare%2:31:00::","synset_id":"compare.v.01"},{"index":2,"sense_key":"compare%2:42:00::","synset_id":"compare.v.02"},{"index":3,"sense_key":"compare%2:31:01::","synset_id":"compare.v.03"},{"index":4,"sense_key":"compare%2:32:00::","synset_id":"compare.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"compare%2:31:00::","call_ids":["senseval2.d001.s076.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"compare%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":852,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001824,"input_uncached_usd":0.001704,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5215692080091685}
{"item_id":"senseval2.d001.s076.t003","gold_sense_keys":["find%2:31:09::","find%2:32:00::","find%2:36:00::","find%2:39:02::"],"candidates":[{"index":1,"sense_key":"find%2:40:02::","synset_id":"find.v.01"},{"index":2,"sense_key":"find%2:39:02::","synset_id":"detect.v.01"},{"index":3,"sense_key":"find%2:40:00::","synset_id":"find.v.03"},{"index":4,"sense_key":"find%2:32:00::","synset_id":"determine.v.01"},{"index":5,"sense_key":"find%2:31:10::","synset_id":"find.v.05"},{"index":6,"sense_key":"find%2:39:01::","synset_id":"witness.v.02"},{"index":7,"sense_key":"find%2:40:01::","synset_id":"line_up.v.02"},{"index":8,"sense_key":"find%2:36:00::","synset_id":"discover.v.03"},{"index":9,"sense_key":"find%2:31:09::","synset_id":"discover.v.04"},{"index":10,"sense_key":"find%2:40:03::","synset_id":"find.v.10"},{"index":11,"sense_key":"find%2:32:01::","synset_id":"rule.v.04"},{"index":12,"sense_key":"find%2:30:00::","synset_id":"receive.v.02"},{"index":13,"sense_key":"find%2:39:05::","synset_id":"find.v.13"},{"index":14,"sense_key":"find%2:40:15::","synset_id":"recover.v.01"},{"index":15,"sense_key":"find%2:38:10::","synset_id":"find.v.15"},{"index":16,"sense_key":"find%2:30:13::","synset_id":"find_oneself.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"find%2:36:00::","call_ids":["senseval2.d001.s076.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"find%2:36:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1764,"cached_input_tokens":0,"output_tokens":250,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.006028,"input_uncached_usd":0.003528,"input_cached_usd":0.0,"output_usd":0.0025,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.3339915830874816}
{"item_id":"senseval2.d001.s076.t004","gold_sense_keys":["human%3:01:01::"],"candidates":[{"index":1,"sense_key":"human%3:01:00::","synset_id":"human.a.01"},{"index":2,"sense_key":"human%3:01:01::","synset_id":"human.a.02"},{"index":3,"sense_key":"human%3:00:00::","synset_id":"human.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"human%3:01:01::","call_ids":["senseval2.d001.s076.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"human%3:01:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":756,"cached_input_tokens":0,"output_tokens":74,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002252,"input_uncached_usd":0.0015119999999999999,"input_cached_usd":0.0,"output_usd":0.0007400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9942567920079455}
{"item_id":"senseval2.d001.s076.t008","gold_sense_keys":["find%2:31:09::","find%2:32:00::","find%2:36:00::","find%2:39:02::"],"candidates":[{"index":1,"sense_key":"find%2:40:02::","synset_id":"find.v.01"},{"index":2,"sense_key":"find%2:39:02::","synset_id":"detect.v.01"},{"index":3,"sense_key":"find%2:40:00::","synset_id":"find.v.03"},{"index":4,"sense_key":"find%2:32:00::","synset_id":"determine.v.01"},{"index":5,"sense_key":"find%2:31:10::","synset_id":"find.v.05"},{"index":6,"sense_key":"find%2:39:01::","synset_id":"witness.v.02"},{"index":7,"sense_key":"find%2:40:01::","synset_id":"line_up.v.02"},{"index":8,"sense_key":"find%2:36:00::","synset_id":"discover.v.03"},{"index":9,"sense_key":"find%2:31:09::","synset_id":"discover.v.04"},{"index":10,"sense_key":"find%2:40:03::","synset_id":"find.v.10"},{"index":11,"sense_key":"find%2:32:01::","synset_id":"rule.v.04"},{"index":12,"sense_key":"find%2:30:00::","synset_id":"receive.v.02"},{"index":13,"sense_key":"find%2:39:05::","synset_id":"find.v.13"},{"index":14,"sense_key":"find%2:40:15::","synset_id":"recover.v.01"},{"index":15,"sense_key":"find%2:38:10::","synset_id":"find.v.15"},{"index":16,"sense_key":"find%2:30:13::","synset_id":"find_oneself.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"find%2:31:09::","call_ids":["senseval2.d001.s076.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"find%2:31:09::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1764,"cached_input_tokens":0,"output_tokens":179,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.005318,"input_uncached_usd":0.003528,"input_cached_usd":0.0,"output_usd":0.0017900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.6317537499126047}
{"item_id":"senseval2.d001.s076.t009","gold_sense_keys":["identical%5:00:00:same:00"],"candidates":[{"index":1,"sense_key":"identical%5:00:00:same:00","synset_id":"identical.s.01"},{"index":2,"sense_key":"identical%5:00:00:same:02","synset_id":"identical.s.02"},{"index":3,"sense_key":"identical%3:00:00::","synset_id":"identical.a.03"},{"index":4,"sense_key":"identical%5:00:00:isotropic:00","synset_id":"identical.s.03"},{"index":5,"sense_key":"identical%5:00:00:congruent:00","synset_id":"identical.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"identical%5:00:00:same:00","call_ids":["senseval2.d001.s076.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"identical%5:00:00:same:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":923,"cached_input_tokens":0,"output_tokens":94,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0027860000000000003,"input_uncached_usd":0.001846,"input_cached_usd":0.0,"output_usd":0.0009400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.400343916960992}
{"item_id":"senseval2.d001.s076.t010","gold_sense_keys":["turn_out%2:42:00::","turn_out%2:42:01::"],"candidates":[{"index":1,"sense_key":"turn_out%2:42:00::","synset_id":"prove.v.01"},{"index":2,"sense_key":"turn_out%2:42:01::","synset_id":"turn_out.v.02"},{"index":3,"sense_key":"turn_out%2:36:01::","synset_id":"turn_out.v.03"},{"index":4,"sense_key":"turn_out%2:42:02::","synset_id":"turn_out.v.04"},{"index":5,"sense_key":"turn_out%2:30:00::","synset_id":"turn_out.v.05"},{"index":6,"sense_key":"turn_out%2:36:00::","synset_id":"bear.v.05"},{"index":7,"sense_key":"turn_out%2:35:00::","synset_id":"eject.v.01"},{"index":8,"sense_key":"turn_out%2:41:03::","synset_id":"turn_out.v.08"},{"index":9,"sense_key":"turn_out%2:40:03::","synset_id":"turn_out.v.09"},{"index":10,"sense_key":"turn_out%2:38:00::","synset_id":"turn_out.v.10"},{"index":11,"sense_key":"turn_out%2:35:01::","synset_id":"switch_off.v.01"},{"index":12,"sense_key":"turn_out%2:29:00::","synset_id":"get_up.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"turn_out%2:42:01::","call_ids":["senseval2.d001.s076.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"turn_out%2:42:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1363,"cached_input_tokens":0,"output_tokens":134,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004066,"input_uncached_usd":0.0027259999999999997,"input_cached_usd":0.0,"output_usd":0.00134,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.6650379160419106}
{"item_id":"senseval2.d001.s076.t011","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s076.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":932,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001984,"input_uncached_usd":0.001864,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6301964579615742}
{"item_id":"senseval2.d001.s076.t012","gold_sense_keys":["study%1:04:00::"],"candidates":[{"index":1,"sense_key":"study%1:04:00::","synset_id":"survey.n.01"},{"index":2,"sense_key":"study%1:09:03::","synset_id":"study.n.02"},{"index":3,"sense_key":"study%1:10:00::","synset_id":"report.n.01"},{"index":4,"sense_key":"study%1:09:04::","synset_id":"study.n.04"},{"index":5,"sense_key":"study%1:06:00::","synset_id":"study.n.05"},{"index":6,"sense_key":"study%1:09:02::","synset_id":"discipline.n.01"},{"index":7,"sense_key":"study%1:06:01::","synset_id":"sketch.n.01"},{"index":8,"sense_key":"study%1:09:01::","synset_id":"cogitation.n.02"},{"index":9,"sense_key":"study%1:18:00::","synset_id":"study.n.09"},{"index":10,"sense_key":"study%1:10:01::","synset_id":"study.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"study%1:10:00::","call_ids":["senseval2.d001.s076.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"study%1:10:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1202,"cached_input_tokens":0,"output_tokens":171,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004114,"input_uncached_usd":0.002404,"input_cached_usd":0.0,"output_usd":0.0017100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.4860371249960735}
{"item_id":"senseval2.d001.s076.t014","gold_sense_keys":["observe%2:39:00::"],"candidates":[{"index":1,"sense_key":"observe%2:39:02::","synset_id":"detect.v.01"},{"index":2,"sense_key":"observe%2:32:00::","synset_id":"note.v.01"},{"index":3,"sense_key":"observe%2:39:01::","synset_id":"note.v.03"},{"index":4,"sense_key":"observe%2:39:00::","synset_id":"observe.v.04"},{"index":5,"sense_key":"observe%2:41:01::","synset_id":"respect.v.02"},{"index":6,"sense_key":"observe%2:41:02::","synset_id":"observe.v.06"},{"index":7,"sense_key":"observe%2:41:00::","synset_id":"watch.v.02"},{"index":8,"sense_key":"observe%2:31:00::","synset_id":"observe.v.08"},{"index":9,"sense_key":"observe%2:41:04::","synset_id":"observe.v.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"observe%2:39:00::","call_ids":["senseval2.d001.s076.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"observe%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1203,"cached_input_tokens":0,"output_tokens":156,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003966,"input_uncached_usd":0.0024059999999999997,"input_cached_usd":0.0,"output_usd":0.0015600000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.3873636660864577}
{"item_id":"senseval2.d001.s076.t015","gold_sense_keys":["damaged%3:00:00::"],"candidates":[{"index":1,"sense_key":"damaged%3:00:00::","synset_id":"damaged.a.01"},{"index":2,"sense_key":"damaged%5:00:00:disreputable:00","synset_id":"discredited.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"damaged%3:00:00::","call_ids":["senseval2.d001.s076.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"damaged%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":750,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0016200000000000001,"input_uncached_usd":0.0015,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.623603916959837}
{"item_id":"senseval2.d001.s076.t016","gold_sense_keys":["form%1:07:03::"],"candidates":[{"index":1,"sense_key":"form%1:10:00::","synset_id":"form.n.01"},{"index":2,"sense_key":"form%1:09:01::","synset_id":"kind.n.01"},{"index":3,"sense_key":"form%1:09:00::","synset_id":"form.n.03"},{"index":4,"sense_key":"form%1:07:01::","synset_id":"shape.n.01"},{"index":5,"sense_key":"form%1:08:00::","synset_id":"human_body.n.01"},{"index":6,"sense_key":"form%1:03:00::","synset_id":"shape.n.02"},{"index":7,"sense_key":"form%1:07:00::","synset_id":"form.n.07"},{"index":8,"sense_key":"form%1:10:01::","synset_id":"form.n.08"},{"index":9,"sense_key":"form%1:14:02::","synset_id":"form.n.09"},{"index":10,"sense_key":"form%1:10:02::","synset_id":"form.n.10"},{"index":11,"sense_key":"form%1:07:03::","synset_id":"form.n.11"},{"index":12,"sense_key":"form%1:26:00::","synset_id":"phase.n.02"},{"index":13,"sense_key":"form%1:14:00::","synset_id":"class.n.02"},{"index":14,"sense_key":"form%1:07:02::","synset_id":"form.n.14"},{"index":15,"sense_key":"form%1:06:00::","synset_id":"mannequin.n.02"},{"index":16,"sense_key":"form%1:06:01::","synset_id":"form.n.16"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"form%1:14:02::","call_ids":["senseval2.d001.s076.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"form%1:14:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1686,"cached_input_tokens":0,"output_tokens":121,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004582,"input_uncached_usd":0.003372,"input_cached_usd":0.0,"output_usd":0.0012100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6592675410211086}
{"item_id":"senseval2.d001.s077.t000","gold_sense_keys":["discovery%1:04:00::","discovery%1:09:00::","discovery%1:10:00::"],"candidates":[{"index":1,"sense_key":"discovery%1:04:00::","synset_id":"discovery.n.01"},{"index":2,"sense_key":"discovery%1:10:00::","synset_id":"discovery.n.02"},{"index":3,"sense_key":"discovery%1:09:00::","synset_id":"discovery.n.03"},{"index":4,"sense_key":"discovery%1:10:01::","synset_id":"discovery.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"discovery%1:10:00::","call_ids":["senseval2.d001.s077.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"discovery%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":816,"cached_input_tokens":0,"output_tokens":149,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003122,"input_uncached_usd":0.001632,"input_cached_usd":0.0,"output_usd":0.0014900000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.142401499906555}
{"item_id":"senseval2.d001.s077.t001","gold_sense_keys":["suddenly%4:02:00::","suddenly%4:02:02::"],"candidates":[{"index":1,"sense_key":"suddenly%4:02:02::","synset_id":"suddenly.r.01"},{"index":2,"sense_key":"suddenly%4:02:00::","synset_id":"abruptly.r.01"},{"index":3,"sense_key":"suddenly%4:02:04::","synset_id":"on_the_spur_of_the_moment.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"suddenly%4:02:02::","call_ids":["senseval2.d001.s077.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"suddenly%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":840,"cached_input_tokens":0,"output_tokens":93,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00261,"input_uncached_usd":0.0016799999999999999,"input_cached_usd":0.0,"output_usd":0.00093,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.573688792064786}
{"item_id":"senseval2.d001.s077.t002","gold_sense_keys":["put%2:35:00::"],"candidates":[{"index":1,"sense_key":"put%2:35:00::","synset_id":"put.v.01"},{"index":2,"sense_key":"put%2:35:05::","synset_id":"put.v.02"},{"index":3,"sense_key":"put%2:32:00::","synset_id":"frame.v.04"},{"index":4,"sense_key":"put%2:34:00::","synset_id":"put.v.04"},{"index":5,"sense_key":"put%2:40:00::","synset_id":"invest.v.01"},{"index":6,"sense_key":"put%2:31:13::","synset_id":"place.v.11"},{"index":7,"sense_key":"put%2:39:13::","synset_id":"put.v.07"},{"index":8,"sense_key":"put%2:36:13::","synset_id":"put.v.08"},{"index":9,"sense_key":"put%2:31:00::","synset_id":"arrange.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"put%2:35:00::","call_ids":["senseval2.d001.s077.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"put%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1137,"cached_input_tokens":0,"output_tokens":74,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003014,"input_uncached_usd":0.002274,"input_cached_usd":0.0,"output_usd":0.0007400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4815124170854688}
{"item_id":"senseval2.d001.s077.t005","gold_sense_keys":["right%4:02:04::","right%4:02:09::"],"candidates":[{"index":1,"sense_key":"right%4:02:04::","synset_id":"right.r.01"},{"index":2,"sense_key":"right%4:02:08::","synset_id":"right.r.02"},{"index":3,"sense_key":"right%4:02:09::","synset_id":"right.r.03"},{"index":4,"sense_key":"right%4:02:03::","synset_id":"right.r.04"},{"index":5,"sense_key":"right%4:02:01::","synset_id":"properly.r.01"},{"index":6,"sense_key":"right%4:02:00::","synset_id":"right.r.06"},{"index":7,"sense_key":"right%4:02:07::","synset_id":"right.r.07"},{"index":8,"sense_key":"right%4:02:06::","synset_id":"mighty.r.01"},{"index":9,"sense_key":"right%4:02:05::","synset_id":"justly.r.02"},{"index":10,"sense_key":"right%4:02:02::","synset_id":"correctly.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"right%4:02:04::","call_ids":["senseval2.d001.s077.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"right%4:02:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1136,"cached_input_tokens":0,"output_tokens":90,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003172,"input_uncached_usd":0.0022719999999999997,"input_cached_usd":0.0,"output_usd":0.0009000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4110598750412464}
{"item_id":"senseval2.d001.s077.t006","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s077.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":944,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002008,"input_uncached_usd":0.001888,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5181282910052687}
{"item_id":"senseval2.d001.s077.t007","gold_sense_keys":["formation%1:22:00::"],"candidates":[{"index":1,"sense_key":"formation%1:14:00::","synset_id":"formation.n.01"},{"index":2,"sense_key":"formation%1:04:00::","synset_id":"formation.n.02"},{"index":3,"sense_key":"formation%1:04:01::","synset_id":"constitution.n.02"},{"index":4,"sense_key":"formation%1:17:00::","synset_id":"geological_formation.n.01"},{"index":5,"sense_key":"formation%1:06:00::","synset_id":"formation.n.05"},{"index":6,"sense_key":"formation%1:22:00::","synset_id":"formation.n.06"},{"index":7,"sense_key":"formation%1:04:02::","synset_id":"formation.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"formation%1:22:00::","call_ids":["senseval2.d001.s077.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"formation%1:22:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":983,"cached_input_tokens":0,"output_tokens":71,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002676,"input_uncached_usd":0.001966,"input_cached_usd":0.0,"output_usd":0.00071,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.735603625071235}
{"item_id":"senseval2.d001.s077.t008","gold_sense_keys":["say%2:32:00::","say%2:32:15::"],"candidates":[{"index":1,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":6,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":9,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":10,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":11,"sense_key":"say%2:32:09::","synset_id":"say.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"say%2:32:00::","call_ids":["senseval2.d001.s077.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"say%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1284,"cached_input_tokens":0,"output_tokens":135,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003918,"input_uncached_usd":0.002568,"input_cached_usd":0.0,"output_usd":0.00135,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.4536389589775354}
{"item_id":"senseval2.d001.s077.t009","gold_sense_keys":["leader%1:18:00::"],"candidates":[{"index":1,"sense_key":"leader%1:18:00::","synset_id":"leader.n.01"},{"index":2,"sense_key":"leader%1:06:00::","synset_id":"drawing_card.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"leader%1:18:00::","call_ids":["senseval2.d001.s077.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"leader%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":721,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001562,"input_uncached_usd":0.001442,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0814045839942992}
{"item_id":"senseval2.d001.s078.t000","gold_sense_keys":["evidence%1:09:00::","evidence%1:10:01::"],"candidates":[{"index":1,"sense_key":"evidence%1:09:00::","synset_id":"evidence.n.01"},{"index":2,"sense_key":"evidence%1:10:01::","synset_id":"evidence.n.02"},{"index":3,"sense_key":"evidence%1:10:00::","synset_id":"evidence.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"evidence%1:09:00::","call_ids":["senseval2.d001.s078.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"evidence%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":850,"cached_input_tokens":0,"output_tokens":122,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00292,"input_uncached_usd":0.0017,"input_cached_usd":0.0,"output_usd":0.0012200000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.892893375013955}
{"item_id":"senseval2.d001.s078.t001","gold_sense_keys":["now%4:02:00::","now%4:02:04::"],"candidates":[{"index":1,"sense_key":"now%4:02:05::","synset_id":"now.r.01"},{"index":2,"sense_key":"now%4:02:00::","synset_id":"nowadays.r.01"},{"index":3,"sense_key":"now%4:02:02::","synset_id":"now.r.03"},{"index":4,"sense_key":"now%4:02:04::","synset_id":"now.r.04"},{"index":5,"sense_key":"now%4:02:01::","synset_id":"immediately.r.01"},{"index":6,"sense_key":"now%4:02:07::","synset_id":"now.r.06"},{"index":7,"sense_key":"now%4:02:06::","synset_id":"now.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"now%4:02:04::","call_ids":["senseval2.d001.s078.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"now%4:02:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1099,"cached_input_tokens":0,"output_tokens":125,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0034479999999999997,"input_uncached_usd":0.002198,"input_cached_usd":0.0,"output_usd":0.00125,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8067162081133574}
{"item_id":"senseval2.d001.s078.t002","gold_sense_keys":["emerge%2:30:01::"],"candidates":[{"index":1,"sense_key":"emerge%2:30:00::","synset_id":"emerge.v.01"},{"index":2,"sense_key":"emerge%2:30:02::","synset_id":"issue.v.04"},{"index":3,"sense_key":"emerge%2:30:01::","synset_id":"emerge.v.03"},{"index":4,"sense_key":"emerge%2:38:00::","synset_id":"emerge.v.04"},{"index":5,"sense_key":"emerge%2:42:00::","synset_id":"come_forth.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"emerge%2:30:01::","call_ids":["senseval2.d001.s078.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"emerge%2:30:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":927,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001974,"input_uncached_usd":0.001854,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.537771625095047}
{"item_id":"senseval2.d001.s078.t004","gold_sense_keys":["involved%3:00:00::"],"candidates":[{"index":1,"sense_key":"involved%3:00:00::","synset_id":"involved.a.01"},{"index":2,"sense_key":"involved%5:00:00:encumbered:00","synset_id":"involved.s.01"},{"index":3,"sense_key":"involved%5:00:00:attached:01","synset_id":"involved.s.02"},{"index":4,"sense_key":"involved%5:00:00:complex:00","synset_id":"byzantine.s.01"},{"index":5,"sense_key":"involved%5:00:00:enclosed:00","synset_id":"involved.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"involved%3:00:00::","call_ids":["senseval2.d001.s078.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"involved%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1023,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002166,"input_uncached_usd":0.002046,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1015754580730572}
{"item_id":"senseval2.d001.s078.t005","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":2,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":3,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"other%3:00:00::","call_ids":["senseval2.d001.s078.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":894,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001908,"input_uncached_usd":0.0017879999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.527438999968581}
{"item_id":"senseval2.d001.s078.t006","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s078.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":978,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0020759999999999997,"input_uncached_usd":0.001956,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5010751250665635}
{"item_id":"senseval2.d001.s078.t007","gold_sense_keys":["too%4:02:01::"],"candidates":[{"index":1,"sense_key":"too%4:02:00::","synset_id":"excessively.r.01"},{"index":2,"sense_key":"too%4:02:01::","synset_id":"besides.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"too%4:02:01::","call_ids":["senseval2.d001.s078.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"too%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":773,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001666,"input_uncached_usd":0.0015459999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3935217909747735}
{"item_id":"senseval2.d001.s079.t001","gold_sense_keys":["find%2:31:09::","find%2:32:00::","find%2:36:00::","find%2:39:02::"],"candidates":[{"index":1,"sense_key":"find%2:40:02::","synset_id":"find.v.01"},{"index":2,"sense_key":"find%2:39:02::","synset_id":"detect.v.01"},{"index":3,"sense_key":"find%2:40:00::","synset_id":"find.v.03"},{"index":4,"sense_key":"find%2:32:00::","synset_id":"determine.v.01"},{"index":5,"sense_key":"find%2:31:10::","synset_id":"find.v.05"},{"index":6,"sense_key":"find%2:39:01::","synset_id":"witness.v.02"},{"index":7,"sense_key":"find%2:40:01::","synset_id":"line_up.v.02"},{"index":8,"sense_key":"find%2:36:00::","synset_id":"discover.v.03"},{"index":9,"sense_key":"find%2:31:09::","synset_id":"discover.v.04"},{"index":10,"sense_key":"find%2:40:03::","synset_id":"find.v.10"},{"index":11,"sense_key":"find%2:32:01::","synset_id":"rule.v.04"},{"index":12,"sense_key":"find%2:30:00::","synset_id":"receive.v.02"},{"index":13,"sense_key":"find%2:39:05::","synset_id":"find.v.13"},{"index":14,"sense_key":"find%2:40:15::","synset_id":"recover.v.01"},{"index":15,"sense_key":"find%2:38:10::","synset_id":"find.v.15"},{"index":16,"sense_key":"find%2:30:13::","synset_id":"find_oneself.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"find%2:31:09::","call_ids":["senseval2.d001.s079.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"find%2:31:09::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1793,"cached_input_tokens":0,"output_tokens":157,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.005156,"input_uncached_usd":0.0035859999999999998,"input_cached_usd":0.0,"output_usd":0.0015700000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8940993329742923}
{"item_id":"senseval2.d001.s079.t002","gold_sense_keys":["breast%1:08:00::"],"candidates":[{"index":1,"sense_key":"breast%1:08:01::","synset_id":"breast.n.01"},{"index":2,"sense_key":"breast%1:08:00::","synset_id":"breast.n.02"},{"index":3,"sense_key":"breast%1:13:00::","synset_id":"breast.n.03"},{"index":4,"sense_key":"breast%1:05:00::","synset_id":"breast.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"breast%1:08:00::","call_ids":["senseval2.d001.s079.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"breast%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":861,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001842,"input_uncached_usd":0.001722,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4610092079965398}
{"item_id":"senseval2.d001.s079.t004","gold_sense_keys":["copy%1:06:00::"],"candidates":[{"index":1,"sense_key":"copy%1:10:00::","synset_id":"transcript.n.02"},{"index":2,"sense_key":"copy%1:06:00::","synset_id":"copy.n.02"},{"index":3,"sense_key":"copy%1:10:01::","synset_id":"copy.n.03"},{"index":4,"sense_key":"copy%1:10:02::","synset_id":"copy.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"copy%1:06:00::","call_ids":["senseval2.d001.s079.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"copy%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":857,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0018340000000000001,"input_uncached_usd":0.001714,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0622472090180963}
{"item_id":"senseval2.d001.s079.t007","gold_sense_keys":["spot%1:15:01::"],"candidates":[{"index":1,"sense_key":"spot%1:15:01::","synset_id":"topographic_point.n.01"},{"index":2,"sense_key":"spot%1:10:01::","synset_id":"spot.n.02"},{"index":3,"sense_key":"spot%1:07:02::","synset_id":"point.n.14"},{"index":4,"sense_key":"spot%1:07:00::","synset_id":"smudge.n.02"},{"index":5,"sense_key":"spot%1:07:01::","synset_id":"spot.n.05"},{"index":6,"sense_key":"spot%1:10:00::","synset_id":"spot.n.06"},{"index":7,"sense_key":"spot%1:06:00::","synset_id":"spot.n.07"},{"index":8,"sense_key":"spot%1:04:01::","synset_id":"position.n.06"},{"index":9,"sense_key":"spot%1:26:00::","synset_id":"touch.n.08"},{"index":10,"sense_key":"spot%1:23:00::","synset_id":"spot.n.10"},{"index":11,"sense_key":"spot%1:10:02::","synset_id":"spot.n.11"},{"index":12,"sense_key":"spot%1:06:01::","synset_id":"spotlight.n.02"},{"index":13,"sense_key":"spot%1:06:02::","synset_id":"spot.n.13"},{"index":14,"sense_key":"spot%1:04:00::","synset_id":"blot.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"spot%1:15:01::","call_ids":["senseval2.d001.s079.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"spot%1:15:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1479,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0030779999999999996,"input_uncached_usd":0.0029579999999999997,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.031094874953851}
{"item_id":"senseval2.d001.s080.t001","gold_sense_keys":["say%2:32:01::","say%2:32:13::"],"candidates":[{"index":1,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":6,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":9,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":10,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":11,"sense_key":"say%2:32:09::","synset_id":"say.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"say%2:32:01::","call_ids":["senseval2.d001.s080.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"say%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1292,"cached_input_tokens":0,"output_tokens":231,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004894,"input_uncached_usd":0.002584,"input_cached_usd":0.0,"output_usd":0.00231,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.66428295802325}
{"item_id":"senseval2.d001.s080.t004","gold_sense_keys":["often%4:02:00::","often%4:02:02::"],"candidates":[{"index":1,"sense_key":"often%4:02:00::","synset_id":"frequently.r.01"},{"index":2,"sense_key":"often%4:02:01::","synset_id":"much.r.05"},{"index":3,"sense_key":"often%4:02:02::","synset_id":"often.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"often%4:02:02::","call_ids":["senseval2.d001.s080.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"often%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":787,"cached_input_tokens":0,"output_tokens":112,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002694,"input_uncached_usd":0.0015739999999999999,"input_cached_usd":0.0,"output_usd":0.0011200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6569380409782752}
{"item_id":"senseval2.d001.s080.t005","gold_sense_keys":["strike%2:35:02::"],"candidates":[{"index":1,"sense_key":"strike%2:35:00::","synset_id":"strike.v.01"},{"index":2,"sense_key":"strike%2:37:00::","synset_id":"affect.v.05"},{"index":3,"sense_key":"strike%2:35:01::","synset_id":"hit.v.02"},{"index":4,"sense_key":"strike%2:33:00::","synset_id":"strike.v.04"},{"index":5,"sense_key":"strike%2:32:00::","synset_id":"strike.v.05"},{"index":6,"sense_key":"strike%2:35:02::","synset_id":"hit.v.05"},{"index":7,"sense_key":"strike%2:41:00::","synset_id":"strike.v.07"},{"index":8,"sense_key":"strike%2:42:00::","synset_id":"fall.v.08"},{"index":9,"sense_key":"strike%2:41:02::","synset_id":"come_to.v.03"},{"index":10,"sense_key":"strike%2:35:03::","synset_id":"strike.v.10"},{"index":11,"sense_key":"strike%2:30:00::","synset_id":"strike.v.11"},{"index":12,"sense_key":"strike%2:40:00::","synset_id":"fall_upon.v.01"},{"index":13,"sense_key":"strike%2:36:00::","synset_id":"strike.v.13"},{"index":14,"sense_key":"strike%2:35:08::","synset_id":"strike.v.14"},{"index":15,"sense_key":"strike%2:39:00::","synset_id":"hit.v.09"},{"index":16,"sense_key":"strike%2:38:02::","synset_id":"hit.v.12"},{"index":17,"sense_key":"strike%2:38:08::","synset_id":"assume.v.05"},{"index":18,"sense_key":"strike%2:36:02::","synset_id":"mint.v.01"},{"index":19,"sense_key":"strike%2:35:10::","synset_id":"strickle.v.02"},{"index":20,"sense_key":"strike%2:35:09::","synset_id":"strike.v.20"},{"index":21,"sense_key":"strike%2:31:00::","synset_id":"strike.v.21"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"strike%2:35:02::","call_ids":["senseval2.d001.s080.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"strike%2:35:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2041,"cached_input_tokens":0,"output_tokens":91,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004992,"input_uncached_usd":0.004082,"input_cached_usd":0.0,"output_usd":0.0009100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8858733749948442}
{"item_id":"senseval2.d001.s080.t007","gold_sense_keys":["member%1:18:00::"],"candidates":[{"index":1,"sense_key":"member%1:18:00::","synset_id":"member.n.01"},{"index":2,"sense_key":"member%1:24:00::","synset_id":"member.n.02"},{"index":3,"sense_key":"member%1:08:00::","synset_id":"extremity.n.01"},{"index":4,"sense_key":"member%1:14:00::","synset_id":"member.n.04"},{"index":5,"sense_key":"member%1:08:01::","synset_id":"penis.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"member%1:24:00::","call_ids":["senseval2.d001.s080.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"member%1:24:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":963,"cached_input_tokens":0,"output_tokens":119,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003116,"input_uncached_usd":0.001926,"input_cached_usd":0.0,"output_usd":0.00119,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.275488750077784}
{"item_id":"senseval2.d001.s080.t008","gold_sense_keys":["certain%5:00:00:definite:00"],"candidates":[{"index":1,"sense_key":"certain%5:00:00:definite:00","synset_id":"certain.s.01"},{"index":2,"sense_key":"certain%3:00:02::","synset_id":"certain.a.02"},{"index":3,"sense_key":"certain%3:00:01::","synset_id":"certain.a.03"},{"index":4,"sense_key":"certain%3:00:03::","synset_id":"certain.a.04"},{"index":5,"sense_key":"certain%3:00:05::","synset_id":"sealed.a.01"},{"index":6,"sense_key":"certain%5:00:00:reliable:00","synset_id":"certain.s.02"},{"index":7,"sense_key":"certain%5:00:00:careful:00","synset_id":"certain.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"certain%5:00:00:definite:00","call_ids":["senseval2.d001.s080.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"certain%5:00:00:definite:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1104,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0023279999999999998,"input_uncached_usd":0.002208,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4882657079724595}
{"item_id":"senseval2.d001.s080.t009","gold_sense_keys":["family%1:14:01::"],"candidates":[{"index":1,"sense_key":"family%1:14:02::","synset_id":"family.n.01"},{"index":2,"sense_key":"family%1:14:00::","synset_id":"family.n.02"},{"index":3,"sense_key":"family%1:14:04::","synset_id":"class.n.01"},{"index":4,"sense_key":"family%1:14:01::","synset_id":"family.n.04"},{"index":5,"sense_key":"family%1:18:00::","synset_id":"kin.n.01"},{"index":6,"sense_key":"family%1:14:03::","synset_id":"family.n.06"},{"index":7,"sense_key":"family%1:14:06::","synset_id":"syndicate.n.01"},{"index":8,"sense_key":"family%1:14:05::","synset_id":"family.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"family%1:14:01::","call_ids":["senseval2.d001.s080.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"family%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1152,"cached_input_tokens":0,"output_tokens":112,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003424,"input_uncached_usd":0.002304,"input_cached_usd":0.0,"output_usd":0.0011200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1529929999960586}
{"item_id":"senseval2.d001.s080.t011","gold_sense_keys":["inherit%2:40:01::"],"candidates":[{"index":1,"sense_key":"inherit%2:40:00::","synset_id":"inherit.v.01"},{"index":2,"sense_key":"inherit%2:40:02::","synset_id":"inherit.v.02"},{"index":3,"sense_key":"inherit%2:40:01::","synset_id":"inherit.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"inherit%2:40:01::","call_ids":["senseval2.d001.s080.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"inherit%2:40:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":789,"cached_input_tokens":0,"output_tokens":46,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002038,"input_uncached_usd":0.001578,"input_cached_usd":0.0,"output_usd":0.00046,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.7826708330539986}
{"item_id":"senseval2.d001.s080.t012","gold_sense_keys":["damaged%3:00:00::"],"candidates":[{"index":1,"sense_key":"damaged%3:00:00::","synset_id":"damaged.a.01"},{"index":2,"sense_key":"damaged%5:00:00:disreputable:00","synset_id":"discredited.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"damaged%3:00:00::","call_ids":["senseval2.d001.s080.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"damaged%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":770,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00166,"input_uncached_usd":0.00154,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2614422499900684}
{"item_id":"senseval2.d001.s080.t013","gold_sense_keys":["form%1:07:03::"],"candidates":[{"index":1,"sense_key":"form%1:10:00::","synset_id":"form.n.01"},{"index":2,"sense_key":"form%1:09:01::","synset_id":"kind.n.01"},{"index":3,"sense_key":"form%1:09:00::","synset_id":"form.n.03"},{"index":4,"sense_key":"form%1:07:01::","synset_id":"shape.n.01"},{"index":5,"sense_key":"form%1:08:00::","synset_id":"human_body.n.01"},{"index":6,"sense_key":"form%1:03:00::","synset_id":"shape.n.02"},{"index":7,"sense_key":"form%1:07:00::","synset_id":"form.n.07"},{"index":8,"sense_key":"form%1:10:01::","synset_id":"form.n.08"},{"index":9,"sense_key":"form%1:14:02::","synset_id":"form.n.09"},{"index":10,"sense_key":"form%1:10:02::","synset_id":"form.n.10"},{"index":11,"sense_key":"form%1:07:03::","synset_id":"form.n.11"},{"index":12,"sense_key":"form%1:26:00::","synset_id":"phase.n.02"},{"index":13,"sense_key":"form%1:14:00::","synset_id":"class.n.02"},{"index":14,"sense_key":"form%1:07:02::","synset_id":"form.n.14"},{"index":15,"sense_key":"form%1:06:00::","synset_id":"mannequin.n.02"},{"index":16,"sense_key":"form%1:06:01::","synset_id":"form.n.16"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"form%1:14:02::","call_ids":["senseval2.d001.s080.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"form%1:14:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1706,"cached_input_tokens":0,"output_tokens":120,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004612,"input_uncached_usd":0.0034119999999999997,"input_cached_usd":0.0,"output_usd":0.0012000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.576728207990527}
{"item_id":"senseval2.d001.s080.t014","gold_sense_keys":["woman%1:18:00::"],"candidates":[{"index":1,"sense_key":"woman%1:18:00::","synset_id":"woman.n.01"},{"index":2,"sense_key":"woman%1:18:01::","synset_id":"woman.n.02"},{"index":3,"sense_key":"woman%1:18:02::","synset_id":"charwoman.n.01"},{"index":4,"sense_key":"woman%1:14:00::","synset_id":"womanhood.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"woman%1:18:00::","call_ids":["senseval2.d001.s080.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"woman%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":916,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002602,"input_uncached_usd":0.0018319999999999999,"input_cached_usd":0.0,"output_usd":0.0007700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.394364916952327}
{"item_id":"senseval2.d001.s080.t015","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s080.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":952,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0020239999999999998,"input_uncached_usd":0.0019039999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1647943339776248}
{"item_id":"senseval2.d001.s081.t002","gold_sense_keys":["implicate%2:42:01::"],"candidates":[{"index":1,"sense_key":"implicate%2:42:01::","synset_id":"implicate.v.01"},{"index":2,"sense_key":"implicate%2:42:00::","synset_id":"entail.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"implicate%2:42:01::","call_ids":["senseval2.d001.s081.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"implicate%2:42:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":752,"cached_input_tokens":0,"output_tokens":114,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002644,"input_uncached_usd":0.001504,"input_cached_usd":0.0,"output_usd":0.0011400000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6289284160593525}
{"item_id":"senseval2.d001.s082.t000","gold_sense_keys":["report%1:10:03::"],"candidates":[{"index":1,"sense_key":"report%1:10:03::","synset_id":"report.n.01"},{"index":2,"sense_key":"report%1:10:01::","synset_id":"report.n.02"},{"index":3,"sense_key":"report%1:10:00::","synset_id":"report.n.03"},{"index":4,"sense_key":"report%1:11:00::","synset_id":"report.n.04"},{"index":5,"sense_key":"report%1:10:04::","synset_id":"report_card.n.01"},{"index":6,"sense_key":"report%1:10:02::","synset_id":"composition.n.08"},{"index":7,"sense_key":"report%1:09:00::","synset_id":"reputation.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"report%1:10:03::","call_ids":["senseval2.d001.s082.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"report%1:10:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1014,"cached_input_tokens":0,"output_tokens":93,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0029579999999999997,"input_uncached_usd":0.002028,"input_cached_usd":0.0,"output_usd":0.00093,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4689960830146447}
{"item_id":"senseval2.d001.s082.t001","gold_sense_keys":["last%5:00:00:past:00"],"candidates":[{"index":1,"sense_key":"last%5:00:00:past:00","synset_id":"last.s.01"},{"index":2,"sense_key":"last%3:00:00::","synset_id":"last.a.02"},{"index":3,"sense_key":"last%5:00:00:closing:00","synset_id":"concluding.s.01"},{"index":4,"sense_key":"last%5:00:00:unlikely:00","synset_id":"last.s.03"},{"index":5,"sense_key":"last%5:00:00:dying:00","synset_id":"last.s.04"},{"index":6,"sense_key":"last%5:00:00:ultimate:00","synset_id":"final.s.02"},{"index":7,"sense_key":"last%5:00:00:high:02","synset_id":"last.s.06"},{"index":8,"sense_key":"last%5:00:00:unalterable:00","synset_id":"final.s.03"},{"index":9,"sense_key":"last%5:00:00:worst:00","synset_id":"last.s.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"last%5:00:00:past:00","call_ids":["senseval2.d001.s082.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"last%5:00:00:past:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1118,"cached_input_tokens":0,"output_tokens":52,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0027559999999999998,"input_uncached_usd":0.0022359999999999997,"input_cached_usd":0.0,"output_usd":0.0005200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5012723330873996}
{"item_id":"senseval2.d001.s082.t002","gold_sense_keys":["week%1:28:02::"],"candidates":[{"index":1,"sense_key":"week%1:28:00::","synset_id":"week.n.01"},{"index":2,"sense_key":"week%1:28:01::","synset_id":"workweek.n.01"},{"index":3,"sense_key":"week%1:28:02::","synset_id":"week.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"week%1:28:00::","call_ids":["senseval2.d001.s082.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"week%1:28:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":721,"cached_input_tokens":0,"output_tokens":96,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002402,"input_uncached_usd":0.001442,"input_cached_usd":0.0,"output_usd":0.0009600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1014792079804465}
{"item_id":"senseval2.d001.s082.t004","gold_sense_keys":["say%2:32:01::"],"candidates":[{"index":1,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":6,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":9,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":10,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":11,"sense_key":"say%2:32:09::","synset_id":"say.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"say%2:32:01::","call_ids":["senseval2.d001.s082.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"say%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1224,"cached_input_tokens":0,"output_tokens":142,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003868,"input_uncached_usd":0.002448,"input_cached_usd":0.0,"output_usd":0.00142,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2467612080508843}
{"item_id":"senseval2.d001.s082.t005","gold_sense_keys":["about%4:02:00::"],"candidates":[{"index":1,"sense_key":"about%4:02:00::","synset_id":"approximately.r.01"},{"index":2,"sense_key":"about%4:02:06::","synset_id":"about.r.02"},{"index":3,"sense_key":"about%4:02:01::","synset_id":"about.r.03"},{"index":4,"sense_key":"about%4:02:04::","synset_id":"about.r.04"},{"index":5,"sense_key":"about%4:02:02::","synset_id":"about.r.05"},{"index":6,"sense_key":"about%4:02:03::","synset_id":"about.r.06"},{"index":7,"sense_key":"about%4:02:05::","synset_id":"about.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"about%4:02:00::","call_ids":["senseval2.d001.s082.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"about%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1016,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002152,"input_uncached_usd":0.002032,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5220155839342624}
{"item_id":"senseval2.d001.s082.t006","gold_sense_keys":["cell%1:03:00::"],"candidates":[{"index":1,"sense_key":"cell%1:06:03::","synset_id":"cell.n.01"},{"index":2,"sense_key":"cell%1:03:00::","synset_id":"cell.n.02"},{"index":3,"sense_key":"cell%1:06:00::","synset_id":"cell.n.03"},{"index":4,"sense_key":"cell%1:14:00::","synset_id":"cell.n.04"},{"index":5,"sense_key":"cell%1:06:04::","synset_id":"cellular_telephone.n.01"},{"index":6,"sense_key":"cell%1:06:02::","synset_id":"cell.n.06"},{"index":7,"sense_key":"cell%1:06:01::","synset_id":"cell.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cell%1:03:00::","call_ids":["senseval2.d001.s082.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cell%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":976,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002072,"input_uncached_usd":0.001952,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6892219580477104}
{"item_id":"senseval2.d001.s082.t007","gold_sense_keys":["take%2:30:00::"],"candidates":[{"index":1,"sense_key":"take%2:41:04::","synset_id":"take.v.01"},{"index":2,"sense_key":"take%2:40:06::","synset_id":"take.v.02"},{"index":3,"sense_key":"take%2:38:09::","synset_id":"lead.v.01"},{"index":4,"sense_key":"take%2:35:00::","synset_id":"take.v.04"},{"index":5,"sense_key":"take%2:30:01::","synset_id":"assume.v.03"},{"index":6,"sense_key":"take%2:31:07::","synset_id":"take.v.06"},{"index":7,"sense_key":"take%2:38:10::","synset_id":"bring.v.01"},{"index":8,"sense_key":"take%2:40:00::","synset_id":"take.v.08"},{"index":9,"sense_key":"take%2:38:02::","synset_id":"take.v.09"},{"index":10,"sense_key":"take%2:31:01::","synset_id":"choose.v.01"},{"index":11,"sense_key":"take%2:40:05::","synset_id":"accept.v.02"},{"index":12,"sense_key":"take%2:41:00::","synset_id":"fill.v.04"},{"index":13,"sense_key":"take%2:31:03::","synset_id":"consider.v.03"},{"index":14,"sense_key":"take%2:42:00::","synset_id":"necessitate.v.01"},{"index":15,"sense_key":"take%2:39:00::","synset_id":"take.v.15"},{"index":16,"sense_key":"take%2:32:02::","synset_id":"film.v.01"},{"index":17,"sense_key":"take%2:30:00::","synset_id":"remove.v.01"},{"index":18,"sense_key":"take%2:34:00::","synset_id":"consume.v.02"},{"index":19,"sense_key":"take%2:31:09::","synset_id":"take.v.19"},{"index":20,"sense_key":"take%2:40:15::","synset_id":"take.v.20"},{"index":21,"sense_key":"take%2:40:01::","synset_id":"take.v.21"},{"index":22,"sense_key":"take%2:38:00::","synset_id":"assume.v.05"},{"index":23,"sense_key":"take%2:40:07::","synset_id":"accept.v.05"},{"index":24,"sense_key":"take%2:30:08::","synset_id":"take.v.24"},{"index":25,"sense_key":"take%2:31:02::","synset_id":"learn.v.04"},{"index":26,"sense_key":"take%2:32:00::","synset_id":"claim.v.05"},{"index":27,"sense_key":"take%2:38:05::","synset_id":"take.v.27"},{"index":28,"sense_key":"take%2:33:09::","synset_id":"aim.v.01"},{"index":29,"sense_key":"take%2:30:09::","synset_id":"take.v.29"},{"index":30,"sense_key":"take%2:42:10::","synset_id":"carry.v.02"},{"index":31,"sense_key":"take%2:40:03::","synset_id":"lease.v.04"},{"index":32,"sense_key":"take%2:40:04::","synset_id":"subscribe.v.05"},{"index":33,"sense_key":"take%2:40:02::","synset_id":"take.v.33"},{"index":34,"sense_key":"take%2:40:08::","synset_id":"take.v.34"},{"index":35,"sense_key":"take%2:35:14::","synset_id":"take.v.35"},{"index":36,"sense_key":"take%2:32:09::","synset_id":"claim.v.04"},{"index":37,"sense_key":"take%2:42:03::","synset_id":"accept.v.08"},{"index":38,"sense_key":"take%2:42:15::","synset_id":"contain.v.05"},{"index":39,"sense_key":"take%2:41:13::","synset_id":"take.v.39"},{"index":40,"sense_key":"take%2:38:11::","synset_id":"drive.v.16"},{"index":41,"sense_key":"take%2:33:08::","synset_id":"take.v.41"},{"index":42,"sense_key":"take%2:29:08::","synset_id":"contract.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":17,"chosen_sense_key":"take%2:30:00::","call_ids":["senseval2.d001.s082.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":17,"predicted_sense_key":"take%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":3168,"cached_input_tokens":0,"output_tokens":86,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.007195999999999999,"input_uncached_usd":0.006336,"input_cached_usd":0.0,"output_usd":0.0008600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9567874580388889}
{"item_id":"senseval2.d001.s082.t010","gold_sense_keys":["tissue%1:08:00::"],"candidates":[{"index":1,"sense_key":"tissue%1:08:00::","synset_id":"tissue.n.01"},{"index":2,"sense_key":"tissue%1:27:02::","synset_id":"tissue.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"tissue%1:08:00::","call_ids":["senseval2.d001.s082.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"tissue%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":665,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0014500000000000001,"input_uncached_usd":0.00133,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0760175829054788}
{"item_id":"senseval2.d001.s082.t011","gold_sense_keys":["test%2:32:08::"],"candidates":[{"index":1,"sense_key":"test%2:41:00::","synset_id":"test.v.01"},{"index":2,"sense_key":"test%2:41:01::","synset_id":"screen.v.01"},{"index":3,"sense_key":"test%2:32:00::","synset_id":"quiz.v.01"},{"index":4,"sense_key":"test%2:42:00::","synset_id":"test.v.04"},{"index":5,"sense_key":"test%2:33:00::","synset_id":"test.v.05"},{"index":6,"sense_key":"test%2:32:08::","synset_id":"test.v.06"},{"index":7,"sense_key":"test%2:31:06::","synset_id":"test.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"test%2:32:08::","call_ids":["senseval2.d001.s082.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"test%2:32:08::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":950,"cached_input_tokens":0,"output_tokens":131,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00321,"input_uncached_usd":0.0019,"input_cached_usd":0.0,"output_usd":0.0013100000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0782833750126883}
{"item_id":"senseval2.d001.s082.t012","gold_sense_keys":["miss%2:42:00::"],"candidates":[{"index":1,"sense_key":"miss%2:39:00::","synset_id":"miss.v.01"},{"index":2,"sense_key":"miss%2:37:00::","synset_id":"miss.v.02"},{"index":3,"sense_key":"miss%2:42:01::","synset_id":"miss.v.03"},{"index":4,"sense_key":"miss%2:31:00::","synset_id":"neglect.v.01"},{"index":5,"sense_key":"miss%2:38:00::","synset_id":"miss.v.05"},{"index":6,"sense_key":"miss%2:42:00::","synset_id":"miss.v.06"},{"index":7,"sense_key":"miss%2:35:00::","synset_id":"miss.v.07"},{"index":8,"sense_key":"miss%2:42:02::","synset_id":"miss.v.08"},{"index":9,"sense_key":"miss%2:32:00::","synset_id":"miss.v.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"miss%2:42:00::","call_ids":["senseval2.d001.s082.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"miss%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1041,"cached_input_tokens":0,"output_tokens":98,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0030620000000000005,"input_uncached_usd":0.002082,"input_cached_usd":0.0,"output_usd":0.0009800000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.02343187504448}
{"item_id":"senseval2.d001.s083.t000","gold_sense_keys":["report%1:10:03::"],"candidates":[{"index":1,"sense_key":"report%1:10:03::","synset_id":"report.n.01"},{"index":2,"sense_key":"report%1:10:01::","synset_id":"report.n.02"},{"index":3,"sense_key":"report%1:10:00::","synset_id":"report.n.03"},{"index":4,"sense_key":"report%1:11:00::","synset_id":"report.n.04"},{"index":5,"sense_key":"report%1:10:04::","synset_id":"report_card.n.01"},{"index":6,"sense_key":"report%1:10:02::","synset_id":"composition.n.08"},{"index":7,"sense_key":"report%1:09:00::","synset_id":"reputation.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"report%1:10:01::","call_ids":["senseval2.d001.s083.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"report%1:10:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":981,"cached_input_tokens":0,"output_tokens":152,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003482,"input_uncached_usd":0.0019619999999999998,"input_cached_usd":0.0,"output_usd":0.00152,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9937646250473335}
{"item_id":"senseval2.d001.s083.t003","gold_sense_keys":["yet%4:02:02::"],"candidates":[{"index":1,"sense_key":"yet%4:02:02::","synset_id":"yet.r.01"},{"index":2,"sense_key":"yet%4:02:04::","synset_id":"so_far.r.01"},{"index":3,"sense_key":"yet%4:02:03::","synset_id":"even.r.03"},{"index":4,"sense_key":"yet%4:02:00::","synset_id":"yet.r.04"},{"index":5,"sense_key":"yet%4:02:05::","synset_id":"so_far.r.03"},{"index":6,"sense_key":"yet%4:02:06::","synset_id":"however.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"yet%4:02:04::","call_ids":["senseval2.d001.s083.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"yet%4:02:04::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":997,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002664,"input_uncached_usd":0.001994,"input_cached_usd":0.0,"output_usd":0.00067,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7686924999579787}
{"item_id":"senseval2.d001.s083.t007","gold_sense_keys":["tissue%1:08:00::"],"candidates":[{"index":1,"sense_key":"tissue%1:08:00::","synset_id":"tissue.n.01"},{"index":2,"sense_key":"tissue%1:27:02::","synset_id":"tissue.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"tissue%1:08:00::","call_ids":["senseval2.d001.s083.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"tissue%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":631,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001382,"input_uncached_usd":0.0012619999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.007882166071795}
{"item_id":"senseval2.d001.s083.t008","gold_sense_keys":["take%2:30:00::"],"candidates":[{"index":1,"sense_key":"take%2:41:04::","synset_id":"take.v.01"},{"index":2,"sense_key":"take%2:40:06::","synset_id":"take.v.02"},{"index":3,"sense_key":"take%2:38:09::","synset_id":"lead.v.01"},{"index":4,"sense_key":"take%2:35:00::","synset_id":"take.v.04"},{"index":5,"sense_key":"take%2:30:01::","synset_id":"assume.v.03"},{"index":6,"sense_key":"take%2:31:07::","synset_id":"take.v.06"},{"index":7,"sense_key":"take%2:38:10::","synset_id":"bring.v.01"},{"index":8,"sense_key":"take%2:40:00::","synset_id":"take.v.08"},{"index":9,"sense_key":"take%2:38:02::","synset_id":"take.v.09"},{"index":10,"sense_key":"take%2:31:01::","synset_id":"choose.v.01"},{"index":11,"sense_key":"take%2:40:05::","synset_id":"accept.v.02"},{"index":12,"sense_key":"take%2:41:00::","synset_id":"fill.v.04"},{"index":13,"sense_key":"take%2:31:03::","synset_id":"consider.v.03"},{"index":14,"sense_key":"take%2:42:00::","synset_id":"necessitate.v.01"},{"index":15,"sense_key":"take%2:39:00::","synset_id":"take.v.15"},{"index":16,"sense_key":"take%2:32:02::","synset_id":"film.v.01"},{"index":17,"sense_key":"take%2:30:00::","synset_id":"remove.v.01"},{"index":18,"sense_key":"take%2:34:00::","synset_id":"consume.v.02"},{"index":19,"sense_key":"take%2:31:09::","synset_id":"take.v.19"},{"index":20,"sense_key":"take%2:40:15::","synset_id":"take.v.20"},{"index":21,"sense_key":"take%2:40:01::","synset_id":"take.v.21"},{"index":22,"sense_key":"take%2:38:00::","synset_id":"assume.v.05"},{"index":23,"sense_key":"take%2:40:07::","synset_id":"accept.v.05"},{"index":24,"sense_key":"take%2:30:08::","synset_id":"take.v.24"},{"index":25,"sense_key":"take%2:31:02::","synset_id":"learn.v.04"},{"index":26,"sense_key":"take%2:32:00::","synset_id":"claim.v.05"},{"index":27,"sense_key":"take%2:38:05::","synset_id":"take.v.27"},{"index":28,"sense_key":"take%2:33:09::","synset_id":"aim.v.01"},{"index":29,"sense_key":"take%2:30:09::","synset_id":"take.v.29"},{"index":30,"sense_key":"take%2:42:10::","synset_id":"carry.v.02"},{"index":31,"sense_key":"take%2:40:03::","synset_id":"lease.v.04"},{"index":32,"sense_key":"take%2:40:04::","synset_id":"subscribe.v.05"},{"index":33,"sense_key":"take%2:40:02::","synset_id":"take.v.33"},{"index":34,"sense_key":"take%2:40:08::","synset_id":"take.v.34"},{"index":35,"sense_key":"take%2:35:14::","synset_id":"take.v.35"},{"index":36,"sense_key":"take%2:32:09::","synset_id":"claim.v.04"},{"index":37,"sense_key":"take%2:42:03::","synset_id":"accept.v.08"},{"index":38,"sense_key":"take%2:42:15::","synset_id":"contain.v.05"},{"index":39,"sense_key":"take%2:41:13::","synset_id":"take.v.39"},{"index":40,"sense_key":"take%2:38:11::","synset_id":"drive.v.16"},{"index":41,"sense_key":"take%2:33:08::","synset_id":"take.v.41"},{"index":42,"sense_key":"take%2:29:08::","synset_id":"contract.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":17,"chosen_sense_key":"take%2:30:00::","call_ids":["senseval2.d001.s083.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":17,"predicted_sense_key":"take%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":3134,"cached_input_tokens":0,"output_tokens":154,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.007808000000000001,"input_uncached_usd":0.006268,"input_cached_usd":0.0,"output_usd":0.0015400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.099701333907433}
{"item_id":"senseval2.d001.s083.t010","gold_sense_keys":["brain%1:08:00::"],"candidates":[{"index":1,"sense_key":"brain%1:08:00::","synset_id":"brain.n.01"},{"index":2,"sense_key":"brain%1:09:00::","synset_id":"brain.n.02"},{"index":3,"sense_key":"brain%1:09:01::","synset_id":"mind.n.01"},{"index":4,"sense_key":"brain%1:18:00::","synset_id":"genius.n.01"},{"index":5,"sense_key":"brain%1:13:00::","synset_id":"brain.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"brain%1:08:00::","call_ids":["senseval2.d001.s083.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"brain%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":891,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001902,"input_uncached_usd":0.001782,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9545242089079693}
{"item_id":"senseval2.d001.s084.t000","gold_sense_keys":["at_the_same_time%4:02:05::"],"candidates":[{"index":1,"sense_key":"at_the_same_time%4:02:03::","synset_id":"simultaneously.r.01"},{"index":2,"sense_key":"at_the_same_time%4:02:05::","synset_id":"concurrently.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"at_the_same_time%4:02:05::","call_ids":["senseval2.d001.s084.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"at_the_same_time%4:02:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":689,"cached_input_tokens":0,"output_tokens":80,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002178,"input_uncached_usd":0.0013779999999999999,"input_cached_usd":0.0,"output_usd":0.0008,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.613904625060968}
{"item_id":"senseval2.d001.s084.t001","gold_sense_keys":["team%1:14:00::"],"candidates":[{"index":1,"sense_key":"team%1:14:00::","synset_id":"team.n.01"},{"index":2,"sense_key":"team%1:14:01::","synset_id":"team.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"team%1:14:00::","call_ids":["senseval2.d001.s084.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"team%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":616,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0013520000000000001,"input_uncached_usd":0.001232,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5590548330219463}
{"item_id":"senseval2.d001.s084.t002","gold_sense_keys":["rush%2:30:00::"],"candidates":[{"index":1,"sense_key":"rush%2:38:00::","synset_id":"rush.v.01"},{"index":2,"sense_key":"rush%2:33:00::","synset_id":"rush.v.02"},{"index":3,"sense_key":"rush%2:30:01::","synset_id":"rush.v.03"},{"index":4,"sense_key":"rush%2:30:00::","synset_id":"rush.v.04"},{"index":5,"sense_key":"rush%2:38:01::","synset_id":"rush.v.05"},{"index":6,"sense_key":"rush%2:38:10::","synset_id":"race.v.04"},{"index":7,"sense_key":"rush%2:36:00::","synset_id":"induce.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"rush%2:30:00::","call_ids":["senseval2.d001.s084.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"rush%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":933,"cached_input_tokens":0,"output_tokens":133,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003196,"input_uncached_usd":0.0018659999999999998,"input_cached_usd":0.0,"output_usd":0.00133,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9175502499565482}
{"item_id":"senseval2.d001.s084.t003","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":2,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":3,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"other%3:00:00::","call_ids":["senseval2.d001.s084.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":773,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001666,"input_uncached_usd":0.0015459999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8741473330883309}
{"item_id":"senseval2.d001.s085.t000","gold_sense_keys":["hope%2:31:00::"],"candidates":[{"index":1,"sense_key":"hope%2:37:01::","synset_id":"hope.v.01"},{"index":2,"sense_key":"hope%2:37:00::","synset_id":"hope.v.02"},{"index":3,"sense_key":"hope%2:31:00::","synset_id":"hope.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"hope%2:31:00::","call_ids":["senseval2.d001.s085.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"hope%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":729,"cached_input_tokens":0,"output_tokens":78,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002238,"input_uncached_usd":0.0014579999999999999,"input_cached_usd":0.0,"output_usd":0.0007800000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.937885707942769}
{"item_id":"senseval2.d001.s085.t004","gold_sense_keys":["involved%3:00:00::"],"candidates":[{"index":1,"sense_key":"involved%3:00:00::","synset_id":"involved.a.01"},{"index":2,"sense_key":"involved%5:00:00:encumbered:00","synset_id":"involved.s.01"},{"index":3,"sense_key":"involved%5:00:00:attached:01","synset_id":"involved.s.02"},{"index":4,"sense_key":"involved%5:00:00:complex:00","synset_id":"byzantine.s.01"},{"index":5,"sense_key":"involved%5:00:00:enclosed:00","synset_id":"involved.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"involved%3:00:00::","call_ids":["senseval2.d001.s085.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"involved%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":901,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001922,"input_uncached_usd":0.001802,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9907865830464289}
{"item_id":"senseval2.d001.s086.t001","gold_sense_keys":["close%3:00:01::","close%3:00:05::"],"candidates":[{"index":1,"sense_key":"close%3:00:01::","synset_id":"close.a.01"},{"index":2,"sense_key":"close%3:00:02::","synset_id":"close.a.02"},{"index":3,"sense_key":"close%3:00:05::","synset_id":"near.a.01"},{"index":4,"sense_key":"close%5:00:00:careful:00","synset_id":"close.s.01"},{"index":5,"sense_key":"close%5:00:00:accurate:00","synset_id":"close.s.02"},{"index":6,"sense_key":"close%5:00:00:equal:00","synset_id":"close.s.03"},{"index":7,"sense_key":"close%5:00:00:confined:00","synset_id":"close.s.04"},{"index":8,"sense_key":"close%5:00:00:unventilated:00","synset_id":"airless.s.01"},{"index":9,"sense_key":"close%5:00:00:fine:00","synset_id":"close.s.06"},{"index":10,"sense_key":"close%5:00:00:restrained:00","synset_id":"close.s.07"},{"index":11,"sense_key":"close%5:00:00:private:00","synset_id":"close.s.08"},{"index":12,"sense_key":"close%5:00:00:tight:01","synset_id":"close.s.09"},{"index":13,"sense_key":"close%5:00:00:short:01","synset_id":"close.s.10"},{"index":14,"sense_key":"close%5:00:00:stingy:00","synset_id":"cheeseparing.s.01"},{"index":15,"sense_key":"close%5:00:00:uncommunicative:00","synset_id":"close.s.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"close%3:00:01::","call_ids":["senseval2.d001.s086.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"close%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1421,"cached_input_tokens":0,"output_tokens":145,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004292,"input_uncached_usd":0.002842,"input_cached_usd":0.0,"output_usd":0.0014500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8544441249687225}
{"item_id":"senseval2.d001.s086.t002","gold_sense_keys":["find%2:31:09::","find%2:32:00::","find%2:36:00::","find%2:39:02::"],"candidates":[{"index":1,"sense_key":"find%2:40:02::","synset_id":"find.v.01"},{"index":2,"sense_key":"find%2:39:02::","synset_id":"detect.v.01"},{"index":3,"sense_key":"find%2:40:00::","synset_id":"find.v.03"},{"index":4,"sense_key":"find%2:32:00::","synset_id":"determine.v.01"},{"index":5,"sense_key":"find%2:31:10::","synset_id":"find.v.05"},{"index":6,"sense_key":"find%2:39:01::","synset_id":"witness.v.02"},{"index":7,"sense_key":"find%2:40:01::","synset_id":"line_up.v.02"},{"index":8,"sense_key":"find%2:36:00::","synset_id":"discover.v.03"},{"index":9,"sense_key":"find%2:31:09::","synset_id":"discover.v.04"},{"index":10,"sense_key":"find%2:40:03::","synset_id":"find.v.10"},{"index":11,"sense_key":"find%2:32:01::","synset_id":"rule.v.04"},{"index":12,"sense_key":"find%2:30:00::","synset_id":"receive.v.02"},{"index":13,"sense_key":"find%2:39:05::","synset_id":"find.v.13"},{"index":14,"sense_key":"find%2:40:15::","synset_id":"recover.v.01"},{"index":15,"sense_key":"find%2:38:10::","synset_id":"find.v.15"},{"index":16,"sense_key":"find%2:30:13::","synset_id":"find_oneself.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"find%2:36:00::","call_ids":["senseval2.d001.s086.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"find%2:36:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1688,"cached_input_tokens":0,"output_tokens":155,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004926,"input_uncached_usd":0.0033759999999999997,"input_cached_usd":0.0,"output_usd":0.0015500000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8785264160251245}
{"item_id":"senseval2.d001.s086.t004","gold_sense_keys":["involved%3:00:00::"],"candidates":[{"index":1,"sense_key":"involved%3:00:00::","synset_id":"involved.a.01"},{"index":2,"sense_key":"involved%5:00:00:encumbered:00","synset_id":"involved.s.01"},{"index":3,"sense_key":"involved%5:00:00:attached:01","synset_id":"involved.s.02"},{"index":4,"sense_key":"involved%5:00:00:complex:00","synset_id":"byzantine.s.01"},{"index":5,"sense_key":"involved%5:00:00:enclosed:00","synset_id":"involved.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"involved%3:00:00::","call_ids":["senseval2.d001.s086.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"involved%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":900,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00192,"input_uncached_usd":0.0018,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8647608750034124}
{"item_id":"senseval2.d001.s086.t005","gold_sense_keys":["type%1:09:00::"],"candidates":[{"index":1,"sense_key":"type%1:09:00::","synset_id":"type.n.01"},{"index":2,"sense_key":"type%1:18:00::","synset_id":"character.n.05"},{"index":3,"sense_key":"type%1:14:01::","synset_id":"type.n.03"},{"index":4,"sense_key":"type%1:10:01::","synset_id":"type.n.04"},{"index":5,"sense_key":"type%1:10:00::","synset_id":"type.n.05"},{"index":6,"sense_key":"type%1:06:00::","synset_id":"type.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"type%1:09:00::","call_ids":["senseval2.d001.s086.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"type%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":862,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001844,"input_uncached_usd":0.0017239999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0363646249752492}
{"item_id":"senseval2.d001.s087.t000","gold_sense_keys":["believe%2:31:04::"],"candidates":[{"index":1,"sense_key":"believe%2:31:00::","synset_id":"believe.v.01"},{"index":2,"sense_key":"believe%2:31:04::","synset_id":"think.v.01"},{"index":3,"sense_key":"believe%2:31:03::","synset_id":"believe.v.03"},{"index":4,"sense_key":"believe%2:31:01::","synset_id":"believe.v.04"},{"index":5,"sense_key":"believe%2:31:02::","synset_id":"believe.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"believe%2:31:04::","call_ids":["senseval2.d001.s087.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"believe%2:31:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":858,"cached_input_tokens":0,"output_tokens":341,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.005126,"input_uncached_usd":0.0017159999999999999,"input_cached_usd":0.0,"output_usd":0.0034100000000000003,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.461200874997303}
{"item_id":"senseval2.d001.s087.t001","gold_sense_keys":["people%1:14:00::"],"candidates":[{"index":1,"sense_key":"people%1:14:00::","synset_id":"people.n.01"},{"index":2,"sense_key":"people%1:14:01::","synset_id":"citizenry.n.01"},{"index":3,"sense_key":"people%1:14:02::","synset_id":"people.n.03"},{"index":4,"sense_key":"people%1:14:03::","synset_id":"multitude.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"people%1:14:00::","call_ids":["senseval2.d001.s087.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"people%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":801,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001722,"input_uncached_usd":0.001602,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0900757499039173}
{"item_id":"senseval2.d001.s087.t002","gold_sense_keys":["inherit%2:40:01::"],"candidates":[{"index":1,"sense_key":"inherit%2:40:00::","synset_id":"inherit.v.01"},{"index":2,"sense_key":"inherit%2:40:02::","synset_id":"inherit.v.02"},{"index":3,"sense_key":"inherit%2:40:01::","synset_id":"inherit.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"inherit%2:40:01::","call_ids":["senseval2.d001.s087.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"inherit%2:40:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":723,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0015660000000000001,"input_uncached_usd":0.001446,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.095151458051987}
{"item_id":"senseval2.d001.s087.t006","gold_sense_keys":["copy%1:06:00::"],"candidates":[{"index":1,"sense_key":"copy%1:10:00::","synset_id":"transcript.n.02"},{"index":2,"sense_key":"copy%1:06:00::","synset_id":"copy.n.02"},{"index":3,"sense_key":"copy%1:10:01::","synset_id":"copy.n.03"},{"index":4,"sense_key":"copy%1:10:02::","synset_id":"copy.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"copy%1:06:00::","call_ids":["senseval2.d001.s087.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"copy%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":782,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001684,"input_uncached_usd":0.0015639999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0473705409094691}
{"item_id":"senseval2.d001.s087.t008","gold_sense_keys":["especially%4:02:00::"],"candidates":[{"index":1,"sense_key":"especially%4:02:00::","synset_id":"particularly.r.01"},{"index":2,"sense_key":"especially%4:02:01::","synset_id":"specially.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"especially%4:02:00::","call_ids":["senseval2.d001.s087.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"especially%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":700,"cached_input_tokens":0,"output_tokens":71,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00211,"input_uncached_usd":0.0014,"input_cached_usd":0.0,"output_usd":0.00071,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7437829159898683}
{"item_id":"senseval2.d001.s088.t001","gold_sense_keys":["report%2:32:04::"],"candidates":[{"index":1,"sense_key":"report%2:32:00::","synset_id":"report.v.01"},{"index":2,"sense_key":"report%2:32:04::","synset_id":"report.v.02"},{"index":3,"sense_key":"report%2:32:03::","synset_id":"report.v.03"},{"index":4,"sense_key":"report%2:32:01::","synset_id":"report.v.04"},{"index":5,"sense_key":"report%2:32:13::","synset_id":"report.v.05"},{"index":6,"sense_key":"report%2:32:02::","synset_id":"report.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"report%2:32:04::","call_ids":["senseval2.d001.s088.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"report%2:32:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":878,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001876,"input_uncached_usd":0.001756,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0438361660344526}
{"item_id":"senseval2.d001.s088.t005","gold_sense_keys":["involved%3:00:00::"],"candidates":[{"index":1,"sense_key":"involved%3:00:00::","synset_id":"involved.a.01"},{"index":2,"sense_key":"involved%5:00:00:encumbered:00","synset_id":"involved.s.01"},{"index":3,"sense_key":"involved%5:00:00:attached:01","synset_id":"involved.s.02"},{"index":4,"sense_key":"involved%5:00:00:complex:00","synset_id":"byzantine.s.01"},{"index":5,"sense_key":"involved%5:00:00:enclosed:00","synset_id":"involved.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"involved%3:00:00::","call_ids":["senseval2.d001.s088.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"involved%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":897,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0019140000000000001,"input_uncached_usd":0.001794,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9413726250641048}
{"item_id":"senseval2.d001.s088.t008","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":2,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":3,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"other%3:00:00::","call_ids":["senseval2.d001.s088.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":768,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0016560000000000001,"input_uncached_usd":0.001536,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3923669169889763}
{"item_id":"senseval2.d001.s088.t009","gold_sense_keys":["more%4:02:00::"],"candidates":[{"index":1,"sense_key":"more%4:02:00::","synset_id":"more.r.01"},{"index":2,"sense_key":"more%4:02:01::","synset_id":"more.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"more%4:02:00::","call_ids":["senseval2.d001.s088.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"more%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":650,"cached_input_tokens":0,"output_tokens":44,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00174,"input_uncached_usd":0.0013,"input_cached_usd":0.0,"output_usd":0.00044,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1945105000631884}
{"item_id":"senseval2.d001.s088.t010","gold_sense_keys":["common%3:00:01::","common%5:00:00:familiar:02"],"candidates":[{"index":1,"sense_key":"common%3:00:02::","synset_id":"common.a.01"},{"index":2,"sense_key":"common%3:00:01::","synset_id":"common.a.02"},{"index":3,"sense_key":"common%5:00:00:shared:00","synset_id":"common.s.01"},{"index":4,"sense_key":"common%5:00:00:familiar:02","synset_id":"common.s.02"},{"index":5,"sense_key":"common%5:00:00:informal:02","synset_id":"common.s.03"},{"index":6,"sense_key":"common%5:00:00:lowborn:00","synset_id":"common.s.04"},{"index":7,"sense_key":"common%5:00:00:inferior:02","synset_id":"coarse.s.02"},{"index":8,"sense_key":"common%5:00:00:unrefined:01","synset_id":"coarse.s.01"},{"index":9,"sense_key":"common%5:00:00:ordinary:00","synset_id":"common.s.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"common%5:00:00:familiar:02","call_ids":["senseval2.d001.s088.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"common%5:00:00:familiar:02","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1163,"cached_input_tokens":0,"output_tokens":80,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003126,"input_uncached_usd":0.002326,"input_cached_usd":0.0,"output_usd":0.0008,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.102612583898008}
{"item_id":"senseval2.d001.s088.t011","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s088.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":852,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001824,"input_uncached_usd":0.001704,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1645752500044182}
{"item_id":"senseval2.d001.s089.t000","gold_sense_keys":["discovery%1:04:00::","discovery%1:09:00::","discovery%1:10:00::"],"candidates":[{"index":1,"sense_key":"discovery%1:04:00::","synset_id":"discovery.n.01"},{"index":2,"sense_key":"discovery%1:10:00::","synset_id":"discovery.n.02"},{"index":3,"sense_key":"discovery%1:09:00::","synset_id":"discovery.n.03"},{"index":4,"sense_key":"discovery%1:10:01::","synset_id":"discovery.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"discovery%1:10:00::","call_ids":["senseval2.d001.s089.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"discovery%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":776,"cached_input_tokens":0,"output_tokens":150,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003052,"input_uncached_usd":0.001552,"input_cached_usd":0.0,"output_usd":0.0015,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.081779667059891}
{"item_id":"senseval2.d001.s089.t003","gold_sense_keys":["only%4:02:02::"],"candidates":[{"index":1,"sense_key":"only%4:02:02::","synset_id":"merely.r.01"},{"index":2,"sense_key":"only%4:02:01::","synset_id":"entirely.r.02"},{"index":3,"sense_key":"only%4:02:00::","synset_id":"only.r.03"},{"index":4,"sense_key":"only%4:02:04::","synset_id":"only.r.04"},{"index":5,"sense_key":"only%4:02:06::","synset_id":"only.r.05"},{"index":6,"sense_key":"only%4:02:03::","synset_id":"only.r.06"},{"index":7,"sense_key":"only%4:02:05::","synset_id":"only.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"only%4:02:02::","call_ids":["senseval2.d001.s089.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"only%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":951,"cached_input_tokens":0,"output_tokens":86,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0027619999999999997,"input_uncached_usd":0.0019019999999999998,"input_cached_usd":0.0,"output_usd":0.0008600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.582224415964447}
{"item_id":"senseval2.d001.s089.t004","gold_sense_keys":["speculate%2:31:01::","speculate%2:32:00::"],"candidates":[{"index":1,"sense_key":"speculate%2:31:01::","synset_id":"speculate.v.01"},{"index":2,"sense_key":"speculate%2:32:00::","synset_id":"speculate.v.02"},{"index":3,"sense_key":"speculate%2:31:00::","synset_id":"chew_over.v.01"},{"index":4,"sense_key":"speculate%2:40:00::","synset_id":"speculate.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"speculate%2:32:00::","call_ids":["senseval2.d001.s089.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"speculate%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":947,"cached_input_tokens":0,"output_tokens":231,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004203999999999999,"input_uncached_usd":0.0018939999999999999,"input_cached_usd":0.0,"output_usd":0.00231,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.5676124589517713}
{"item_id":"senseval2.d001.s090.t001","gold_sense_keys":["major%3:00:06::"],"candidates":[{"index":1,"sense_key":"major%3:00:06::","synset_id":"major.a.01"},{"index":2,"sense_key":"major%3:00:02::","synset_id":"major.a.02"},{"index":3,"sense_key":"major%3:00:01::","synset_id":"major.a.03"},{"index":4,"sense_key":"major%3:00:05::","synset_id":"major.a.04"},{"index":5,"sense_key":"major%3:00:04::","synset_id":"major.a.05"},{"index":6,"sense_key":"major%3:00:07::","synset_id":"major.a.06"},{"index":7,"sense_key":"major%3:00:03::","synset_id":"major.a.07"},{"index":8,"sense_key":"major%5:00:00:senior:00","synset_id":"major.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"major%3:00:06::","call_ids":["senseval2.d001.s090.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"major%3:00:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":944,"cached_input_tokens":0,"output_tokens":84,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002728,"input_uncached_usd":0.001888,"input_cached_usd":0.0,"output_usd":0.00084,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3610929580172524}
{"item_id":"senseval2.d001.s090.t004","gold_sense_keys":["unit%1:14:00::"],"candidates":[{"index":1,"sense_key":"unit%1:23:00::","synset_id":"unit_of_measurement.n.01"},{"index":2,"sense_key":"unit%1:24:00::","synset_id":"unit.n.02"},{"index":3,"sense_key":"unit%1:14:00::","synset_id":"unit.n.03"},{"index":4,"sense_key":"unit%1:09:00::","synset_id":"unit.n.04"},{"index":5,"sense_key":"unit%1:17:00::","synset_id":"unit.n.05"},{"index":6,"sense_key":"unit%1:03:00::","synset_id":"whole.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"unit%1:14:00::","call_ids":["senseval2.d001.s090.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"unit%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1024,"cached_input_tokens":0,"output_tokens":84,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002888,"input_uncached_usd":0.002048,"input_cached_usd":0.0,"output_usd":0.00084,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4700297499075532}
{"item_id":"senseval2.d001.s090.t005","gold_sense_keys":["collaborate%2:41:00::"],"candidates":[{"index":1,"sense_key":"collaborate%2:41:00::","synset_id":"collaborate.v.01"},{"index":2,"sense_key":"collaborate%2:41:01::","synset_id":"collaborate.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"collaborate%2:41:00::","call_ids":["senseval2.d001.s090.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"collaborate%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":734,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001588,"input_uncached_usd":0.001468,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1977093749446794}
{"item_id":"senseval2.d001.s090.t007","gold_sense_keys":["hunter%1:18:01::"],"candidates":[{"index":1,"sense_key":"hunter%1:18:00::","synset_id":"hunter.n.01"},{"index":2,"sense_key":"hunter%1:18:01::","synset_id":"hunter.n.02"},{"index":3,"sense_key":"hunter%1:17:00::","synset_id":"orion.n.02"},{"index":4,"sense_key":"hunter%1:06:00::","synset_id":"hunter.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"hunter%1:18:01::","call_ids":["senseval2.d001.s090.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"hunter%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":792,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001704,"input_uncached_usd":0.001584,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.170897166011855}
{"item_id":"senseval2.d001.s090.t008","gold_sense_keys":["turn%2:42:13::"],"candidates":[{"index":1,"sense_key":"turn%2:38:00::","synset_id":"turn.v.01"},{"index":2,"sense_key":"turn%2:30:04::","synset_id":"change_state.v.01"},{"index":3,"sense_key":"turn%2:42:00::","synset_id":"become.v.02"},{"index":4,"sense_key":"turn%2:38:02::","synset_id":"turn.v.04"},{"index":5,"sense_key":"turn%2:30:00::","synset_id":"change_by_reversal.v.01"},{"index":6,"sense_key":"turn%2:38:03::","synset_id":"turn.v.06"},{"index":7,"sense_key":"turn%2:30:03::","synset_id":"turn.v.07"},{"index":8,"sense_key":"turn%2:35:08::","synset_id":"turn.v.08"},{"index":9,"sense_key":"turn%2:38:04::","synset_id":"turn.v.09"},{"index":10,"sense_key":"turn%2:38:01::","synset_id":"turn.v.10"},{"index":11,"sense_key":"turn%2:38:06::","synset_id":"turn.v.11"},{"index":12,"sense_key":"turn%2:36:00::","synset_id":"plow.v.01"},{"index":13,"sense_key":"turn%2:35:06::","synset_id":"turn.v.13"},{"index":14,"sense_key":"turn%2:30:09::","synset_id":"turn.v.14"},{"index":15,"sense_key":"turn%2:29:00::","synset_id":"twist.v.10"},{"index":16,"sense_key":"turn%2:42:13::","synset_id":"turn.v.16"},{"index":17,"sense_key":"turn%2:41:10::","synset_id":"turn.v.17"},{"index":18,"sense_key":"turn%2:40:13::","synset_id":"turn.v.18"},{"index":19,"sense_key":"turn%2:38:13::","synset_id":"turn.v.19"},{"index":20,"sense_key":"turn%2:38:14::","synset_id":"turn.v.20"},{"index":21,"sense_key":"turn%2:35:11::","synset_id":"flex.v.05"},{"index":22,"sense_key":"turn%2:35:10::","synset_id":"turn.v.22"},{"index":23,"sense_key":"turn%2:33:13::","synset_id":"turn.v.23"},{"index":24,"sense_key":"turn%2:32:14::","synset_id":"call_on.v.01"},{"index":25,"sense_key":"turn%2:30:01::","synset_id":"sour.v.01"},{"index":26,"sense_key":"turn%2:30:14::","synset_id":"turn.v.26"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":16,"chosen_sense_key":"turn%2:42:13::","call_ids":["senseval2.d001.s090.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":16,"predicted_sense_key":"turn%2:42:13::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2154,"cached_input_tokens":0,"output_tokens":89,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.005198,"input_uncached_usd":0.004308,"input_cached_usd":0.0,"output_usd":0.0008900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.070400416967459}
{"item_id":"senseval2.d001.s090.t010","gold_sense_keys":["discovery%1:10:00::"],"candidates":[{"index":1,"sense_key":"discovery%1:04:00::","synset_id":"discovery.n.01"},{"index":2,"sense_key":"discovery%1:10:00::","synset_id":"discovery.n.02"},{"index":3,"sense_key":"discovery%1:09:00::","synset_id":"discovery.n.03"},{"index":4,"sense_key":"discovery%1:10:01::","synset_id":"discovery.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"discovery%1:10:00::","call_ids":["senseval2.d001.s090.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"discovery%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":783,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002296,"input_uncached_usd":0.001566,"input_cached_usd":0.0,"output_usd":0.0007300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4639743330189958}
{"item_id":"senseval2.d001.s090.t014","gold_sense_keys":["new%3:00:00::","new%5:00:00:original:00","new%5:00:00:other:00","new%5:00:00:unprecedented:00"],"candidates":[{"index":1,"sense_key":"new%3:00:00::","synset_id":"new.a.01"},{"index":2,"sense_key":"new%5:00:00:original:00","synset_id":"fresh.s.03"},{"index":3,"sense_key":"new%5:00:00:inexperienced:00","synset_id":"raw.s.11"},{"index":4,"sense_key":"new%5:00:00:unprecedented:00","synset_id":"new.s.03"},{"index":5,"sense_key":"new%5:00:00:other:00","synset_id":"new.s.04"},{"index":6,"sense_key":"new%3:00:09::","synset_id":"new.a.06"},{"index":7,"sense_key":"new%5:00:02:original:00","synset_id":"newfangled.s.01"},{"index":8,"sense_key":"new%5:00:02:late:01","synset_id":"new.s.06"},{"index":9,"sense_key":"new%5:00:01:late:01","synset_id":"modern.s.04"},{"index":10,"sense_key":"new%5:00:00:early:02","synset_id":"new.s.08"},{"index":11,"sense_key":"new%5:00:00:unaccustomed:00","synset_id":"new.s.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"new%3:00:00::","call_ids":["senseval2.d001.s090.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"new%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1282,"cached_input_tokens":0,"output_tokens":140,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0039640000000000005,"input_uncached_usd":0.002564,"input_cached_usd":0.0,"output_usd":0.0014000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.009602249949239}
{"item_id":"senseval2.d001.s091.t001","gold_sense_keys":["say%2:32:01::","say%2:32:13::"],"candidates":[{"index":1,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":6,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":9,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":10,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":11,"sense_key":"say%2:32:09::","synset_id":"say.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"say%2:32:01::","call_ids":["senseval2.d001.s091.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"say%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1248,"cached_input_tokens":0,"output_tokens":149,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003986,"input_uncached_usd":0.002496,"input_cached_usd":0.0,"output_usd":0.0014900000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8447187079582363}
{"item_id":"senseval2.d001.s091.t002","gold_sense_keys":["new%3:00:00::","new%3:00:09::","new%5:00:00:original:00","new%5:00:00:unprecedented:00"],"candidates":[{"index":1,"sense_key":"new%3:00:00::","synset_id":"new.a.01"},{"index":2,"sense_key":"new%5:00:00:original:00","synset_id":"fresh.s.03"},{"index":3,"sense_key":"new%5:00:00:inexperienced:00","synset_id":"raw.s.11"},{"index":4,"sense_key":"new%5:00:00:unprecedented:00","synset_id":"new.s.03"},{"index":5,"sense_key":"new%5:00:00:other:00","synset_id":"new.s.04"},{"index":6,"sense_key":"new%3:00:09::","synset_id":"new.a.06"},{"index":7,"sense_key":"new%5:00:02:original:00","synset_id":"newfangled.s.01"},{"index":8,"sense_key":"new%5:00:02:late:01","synset_id":"new.s.06"},{"index":9,"sense_key":"new%5:00:01:late:01","synset_id":"modern.s.04"},{"index":10,"sense_key":"new%5:00:00:early:02","synset_id":"new.s.08"},{"index":11,"sense_key":"new%5:00:00:unaccustomed:00","synset_id":"new.s.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"new%3:00:00::","call_ids":["senseval2.d001.s091.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"new%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1280,"cached_input_tokens":0,"output_tokens":119,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00375,"input_uncached_usd":0.0025599999999999998,"input_cached_usd":0.0,"output_usd":0.00119,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9203072499949485}
{"item_id":"senseval2.d001.s091.t005","gold_sense_keys":["slow%2:30:05::"],"candidates":[{"index":1,"sense_key":"slow%2:30:00::","synset_id":"decelerate.v.01"},{"index":2,"sense_key":"slow%2:30:01::","synset_id":"slow.v.02"},{"index":3,"sense_key":"slow%2:30:05::","synset_id":"slow.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"slow%2:30:05::","call_ids":["senseval2.d001.s091.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"slow%2:30:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":769,"cached_input_tokens":0,"output_tokens":81,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002348,"input_uncached_usd":0.0015379999999999999,"input_cached_usd":0.0,"output_usd":0.0008100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2519628750160336}
{"item_id":"senseval2.d001.s091.t006","gold_sense_keys":["reverse%2:30:00::"],"candidates":[{"index":1,"sense_key":"reverse%2:30:00::","synset_id":"change_by_reversal.v.01"},{"index":2,"sense_key":"reverse%2:30:01::","synset_id":"turn_back.v.05"},{"index":3,"sense_key":"reverse%2:31:00::","synset_id":"overrule.v.01"},{"index":4,"sense_key":"reverse%2:32:00::","synset_id":"revoke.v.02"},{"index":5,"sense_key":"reverse%2:30:03::","synset_id":"invert.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"reverse%2:30:00::","call_ids":["senseval2.d001.s091.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"reverse%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":909,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001938,"input_uncached_usd":0.001818,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.601583915995434}
{"item_id":"senseval2.d001.s091.t008","gold_sense_keys":["growth%1:22:00::","growth%1:26:00::"],"candidates":[{"index":1,"sense_key":"growth%1:22:00::","synset_id":"growth.n.01"},{"index":2,"sense_key":"growth%1:22:01::","synset_id":"growth.n.02"},{"index":3,"sense_key":"growth%1:22:02::","synset_id":"increase.n.03"},{"index":4,"sense_key":"growth%1:14:00::","synset_id":"growth.n.04"},{"index":5,"sense_key":"growth%1:11:01::","synset_id":"emergence.n.01"},{"index":6,"sense_key":"growth%1:26:00::","synset_id":"growth.n.06"},{"index":7,"sense_key":"growth%1:17:00::","synset_id":"growth.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"growth%1:22:02::","call_ids":["senseval2.d001.s091.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"growth%1:22:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1031,"cached_input_tokens":0,"output_tokens":116,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003222,"input_uncached_usd":0.002062,"input_cached_usd":0.0,"output_usd":0.00116,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8785801250487566}
{"item_id":"senseval2.d001.s091.t009","gold_sense_keys":["base%2:31:00::"],"candidates":[{"index":1,"sense_key":"base%2:31:00::","synset_id":"establish.v.08"},{"index":2,"sense_key":"base%2:42:04::","synset_id":"base.v.02"},{"index":3,"sense_key":"base%2:34:00::","synset_id":"free-base.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"base%2:31:00::","call_ids":["senseval2.d001.s091.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"base%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":739,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001598,"input_uncached_usd":0.001478,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8608052920317277}
{"item_id":"senseval2.d001.s091.t012","gold_sense_keys":["produce%2:36:02::","produce%2:36:03::"],"candidates":[{"index":1,"sense_key":"produce%2:36:02::","synset_id":"produce.v.01"},{"index":2,"sense_key":"produce%2:36:00::","synset_id":"produce.v.02"},{"index":3,"sense_key":"produce%2:36:03::","synset_id":"produce.v.03"},{"index":4,"sense_key":"produce%2:39:01::","synset_id":"produce.v.04"},{"index":5,"sense_key":"produce%2:36:05::","synset_id":"grow.v.07"},{"index":6,"sense_key":"produce%2:39:00::","synset_id":"produce.v.06"},{"index":7,"sense_key":"produce%2:29:00::","synset_id":"grow.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"produce%2:36:02::","call_ids":["senseval2.d001.s091.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"produce%2:36:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1092,"cached_input_tokens":0,"output_tokens":117,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0033539999999999998,"input_uncached_usd":0.0021839999999999997,"input_cached_usd":0.0,"output_usd":0.00117,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.868816833011806}
{"item_id":"senseval2.d001.s092.t000","gold_sense_keys":["idea%1:09:01::"],"candidates":[{"index":1,"sense_key":"idea%1:09:00::","synset_id":"idea.n.01"},{"index":2,"sense_key":"idea%1:09:01::","synset_id":"mind.n.06"},{"index":3,"sense_key":"idea%1:09:03::","synset_id":"idea.n.03"},{"index":4,"sense_key":"idea%1:09:02::","synset_id":"estimate.n.01"},{"index":5,"sense_key":"idea%1:10:00::","synset_id":"theme.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"idea%1:09:01::","call_ids":["senseval2.d001.s092.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"idea%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":913,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002546,"input_uncached_usd":0.001826,"input_cached_usd":0.0,"output_usd":0.00072,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.493974333978258}
{"item_id":"senseval2.d001.s092.t001","gold_sense_keys":["administer%2:29:00::"],"candidates":[{"index":1,"sense_key":"administer%2:41:00::","synset_id":"administer.v.01"},{"index":2,"sense_key":"administer%2:40:01::","synset_id":"administer.v.02"},{"index":3,"sense_key":"administer%2:40:00::","synset_id":"distribute.v.01"},{"index":4,"sense_key":"administer%2:29:00::","synset_id":"administer.v.04"},{"index":5,"sense_key":"administer%2:41:01::","synset_id":"administer.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"administer%2:29:00::","call_ids":["senseval2.d001.s092.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"administer%2:29:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":839,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0017980000000000001,"input_uncached_usd":0.001678,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0574623750289902}
{"item_id":"senseval2.d001.s092.t002","gold_sense_keys":["patient%1:18:00::"],"candidates":[{"index":1,"sense_key":"patient%1:18:00::","synset_id":"patient.n.01"},{"index":2,"sense_key":"patient%1:10:00::","synset_id":"affected_role.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"patient%1:18:00::","call_ids":["senseval2.d001.s092.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"patient%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":687,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001494,"input_uncached_usd":0.001374,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1086098329396918}
{"item_id":"senseval2.d001.s092.t005","gold_sense_keys":["healthy%3:00:00::"],"candidates":[{"index":1,"sense_key":"healthy%3:00:00::","synset_id":"healthy.a.01"},{"index":2,"sense_key":"healthy%5:00:00:sound:00","synset_id":"healthy.s.01"},{"index":3,"sense_key":"healthy%5:00:00:wholesome:00","synset_id":"healthy.s.02"},{"index":4,"sense_key":"healthy%5:00:00:reasonable:00","synset_id":"healthy.s.03"},{"index":5,"sense_key":"healthy%5:00:00:considerable:00","synset_id":"goodly.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"healthy%3:00:00::","call_ids":["senseval2.d001.s092.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"healthy%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":935,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00199,"input_uncached_usd":0.00187,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4987909579649568}
{"item_id":"senseval2.d001.s092.t006","gold_sense_keys":["version%1:09:01::"],"candidates":[{"index":1,"sense_key":"version%1:10:01::","synset_id":"version.n.01"},{"index":2,"sense_key":"version%1:09:01::","synset_id":"version.n.02"},{"index":3,"sense_key":"version%1:10:02::","synset_id":"adaptation.n.01"},{"index":4,"sense_key":"version%1:10:00::","synset_id":"translation.n.01"},{"index":5,"sense_key":"version%1:09:00::","synset_id":"interpretation.n.01"},{"index":6,"sense_key":"version%1:04:00::","synset_id":"version.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"version%1:09:01::","call_ids":["senseval2.d001.s092.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"version%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":908,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002426,"input_uncached_usd":0.001816,"input_cached_usd":0.0,"output_usd":0.0006100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.028144208015874}
{"item_id":"senseval2.d001.s092.t007","gold_sense_keys":["damaged%3:00:00::"],"candidates":[{"index":1,"sense_key":"damaged%3:00:00::","synset_id":"damaged.a.01"},{"index":2,"sense_key":"damaged%5:00:00:disreputable:00","synset_id":"discredited.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"damaged%3:00:00::","call_ids":["senseval2.d001.s092.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"damaged%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":698,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001516,"input_uncached_usd":0.0013959999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6231527919881046}
{"item_id":"senseval2.d001.s093.t000","gold_sense_keys":["possible%3:00:00::"],"candidates":[{"index":1,"sense_key":"possible%3:00:00::","synset_id":"possible.a.01"},{"index":2,"sense_key":"possible%3:00:04::","synset_id":"potential.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"possible%3:00:00::","call_ids":["senseval2.d001.s093.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"possible%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":673,"cached_input_tokens":0,"output_tokens":103,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002376,"input_uncached_usd":0.001346,"input_cached_usd":0.0,"output_usd":0.00103,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3794137500226498}
{"item_id":"senseval2.d001.s093.t001","gold_sense_keys":["replace%2:30:00::"],"candidates":[{"index":1,"sense_key":"replace%2:30:00::","synset_id":"replace.v.01"},{"index":2,"sense_key":"replace%2:41:00::","synset_id":"supplant.v.01"},{"index":3,"sense_key":"replace%2:35:00::","synset_id":"replace.v.03"},{"index":4,"sense_key":"replace%2:40:00::","synset_id":"substitute.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"replace%2:30:00::","call_ids":["senseval2.d001.s093.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"replace%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":920,"cached_input_tokens":0,"output_tokens":164,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00348,"input_uncached_usd":0.0018399999999999998,"input_cached_usd":0.0,"output_usd":0.0016400000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1801368329906836}
{"item_id":"senseval2.d001.s093.t004","gold_sense_keys":["healthy%3:00:00::"],"candidates":[{"index":1,"sense_key":"healthy%3:00:00::","synset_id":"healthy.a.01"},{"index":2,"sense_key":"healthy%5:00:00:sound:00","synset_id":"healthy.s.01"},{"index":3,"sense_key":"healthy%5:00:00:wholesome:00","synset_id":"healthy.s.02"},{"index":4,"sense_key":"healthy%5:00:00:reasonable:00","synset_id":"healthy.s.03"},{"index":5,"sense_key":"healthy%5:00:00:considerable:00","synset_id":"goodly.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"healthy%3:00:00::","call_ids":["senseval2.d001.s093.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"healthy%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":937,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002554,"input_uncached_usd":0.001874,"input_cached_usd":0.0,"output_usd":0.00068,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7970240000868216}
{"item_id":"senseval2.d001.s093.t005","gold_sense_keys":["version%1:09:01::"],"candidates":[{"index":1,"sense_key":"version%1:10:01::","synset_id":"version.n.01"},{"index":2,"sense_key":"version%1:09:01::","synset_id":"version.n.02"},{"index":3,"sense_key":"version%1:10:02::","synset_id":"adaptation.n.01"},{"index":4,"sense_key":"version%1:10:00::","synset_id":"translation.n.01"},{"index":5,"sense_key":"version%1:09:00::","synset_id":"interpretation.n.01"},{"index":6,"sense_key":"version%1:04:00::","synset_id":"version.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"version%1:09:01::","call_ids":["senseval2.d001.s093.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"version%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":910,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00194,"input_uncached_usd":0.00182,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5476662078872323}
{"item_id":"senseval2.d001.s093.t006","gold_sense_keys":["come_close%2:41:00::"],"candidates":[{"index":1,"sense_key":"come_close%2:41:00::","synset_id":"come_close.v.01"},{"index":2,"sense_key":"come_close%2:42:00::","synset_id":"approximate.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"come_close%2:41:00::","call_ids":["senseval2.d001.s093.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"come_close%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":656,"cached_input_tokens":0,"output_tokens":93,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002242,"input_uncached_usd":0.001312,"input_cached_usd":0.0,"output_usd":0.00093,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.3365222919965163}
{"item_id":"senseval2.d001.s094.t001","gold_sense_keys":["say%2:32:00::","say%2:32:15::"],"candidates":[{"index":1,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":6,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":9,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":10,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":11,"sense_key":"say%2:32:09::","synset_id":"say.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"say%2:32:00::","call_ids":["senseval2.d001.s094.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"say%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1172,"cached_input_tokens":0,"output_tokens":213,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0044740000000000005,"input_uncached_usd":0.0023439999999999997,"input_cached_usd":0.0,"output_usd":0.0021300000000000004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.5365510830888525}
{"item_id":"senseval2.d001.s094.t002","gold_sense_keys":["witness%2:39:01::"],"candidates":[{"index":1,"sense_key":"witness%2:39:00::","synset_id":"witness.v.01"},{"index":2,"sense_key":"witness%2:39:01::","synset_id":"witness.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"witness%2:39:01::","call_ids":["senseval2.d001.s094.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"witness%2:39:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":636,"cached_input_tokens":0,"output_tokens":112,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002392,"input_uncached_usd":0.001272,"input_cached_usd":0.0,"output_usd":0.0011200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7781754169845954}
{"item_id":"senseval2.d001.s094.t003","gold_sense_keys":["discovery%1:04:00::"],"candidates":[{"index":1,"sense_key":"discovery%1:04:00::","synset_id":"discovery.n.01"},{"index":2,"sense_key":"discovery%1:10:00::","synset_id":"discovery.n.02"},{"index":3,"sense_key":"discovery%1:09:00::","synset_id":"discovery.n.03"},{"index":4,"sense_key":"discovery%1:10:01::","synset_id":"discovery.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"discovery%1:04:00::","call_ids":["senseval2.d001.s094.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"discovery%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":704,"cached_input_tokens":0,"output_tokens":133,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002738,"input_uncached_usd":0.001408,"input_cached_usd":0.0,"output_usd":0.00133,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7021085420856252}
{"item_id":"senseval2.d001.s094.t004","gold_sense_keys":["most%4:02:00::"],"candidates":[{"index":1,"sense_key":"most%4:02:00::","synset_id":"most.r.01"},{"index":2,"sense_key":"most%4:02:01::","synset_id":"most.r.02"},{"index":3,"sense_key":"most%4:02:02::","synset_id":"about.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"most%4:02:00::","call_ids":["senseval2.d001.s094.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"most%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":702,"cached_input_tokens":0,"output_tokens":152,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002924,"input_uncached_usd":0.0014039999999999999,"input_cached_usd":0.0,"output_usd":0.00152,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9265346670290455}
{"item_id":"senseval2.d001.s094.t005","gold_sense_keys":["important%3:00:00::","important%3:00:02::","important%3:00:04::"],"candidates":[{"index":1,"sense_key":"important%3:00:00::","synset_id":"important.a.01"},{"index":2,"sense_key":"important%3:00:04::","synset_id":"significant.a.01"},{"index":3,"sense_key":"important%3:00:02::","synset_id":"crucial.a.01"},{"index":4,"sense_key":"important%5:00:00:influential:00","synset_id":"authoritative.s.01"},{"index":5,"sense_key":"important%5:00:00:immodest:02","synset_id":"important.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"important%3:00:00::","call_ids":["senseval2.d001.s094.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"important%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":822,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001764,"input_uncached_usd":0.0016439999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0937467499170452}
{"item_id":"senseval2.d001.s094.t006","gold_sense_keys":["step%1:04:02::"],"candidates":[{"index":1,"sense_key":"step%1:04:02::","synset_id":"measure.n.01"},{"index":2,"sense_key":"step%1:23:00::","synset_id":"footstep.n.03"},{"index":3,"sense_key":"step%1:04:00::","synset_id":"step.n.03"},{"index":4,"sense_key":"step%1:06:00::","synset_id":"step.n.04"},{"index":5,"sense_key":"step%1:26:00::","synset_id":"gradation.n.01"},{"index":6,"sense_key":"step%1:23:01::","synset_id":"step.n.06"},{"index":7,"sense_key":"step%1:11:00::","synset_id":"footfall.n.01"},{"index":8,"sense_key":"step%1:10:00::","synset_id":"tone.n.09"},{"index":9,"sense_key":"step%1:10:02::","synset_id":"footprint.n.01"},{"index":10,"sense_key":"step%1:06:01::","synset_id":"step.n.10"},{"index":11,"sense_key":"step%1:04:03::","synset_id":"dance_step.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"step%1:04:02::","call_ids":["senseval2.d001.s094.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"step%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1168,"cached_input_tokens":0,"output_tokens":128,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0036160000000000003,"input_uncached_usd":0.002336,"input_cached_usd":0.0,"output_usd":0.00128,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.429306000005454}
{"item_id":"senseval2.d001.s094.t007","gold_sense_keys":["genesis%1:11:00::"],"candidates":[{"index":1,"sense_key":"genesis%1:11:00::","synset_id":"genesis.n.01"},{"index":2,"sense_key":"genesis%1:10:00::","synset_id":"genesis.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"genesis%1:11:00::","call_ids":["senseval2.d001.s094.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"genesis%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":678,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001476,"input_uncached_usd":0.001356,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2985907080583274}
{"item_id":"senseval2.d001.s094.t008","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s094.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":832,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001784,"input_uncached_usd":0.001664,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9686645000474527}
{"item_id":"senseval2.d002.s000.t000","gold_sense_keys":["teach%2:32:00::"],"candidates":[{"index":1,"sense_key":"teach%2:32:00::","synset_id":"teach.v.01"},{"index":2,"sense_key":"teach%2:30:00::","synset_id":"teach.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"teach%2:32:00::","call_ids":["senseval2.d002.s000.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"teach%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":379,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.000878,"input_uncached_usd":0.000758,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8306552090216428}
{"item_id":"senseval2.d002.s000.t001","gold_sense_keys":["child%1:18:00::"],"candidates":[{"index":1,"sense_key":"child%1:18:00::","synset_id":"child.n.01"},{"index":2,"sense_key":"child%1:18:01::","synset_id":"child.n.02"},{"index":3,"sense_key":"child%1:18:06::","synset_id":"child.n.03"},{"index":4,"sense_key":"child%1:18:07::","synset_id":"child.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"child%1:18:00::","call_ids":["senseval2.d002.s000.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"child%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":501,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001122,"input_uncached_usd":0.0010019999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4941356250783429}
{"item_id":"senseval2.d002.s000.t002","gold_sense_keys":["read%2:31:00::"],"candidates":[{"index":1,"sense_key":"read%2:31:00::","synset_id":"read.v.01"},{"index":2,"sense_key":"read%2:42:00::","synset_id":"read.v.02"},{"index":3,"sense_key":"read%2:31:01::","synset_id":"read.v.03"},{"index":4,"sense_key":"read%2:31:06::","synset_id":"read.v.04"},{"index":5,"sense_key":"read%2:31:02::","synset_id":"read.v.05"},{"index":6,"sense_key":"read%2:31:09::","synset_id":"take.v.06"},{"index":7,"sense_key":"read%2:31:05::","synset_id":"learn.v.04"},{"index":8,"sense_key":"read%2:32:02::","synset_id":"read.v.08"},{"index":9,"sense_key":"read%2:36:05::","synset_id":"read.v.09"},{"index":10,"sense_key":"read%2:31:08::","synset_id":"read.v.10"},{"index":11,"sense_key":"read%2:31:04::","synset_id":"understand.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"read%2:31:00::","call_ids":["senseval2.d002.s000.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"read%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":936,"cached_input_tokens":0,"output_tokens":169,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003562,"input_uncached_usd":0.001872,"input_cached_usd":0.0,"output_usd":0.00169,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.744326874963008}
{"item_id":"senseval2.d002.s000.t003","gold_sense_keys":["write%2:36:01::"],"candidates":[{"index":1,"sense_key":"write%2:36:00::","synset_id":"write.v.01"},{"index":2,"sense_key":"write%2:32:00::","synset_id":"write.v.02"},{"index":3,"sense_key":"write%2:36:02::","synset_id":"publish.v.03"},{"index":4,"sense_key":"write%2:32:08::","synset_id":"write.v.04"},{"index":5,"sense_key":"write%2:32:01::","synset_id":"write.v.05"},{"index":6,"sense_key":"write%2:36:04::","synset_id":"compose.v.02"},{"index":7,"sense_key":"write%2:36:01::","synset_id":"write.v.07"},{"index":8,"sense_key":"write%2:32:03::","synset_id":"write.v.08"},{"index":9,"sense_key":"write%2:36:03::","synset_id":"spell.v.03"},{"index":10,"sense_key":"write%2:36:10::","synset_id":"write.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"write%2:36:01::","call_ids":["senseval2.d002.s000.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"write%2:36:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":819,"cached_input_tokens":0,"output_tokens":195,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003588,"input_uncached_usd":0.001638,"input_cached_usd":0.0,"output_usd":0.0019500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.246157709043473}
{"item_id":"senseval2.d002.s000.t004","gold_sense_keys":["reckon%2:31:01::"],"candidates":[{"index":1,"sense_key":"reckon%2:31:02::","synset_id":"think.v.02"},{"index":2,"sense_key":"reckon%2:31:03::","synset_id":"calculate.v.02"},{"index":3,"sense_key":"reckon%2:31:00::","synset_id":"see.v.05"},{"index":4,"sense_key":"reckon%2:31:01::","synset_id":"calculate.v.01"},{"index":5,"sense_key":"reckon%2:31:05::","synset_id":"count.v.08"},{"index":6,"sense_key":"reckon%2:31:13::","synset_id":"reckon.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"reckon%2:31:01::","call_ids":["senseval2.d002.s000.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"reckon%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":662,"cached_input_tokens":0,"output_tokens":71,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002034,"input_uncached_usd":0.0013239999999999999,"input_cached_usd":0.0,"output_usd":0.00071,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.420886000036262}
{"item_id":"senseval2.d002.s001.t000","gold_sense_keys":["know%2:31:03::"],"candidates":[{"index":1,"sense_key":"know%2:31:01::","synset_id":"know.v.01"},{"index":2,"sense_key":"know%2:31:03::","synset_id":"know.v.02"},{"index":3,"sense_key":"know%2:31:02::","synset_id":"know.v.03"},{"index":4,"sense_key":"know%2:31:00::","synset_id":"know.v.04"},{"index":5,"sense_key":"know%2:31:04::","synset_id":"know.v.05"},{"index":6,"sense_key":"know%2:31:06::","synset_id":"acknowledge.v.06"},{"index":7,"sense_key":"know%2:31:15::","synset_id":"know.v.07"},{"index":8,"sense_key":"know%2:35:00::","synset_id":"sleep_together.v.01"},{"index":9,"sense_key":"know%2:31:11::","synset_id":"know.v.09"},{"index":10,"sense_key":"know%2:31:12::","synset_id":"know.v.10"},{"index":11,"sense_key":"know%2:31:14::","synset_id":"know.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"know%2:31:03::","call_ids":["senseval2.d002.s001.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"know%2:31:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":994,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0021079999999999996,"input_uncached_usd":0.0019879999999999997,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1123392909066752}
{"item_id":"senseval2.d002.s002.t000","gold_sense_keys":["want%2:37:00::"],"candidates":[{"index":1,"sense_key":"want%2:37:00::","synset_id":"desire.v.01"},{"index":2,"sense_key":"want%2:34:00::","synset_id":"want.v.02"},{"index":3,"sense_key":"want%2:35:00::","synset_id":"want.v.03"},{"index":4,"sense_key":"want%2:31:01::","synset_id":"want.v.04"},{"index":5,"sense_key":"want%2:42:01::","synset_id":"want.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"want%2:37:00::","call_ids":["senseval2.d002.s002.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"want%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":607,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0013340000000000001,"input_uncached_usd":0.001214,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5915200000163168}
{"item_id":"senseval2.d002.s003.t000","gold_sense_keys":["reason%1:09:01::","reason%1:10:00::","reason%1:10:01::","reason%1:16:00::"],"candidates":[{"index":1,"sense_key":"reason%1:16:00::","synset_id":"reason.n.01"},{"index":2,"sense_key":"reason%1:10:01::","synset_id":"reason.n.02"},{"index":3,"sense_key":"reason%1:09:00::","synset_id":"reason.n.03"},{"index":4,"sense_key":"reason%1:26:00::","synset_id":"rationality.n.01"},{"index":5,"sense_key":"reason%1:10:00::","synset_id":"cause.n.02"},{"index":6,"sense_key":"reason%1:09:01::","synset_id":"reason.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"reason%1:10:01::","call_ids":["senseval2.d002.s003.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"reason%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":800,"cached_input_tokens":0,"output_tokens":130,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0029,"input_uncached_usd":0.0015999999999999999,"input_cached_usd":0.0,"output_usd":0.0013000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6157108750194311}
{"item_id":"senseval2.d002.s003.t001","gold_sense_keys":["want%2:37:00::"],"candidates":[{"index":1,"sense_key":"want%2:37:00::","synset_id":"desire.v.01"},{"index":2,"sense_key":"want%2:34:00::","synset_id":"want.v.02"},{"index":3,"sense_key":"want%2:35:00::","synset_id":"want.v.03"},{"index":4,"sense_key":"want%2:31:01::","synset_id":"want.v.04"},{"index":5,"sense_key":"want%2:42:01::","synset_id":"want.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"want%2:37:00::","call_ids":["senseval2.d002.s003.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"want%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":653,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001426,"input_uncached_usd":0.0013059999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0181274579372257}
{"item_id":"senseval2.d002.s003.t002","gold_sense_keys":["effective%3:00:00::","effective%5:00:00:competent:00"],"candidates":[{"index":1,"sense_key":"effective%3:00:00::","synset_id":"effective.a.01"},{"index":2,"sense_key":"effective%5:00:00:competent:00","synset_id":"effective.s.01"},{"index":3,"sense_key":"effective%5:00:00:efficacious:00","synset_id":"effective.s.02"},{"index":4,"sense_key":"effective%5:00:00:operative:00","synset_id":"effective.s.03"},{"index":5,"sense_key":"effective%5:00:00:actual:00","synset_id":"effective.s.04"},{"index":6,"sense_key":"effective%5:00:00:operational:00","synset_id":"effective.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"effective%3:00:00::","call_ids":["senseval2.d002.s003.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"effective%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":853,"cached_input_tokens":0,"output_tokens":74,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002446,"input_uncached_usd":0.0017059999999999998,"input_cached_usd":0.0,"output_usd":0.0007400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9786612909520045}
{"item_id":"senseval2.d002.s003.t003","gold_sense_keys":["education%1:04:00::"],"candidates":[{"index":1,"sense_key":"education%1:04:00::","synset_id":"education.n.01"},{"index":2,"sense_key":"education%1:09:00::","synset_id":"education.n.02"},{"index":3,"sense_key":"education%1:09:01::","synset_id":"education.n.03"},{"index":4,"sense_key":"education%1:04:01::","synset_id":"education.n.04"},{"index":5,"sense_key":"education%1:07:01::","synset_id":"education.n.05"},{"index":6,"sense_key":"education%1:14:00::","synset_id":"department_of_education.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"education%1:04:00::","call_ids":["senseval2.d002.s003.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"education%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":776,"cached_input_tokens":0,"output_tokens":110,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002652,"input_uncached_usd":0.001552,"input_cached_usd":0.0,"output_usd":0.0011,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7195410419953987}
{"item_id":"senseval2.d002.s003.t004","gold_sense_keys":["require%2:42:00::"],"candidates":[{"index":1,"sense_key":"require%2:42:00::","synset_id":"necessitate.v.01"},{"index":2,"sense_key":"require%2:32:01::","synset_id":"ask.v.04"},{"index":3,"sense_key":"require%2:32:00::","synset_id":"command.v.02"},{"index":4,"sense_key":"require%2:34:00::","synset_id":"want.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"require%2:42:00::","call_ids":["senseval2.d002.s003.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"require%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":620,"cached_input_tokens":0,"output_tokens":149,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0027300000000000002,"input_uncached_usd":0.00124,"input_cached_usd":0.0,"output_usd":0.0014900000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0978419589810073}
{"item_id":"senseval2.d002.s003.t005","gold_sense_keys":["relinquish%2:41:00::"],"candidates":[{"index":1,"sense_key":"relinquish%2:40:01::","synset_id":"release.v.08"},{"index":2,"sense_key":"relinquish%2:41:00::","synset_id":"waive.v.01"},{"index":3,"sense_key":"relinquish%2:40:00::","synset_id":"foreswear.v.02"},{"index":4,"sense_key":"relinquish%2:35:02::","synset_id":"let_go_of.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"relinquish%2:41:00::","call_ids":["senseval2.d002.s003.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"relinquish%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":698,"cached_input_tokens":0,"output_tokens":70,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0020959999999999998,"input_uncached_usd":0.0013959999999999999,"input_cached_usd":0.0,"output_usd":0.0007000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3449965410400182}
{"item_id":"senseval2.d002.s003.t008","gold_sense_keys":["belief%1:09:00::"],"candidates":[{"index":1,"sense_key":"belief%1:09:00::","synset_id":"belief.n.01"},{"index":2,"sense_key":"belief%1:09:01::","synset_id":"impression.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"belief%1:09:00::","call_ids":["senseval2.d002.s003.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"belief%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":501,"cached_input_tokens":0,"output_tokens":104,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002042,"input_uncached_usd":0.0010019999999999999,"input_cached_usd":0.0,"output_usd":0.0010400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.576098290970549}
{"item_id":"senseval2.d002.s003.t013","gold_sense_keys":["young%3:00:00::"],"candidates":[{"index":1,"sense_key":"young%3:00:00::","synset_id":"young.a.01"},{"index":2,"sense_key":"young%5:00:00:early:02","synset_id":"new.s.08"},{"index":3,"sense_key":"young%5:00:01:young:00","synset_id":"youthful.s.01"},{"index":4,"sense_key":"young%5:00:00:new:00","synset_id":"young.s.03"},{"index":5,"sense_key":"young%5:00:00:inexperienced:00","synset_id":"unseasoned.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"young%3:00:00::","call_ids":["senseval2.d002.s003.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"young%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":691,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001502,"input_uncached_usd":0.001382,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":11.45718087500427}
{"item_id":"senseval2.d002.s003.t014","gold_sense_keys":["people%1:14:00::"],"candidates":[{"index":1,"sense_key":"people%1:14:00::","synset_id":"people.n.01"},{"index":2,"sense_key":"people%1:14:01::","synset_id":"citizenry.n.01"},{"index":3,"sense_key":"people%1:14:02::","synset_id":"people.n.03"},{"index":4,"sense_key":"people%1:14:03::","synset_id":"multitude.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"people%1:14:00::","call_ids":["senseval2.d002.s003.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"people%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":619,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001358,"input_uncached_usd":0.001238,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5117167079588398}
{"item_id":"senseval2.d002.s003.t016","gold_sense_keys":["violate%2:41:00::"],"candidates":[{"index":1,"sense_key":"violate%2:42:00::","synset_id":"violate.v.01"},{"index":2,"sense_key":"violate%2:41:00::","synset_id":"transgress.v.01"},{"index":3,"sense_key":"violate%2:35:00::","synset_id":"violate.v.03"},{"index":4,"sense_key":"violate%2:41:02::","synset_id":"desecrate.v.01"},{"index":5,"sense_key":"violate%2:41:01::","synset_id":"rape.v.01"},{"index":6,"sense_key":"violate%2:35:01::","synset_id":"rape.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"violate%2:41:00::","call_ids":["senseval2.d002.s003.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"violate%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":814,"cached_input_tokens":0,"output_tokens":215,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003778,"input_uncached_usd":0.0016279999999999999,"input_cached_usd":0.0,"output_usd":0.00215,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.020957999979146}
{"item_id":"senseval2.d002.s004.t000","gold_sense_keys":["belief%1:09:00::"],"candidates":[{"index":1,"sense_key":"belief%1:09:00::","synset_id":"belief.n.01"},{"index":2,"sense_key":"belief%1:09:01::","synset_id":"impression.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"belief%1:09:00::","call_ids":["senseval2.d002.s004.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"belief%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":507,"cached_input_tokens":0,"output_tokens":85,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001864,"input_uncached_usd":0.001014,"input_cached_usd":0.0,"output_usd":0.0008500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.108367708977312}
{"item_id":"senseval2.d002.s004.t001","gold_sense_keys":["so%4:02:03::"],"candidates":[{"index":1,"sense_key":"so%4:02:02::","synset_id":"so.r.01"},{"index":2,"sense_key":"so%4:02:06::","synset_id":"so.r.02"},{"index":3,"sense_key":"so%4:02:05::","synset_id":"so.r.03"},{"index":4,"sense_key":"so%4:02:07::","synset_id":"so.r.04"},{"index":5,"sense_key":"so%4:02:08::","synset_id":"so.r.05"},{"index":6,"sense_key":"so%4:02:04::","synset_id":"thus.r.02"},{"index":7,"sense_key":"so%4:02:03::","synset_id":"so.r.07"},{"index":8,"sense_key":"so%4:02:09::","synset_id":"then.r.01"},{"index":9,"sense_key":"so%4:02:10::","synset_id":"therefore.r.01"},{"index":10,"sense_key":"so%4:02:00::","synset_id":"indeed.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"so%4:02:03::","call_ids":["senseval2.d002.s004.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"so%4:02:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1053,"cached_input_tokens":0,"output_tokens":133,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003436,"input_uncached_usd":0.002106,"input_cached_usd":0.0,"output_usd":0.00133,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7213583330158144}
{"item_id":"senseval2.d002.s004.t003","gold_sense_keys":["educational%3:01:00::"],"candidates":[{"index":1,"sense_key":"educational%3:01:00::","synset_id":"educational.a.01"},{"index":2,"sense_key":"educational%5:00:00:instructive:00","synset_id":"educational.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"educational%3:01:00::","call_ids":["senseval2.d002.s004.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"educational%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":472,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001554,"input_uncached_usd":0.000944,"input_cached_usd":0.0,"output_usd":0.0006100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.482769374968484}
{"item_id":"senseval2.d002.s004.t004","gold_sense_keys":["establishment%1:14:00::","establishment%1:14:02::"],"candidates":[{"index":1,"sense_key":"establishment%1:04:00::","synset_id":"constitution.n.02"},{"index":2,"sense_key":"establishment%1:14:00::","synset_id":"institution.n.01"},{"index":3,"sense_key":"establishment%1:14:02::","synset_id":"administration.n.02"},{"index":4,"sense_key":"establishment%1:06:00::","synset_id":"establishment.n.04"},{"index":5,"sense_key":"establishment%1:14:01::","synset_id":"establishment.n.05"},{"index":6,"sense_key":"establishment%1:22:00::","synset_id":"establishment.n.06"},{"index":7,"sense_key":"establishment%1:09:00::","synset_id":"establishment.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"establishment%1:14:02::","call_ids":["senseval2.d002.s004.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"establishment%1:14:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":810,"cached_input_tokens":0,"output_tokens":141,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0030299999999999997,"input_uncached_usd":0.00162,"input_cached_usd":0.0,"output_usd":0.00141,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.010053582955152}
{"item_id":"senseval2.d002.s004.t005","gold_sense_keys":["politician%1:18:00::","politician%1:18:01::"],"candidates":[{"index":1,"sense_key":"politician%1:18:01::","synset_id":"politician.n.01"},{"index":2,"sense_key":"politician%1:18:00::","synset_id":"politician.n.02"},{"index":3,"sense_key":"politician%1:18:02::","synset_id":"politician.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"politician%1:18:00::","call_ids":["senseval2.d002.s004.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"politician%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":520,"cached_input_tokens":0,"output_tokens":139,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00243,"input_uncached_usd":0.00104,"input_cached_usd":0.0,"output_usd":0.0013900000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9426848329603672}
{"item_id":"senseval2.d002.s004.t006","gold_sense_keys":["even%4:02:00::"],"candidates":[{"index":1,"sense_key":"even%4:02:00::","synset_id":"even.r.01"},{"index":2,"sense_key":"even%4:02:03::","synset_id":"even.r.02"},{"index":3,"sense_key":"even%4:02:01::","synset_id":"even.r.03"},{"index":4,"sense_key":"even%4:02:04::","synset_id":"even.r.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"even%4:02:00::","call_ids":["senseval2.d002.s004.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"even%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":619,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001358,"input_uncached_usd":0.001238,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4291899589588866}
{"item_id":"senseval2.d002.s004.t007","gold_sense_keys":["parent%1:18:00::"],"candidates":[{"index":1,"sense_key":"parent%1:18:00::","synset_id":"parent.n.01"},{"index":2,"sense_key":"parent%1:03:00::","synset_id":"parent.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"parent%1:18:00::","call_ids":["senseval2.d002.s004.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"parent%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":498,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001116,"input_uncached_usd":0.000996,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5072312919655815}
{"item_id":"senseval2.d002.s004.t008","gold_sense_keys":["seem%2:39:01::"],"candidates":[{"index":1,"sense_key":"seem%2:39:00::","synset_id":"look.v.02"},{"index":2,"sense_key":"seem%2:39:01::","synset_id":"appear.v.04"},{"index":3,"sense_key":"seem%2:42:00::","synset_id":"seem.v.03"},{"index":4,"sense_key":"seem%2:39:02::","synset_id":"seem.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"seem%2:39:01::","call_ids":["senseval2.d002.s004.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"seem%2:39:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":652,"cached_input_tokens":0,"output_tokens":251,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003814,"input_uncached_usd":0.001304,"input_cached_usd":0.0,"output_usd":0.00251,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.8090632079401985}
{"item_id":"senseval2.d002.s004.t010","gold_sense_keys":["blasphemous%5:00:00:irreverent:00"],"candidates":[{"index":1,"sense_key":"blasphemous%5:00:00:irreverent:00","synset_id":"blasphemous.s.01"},{"index":2,"sense_key":"blasphemous%5:00:00:dirty:02","synset_id":"blasphemous.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"blasphemous%5:00:00:irreverent:00","call_ids":["senseval2.d002.s004.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"blasphemous%5:00:00:irreverent:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":566,"cached_input_tokens":0,"output_tokens":76,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001892,"input_uncached_usd":0.001132,"input_cached_usd":0.0,"output_usd":0.00076,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.36158258409705}
{"item_id":"senseval2.d002.s004.t011","gold_sense_keys":["challenge%2:32:01::"],"candidates":[{"index":1,"sense_key":"challenge%2:32:01::","synset_id":"challenge.v.01"},{"index":2,"sense_key":"challenge%2:32:00::","synset_id":"challenge.v.02"},{"index":3,"sense_key":"challenge%2:32:03::","synset_id":"challenge.v.03"},{"index":4,"sense_key":"challenge%2:32:02::","synset_id":"challenge.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"challenge%2:32:01::","call_ids":["senseval2.d002.s004.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"challenge%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":588,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001296,"input_uncached_usd":0.001176,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0106074169743806}
{"item_id":"senseval2.d002.s005.t000","gold_sense_keys":["here%4:02:02::"],"candidates":[{"index":1,"sense_key":"here%4:02:00::","synset_id":"here.r.01"},{"index":2,"sense_key":"here%4:02:02::","synset_id":"here.r.02"},{"index":3,"sense_key":"here%4:02:01::","synset_id":"here.r.03"},{"index":4,"sense_key":"here%4:02:04::","synset_id":"here.r.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"here%4:02:02::","call_ids":["senseval2.d002.s005.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"here%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":659,"cached_input_tokens":0,"output_tokens":115,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002468,"input_uncached_usd":0.001318,"input_cached_usd":0.0,"output_usd":0.0011500000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2449436669703573}
{"item_id":"senseval2.d002.s005.t001","gold_sense_keys":["example%1:09:00::"],"candidates":[{"index":1,"sense_key":"example%1:09:00::","synset_id":"example.n.01"},{"index":2,"sense_key":"example%1:09:02::","synset_id":"model.n.07"},{"index":3,"sense_key":"example%1:09:01::","synset_id":"exemplar.n.01"},{"index":4,"sense_key":"example%1:10:00::","synset_id":"example.n.04"},{"index":5,"sense_key":"example%1:11:00::","synset_id":"case.n.01"},{"index":6,"sense_key":"example%1:04:00::","synset_id":"exercise.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"example%1:09:00::","call_ids":["senseval2.d002.s005.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"example%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":834,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001788,"input_uncached_usd":0.001668,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1054749169852585}
{"item_id":"senseval2.d002.s006.t000","gold_sense_keys":["ask%2:32:04::"],"candidates":[{"index":1,"sense_key":"ask%2:32:00::","synset_id":"ask.v.01"},{"index":2,"sense_key":"ask%2:32:01::","synset_id":"ask.v.02"},{"index":3,"sense_key":"ask%2:32:02::","synset_id":"ask.v.03"},{"index":4,"sense_key":"ask%2:32:05::","synset_id":"ask.v.04"},{"index":5,"sense_key":"ask%2:32:04::","synset_id":"ask.v.05"},{"index":6,"sense_key":"ask%2:42:00::","synset_id":"necessitate.v.01"},{"index":7,"sense_key":"ask%2:32:09::","synset_id":"ask.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"ask%2:32:04::","call_ids":["senseval2.d002.s006.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"ask%2:32:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":884,"cached_input_tokens":0,"output_tokens":167,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003438,"input_uncached_usd":0.0017679999999999998,"input_cached_usd":0.0,"output_usd":0.00167,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.992391875013709}
{"item_id":"senseval2.d002.s006.t001","gold_sense_keys":["sample%1:09:00::","sample%1:09:01::"],"candidates":[{"index":1,"sense_key":"sample%1:09:00::","synset_id":"sample.n.01"},{"index":2,"sense_key":"sample%1:09:01::","synset_id":"sample_distribution.n.01"},{"index":3,"sense_key":"sample%1:17:00::","synset_id":"sample.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"sample%1:09:01::","call_ids":["senseval2.d002.s006.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"sample%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":588,"cached_input_tokens":0,"output_tokens":118,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002356,"input_uncached_usd":0.001176,"input_cached_usd":0.0,"output_usd":0.00118,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9869258749531582}
{"item_id":"senseval2.d002.s006.t002","gold_sense_keys":["american%3:01:01::"],"candidates":[{"index":1,"sense_key":"american%3:01:01::","synset_id":"american.a.01"},{"index":2,"sense_key":"american%3:01:00::","synset_id":"american.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"american%3:01:01::","call_ids":["senseval2.d002.s006.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"american%3:01:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":561,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001242,"input_uncached_usd":0.001122,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1322954579954967}
{"item_id":"senseval2.d002.s006.t003","gold_sense_keys":["parent%1:18:00::"],"candidates":[{"index":1,"sense_key":"parent%1:18:00::","synset_id":"parent.n.01"},{"index":2,"sense_key":"parent%1:03:00::","synset_id":"parent.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"parent%1:18:00::","call_ids":["senseval2.d002.s006.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"parent%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":556,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001232,"input_uncached_usd":0.001112,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5031862909672782}
{"item_id":"senseval2.d002.s006.t006","gold_sense_keys":["encourage%2:41:00::"],"candidates":[{"index":1,"sense_key":"encourage%2:41:00::","synset_id":"promote.v.01"},{"index":2,"sense_key":"encourage%2:37:00::","synset_id":"encourage.v.02"},{"index":3,"sense_key":"encourage%2:32:00::","synset_id":"encourage.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"encourage%2:41:00::","call_ids":["senseval2.d002.s006.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"encourage%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":605,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00181,"input_uncached_usd":0.00121,"input_cached_usd":0.0,"output_usd":0.0006000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8396379170008004}
{"item_id":"senseval2.d002.s006.t008","gold_sense_keys":["child%1:18:01::"],"candidates":[{"index":1,"sense_key":"child%1:18:00::","synset_id":"child.n.01"},{"index":2,"sense_key":"child%1:18:01::","synset_id":"child.n.02"},{"index":3,"sense_key":"child%1:18:06::","synset_id":"child.n.03"},{"index":4,"sense_key":"child%1:18:07::","synset_id":"child.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"child%1:18:00::","call_ids":["senseval2.d002.s006.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"child%1:18:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":687,"cached_input_tokens":0,"output_tokens":161,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002984,"input_uncached_usd":0.001374,"input_cached_usd":0.0,"output_usd":0.00161,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.463540166965686}
{"item_id":"senseval2.d002.s006.t009","gold_sense_keys":["answer%1:10:01::"],"candidates":[{"index":1,"sense_key":"answer%1:10:01::","synset_id":"answer.n.01"},{"index":2,"sense_key":"answer%1:10:02::","synset_id":"solution.n.02"},{"index":3,"sense_key":"answer%1:10:00::","synset_id":"answer.n.03"},{"index":4,"sense_key":"answer%1:10:03::","synset_id":"answer.n.04"},{"index":5,"sense_key":"answer%1:04:00::","synset_id":"answer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"answer%1:10:01::","call_ids":["senseval2.d002.s006.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"answer%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":855,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00248,"input_uncached_usd":0.00171,"input_cached_usd":0.0,"output_usd":0.0007700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4087562910281122}
{"item_id":"senseval2.d002.s007.t000","gold_sense_keys":["mean%2:32:01::"],"candidates":[{"index":1,"sense_key":"mean%2:32:01::","synset_id":"mean.v.01"},{"index":2,"sense_key":"mean%2:42:00::","synset_id":"entail.v.01"},{"index":3,"sense_key":"mean%2:32:03::","synset_id":"mean.v.03"},{"index":4,"sense_key":"mean%2:31:00::","synset_id":"intend.v.01"},{"index":5,"sense_key":"mean%2:42:03::","synset_id":"mean.v.05"},{"index":6,"sense_key":"mean%2:31:01::","synset_id":"think_of.v.04"},{"index":7,"sense_key":"mean%2:31:05::","synset_id":"mean.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"mean%2:32:01::","call_ids":["senseval2.d002.s007.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"mean%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":873,"cached_input_tokens":0,"output_tokens":150,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0032459999999999998,"input_uncached_usd":0.001746,"input_cached_usd":0.0,"output_usd":0.0015,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8654110840288922}
{"item_id":"senseval2.d002.s008.t000","gold_sense_keys":["one%1:09:00::"],"candidates":[{"index":1,"sense_key":"one%1:23:00::","synset_id":"one.n.01"},{"index":2,"sense_key":"one%1:09:00::","synset_id":"one.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"one%1:09:00::","call_ids":["senseval2.d002.s008.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"one%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":593,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001786,"input_uncached_usd":0.001186,"input_cached_usd":0.0,"output_usd":0.0006000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7175620420603082}
{"item_id":"senseval2.d002.s009.t002","gold_sense_keys":["equate%2:31:00::"],"candidates":[{"index":1,"sense_key":"equate%2:31:00::","synset_id":"compare.v.03"},{"index":2,"sense_key":"equate%2:42:00::","synset_id":"equate.v.02"},{"index":3,"sense_key":"equate%2:30:00::","synset_id":"equal.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"equate%2:31:00::","call_ids":["senseval2.d002.s009.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"equate%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":662,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001444,"input_uncached_usd":0.0013239999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.7559531660517678}
{"item_id":"senseval2.d002.s009.t004","gold_sense_keys":["encourage%2:41:00::"],"candidates":[{"index":1,"sense_key":"encourage%2:41:00::","synset_id":"promote.v.01"},{"index":2,"sense_key":"encourage%2:37:00::","synset_id":"encourage.v.02"},{"index":3,"sense_key":"encourage%2:32:00::","synset_id":"encourage.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"encourage%2:41:00::","call_ids":["senseval2.d002.s009.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"encourage%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":595,"cached_input_tokens":0,"output_tokens":122,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00241,"input_uncached_usd":0.0011899999999999999,"input_cached_usd":0.0,"output_usd":0.0012200000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0055968750966713}
{"item_id":"senseval2.d002.s010.t000","gold_sense_keys":["result%1:19:00::"],"candidates":[{"index":1,"sense_key":"result%1:19:00::","synset_id":"consequence.n.01"},{"index":2,"sense_key":"result%1:10:00::","synset_id":"solution.n.02"},{"index":3,"sense_key":"result%1:11:00::","synset_id":"result.n.03"},{"index":4,"sense_key":"result%1:10:01::","synset_id":"resultant_role.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"result%1:19:00::","call_ids":["senseval2.d002.s010.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"result%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":814,"cached_input_tokens":0,"output_tokens":79,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002418,"input_uncached_usd":0.0016279999999999999,"input_cached_usd":0.0,"output_usd":0.00079,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6226499159820378}
{"item_id":"senseval2.d002.s010.t001","gold_sense_keys":["generation%1:14:01::"],"candidates":[{"index":1,"sense_key":"generation%1:14:01::","synset_id":"coevals.n.01"},{"index":2,"sense_key":"generation%1:14:00::","synset_id":"generation.n.02"},{"index":3,"sense_key":"generation%1:28:00::","synset_id":"generation.n.03"},{"index":4,"sense_key":"generation%1:28:01::","synset_id":"generation.n.04"},{"index":5,"sense_key":"generation%1:11:00::","synset_id":"genesis.n.01"},{"index":6,"sense_key":"generation%1:04:01::","synset_id":"generation.n.06"},{"index":7,"sense_key":"generation%1:04:00::","synset_id":"generation.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"generation%1:14:01::","call_ids":["senseval2.d002.s010.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"generation%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":825,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00177,"input_uncached_usd":0.00165,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3227418339811265}
{"item_id":"senseval2.d002.s010.t002","gold_sense_keys":["young%3:00:00::"],"candidates":[{"index":1,"sense_key":"young%3:00:00::","synset_id":"young.a.01"},{"index":2,"sense_key":"young%5:00:00:early:02","synset_id":"new.s.08"},{"index":3,"sense_key":"young%5:00:01:young:00","synset_id":"youthful.s.01"},{"index":4,"sense_key":"young%5:00:00:new:00","synset_id":"young.s.03"},{"index":5,"sense_key":"young%5:00:00:inexperienced:00","synset_id":"unseasoned.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"young%3:00:00::","call_ids":["senseval2.d002.s010.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"young%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":802,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001724,"input_uncached_usd":0.001604,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1135282500181347}
{"item_id":"senseval2.d002.s010.t003","gold_sense_keys":["people%1:14:00::"],"candidates":[{"index":1,"sense_key":"people%1:14:00::","synset_id":"people.n.01"},{"index":2,"sense_key":"people%1:14:01::","synset_id":"citizenry.n.01"},{"index":3,"sense_key":"people%1:14:02::","synset_id":"people.n.03"},{"index":4,"sense_key":"people%1:14:03::","synset_id":"multitude.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"people%1:14:00::","call_ids":["senseval2.d002.s010.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"people%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":730,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00158,"input_uncached_usd":0.00146,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.558454499929212}
{"item_id":"senseval2.d002.s010.t006","gold_sense_keys":["incompetence%1:07:00::"],"candidates":[{"index":1,"sense_key":"incompetence%1:07:00::","synset_id":"incompetence.n.01"},{"index":2,"sense_key":"incompetence%1:26:00::","synset_id":"incompetence.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"incompetence%1:07:00::","call_ids":["senseval2.d002.s010.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"incompetence%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":589,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0012980000000000001,"input_uncached_usd":0.001178,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.046670916955918}
{"item_id":"senseval2.d002.s010.t007","gold_sense_keys":["match%2:42:01::"],"candidates":[{"index":1,"sense_key":"match%2:42:00::","synset_id":"match.v.01"},{"index":2,"sense_key":"match%2:40:00::","synset_id":"match.v.02"},{"index":3,"sense_key":"match%2:35:08::","synset_id":"match.v.03"},{"index":4,"sense_key":"match%2:42:01::","synset_id":"equal.v.02"},{"index":5,"sense_key":"match%2:30:01::","synset_id":"match.v.05"},{"index":6,"sense_key":"match%2:41:00::","synset_id":"meet.v.06"},{"index":7,"sense_key":"match%2:35:10::","synset_id":"match.v.07"},{"index":8,"sense_key":"match%2:33:00::","synset_id":"pit.v.01"},{"index":9,"sense_key":"match%2:30:00::","synset_id":"match.v.09"},{"index":10,"sense_key":"match%2:30:07::","synset_id":"equal.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"match%2:42:01::","call_ids":["senseval2.d002.s010.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"match%2:42:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1149,"cached_input_tokens":0,"output_tokens":85,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0031479999999999998,"input_uncached_usd":0.0022979999999999997,"input_cached_usd":0.0,"output_usd":0.0008500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5717657499480993}
{"item_id":"senseval2.d002.s010.t008","gold_sense_keys":["only%4:02:01::"],"candidates":[{"index":1,"sense_key":"only%4:02:02::","synset_id":"merely.r.01"},{"index":2,"sense_key":"only%4:02:01::","synset_id":"entirely.r.02"},{"index":3,"sense_key":"only%4:02:00::","synset_id":"only.r.03"},{"index":4,"sense_key":"only%4:02:04::","synset_id":"only.r.04"},{"index":5,"sense_key":"only%4:02:06::","synset_id":"only.r.05"},{"index":6,"sense_key":"only%4:02:03::","synset_id":"only.r.06"},{"index":7,"sense_key":"only%4:02:05::","synset_id":"only.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"only%4:02:01::","call_ids":["senseval2.d002.s010.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"only%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":863,"cached_input_tokens":0,"output_tokens":139,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003116,"input_uncached_usd":0.0017259999999999999,"input_cached_usd":0.0,"output_usd":0.0013900000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.190400750027038}
{"item_id":"senseval2.d002.s010.t009","gold_sense_keys":["good%3:00:01::"],"candidates":[{"index":1,"sense_key":"good%3:00:01::","synset_id":"good.a.01"},{"index":2,"sense_key":"good%5:00:00:ample:00","synset_id":"full.s.04"},{"index":3,"sense_key":"good%3:00:02::","synset_id":"good.a.03"},{"index":4,"sense_key":"good%5:00:00:reputable:00","synset_id":"estimable.s.01"},{"index":5,"sense_key":"good%5:00:00:advantageous:00","synset_id":"beneficial.s.01"},{"index":6,"sense_key":"good%5:00:00:nice:00","synset_id":"good.s.04"},{"index":7,"sense_key":"good%5:00:00:righteous:00","synset_id":"good.s.05"},{"index":8,"sense_key":"good%5:00:00:skilled:00","synset_id":"adept.s.01"},{"index":9,"sense_key":"good%5:00:00:complete:00","synset_id":"good.s.07"},{"index":10,"sense_key":"good%5:00:00:close:02","synset_id":"dear.s.02"},{"index":11,"sense_key":"good%5:00:01:sound:00","synset_id":"dependable.s.03"},{"index":12,"sense_key":"good%5:00:00:opportune:00","synset_id":"good.s.10"},{"index":13,"sense_key":"good%5:00:00:fortunate:00","synset_id":"good.s.11"},{"index":14,"sense_key":"good%5:00:00:operative:00","synset_id":"effective.s.03"},{"index":15,"sense_key":"good%5:00:00:pleasing:00","synset_id":"good.s.13"},{"index":16,"sense_key":"good%5:00:00:intellectual:00","synset_id":"good.s.14"},{"index":17,"sense_key":"good%5:00:00:healthy:00","synset_id":"good.s.15"},{"index":18,"sense_key":"good%5:00:00:healthful:00","synset_id":"good.s.16"},{"index":19,"sense_key":"good%5:00:00:genuine:00","synset_id":"good.s.17"},{"index":20,"sense_key":"good%5:00:00:fresh:01","synset_id":"good.s.18"},{"index":21,"sense_key":"good%5:00:00:discriminating:00","synset_id":"good.s.19"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"good%3:00:01::","call_ids":["senseval2.d002.s010.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"good%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1750,"cached_input_tokens":0,"output_tokens":159,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00509,"input_uncached_usd":0.0034999999999999996,"input_cached_usd":0.0,"output_usd":0.00159,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2092329579172656}
{"item_id":"senseval2.d002.s010.t010","gold_sense_keys":["opinion%1:09:00::"],"candidates":[{"index":1,"sense_key":"opinion%1:09:00::","synset_id":"opinion.n.01"},{"index":2,"sense_key":"opinion%1:10:01::","synset_id":"opinion.n.02"},{"index":3,"sense_key":"opinion%1:09:01::","synset_id":"public_opinion.n.01"},{"index":4,"sense_key":"opinion%1:10:00::","synset_id":"opinion.n.04"},{"index":5,"sense_key":"opinion%1:04:01::","synset_id":"opinion.n.05"},{"index":6,"sense_key":"opinion%1:09:02::","synset_id":"impression.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"opinion%1:09:00::","call_ids":["senseval2.d002.s010.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"opinion%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":968,"cached_input_tokens":0,"output_tokens":109,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003026,"input_uncached_usd":0.001936,"input_cached_usd":0.0,"output_usd":0.00109,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7417015410028398}
{"item_id":"senseval2.d002.s011.t000","gold_sense_keys":["whole%3:00:00::"],"candidates":[{"index":1,"sense_key":"whole%3:00:00::","synset_id":"whole.a.01"},{"index":2,"sense_key":"whole%3:00:02::","synset_id":"whole.a.02"},{"index":3,"sense_key":"whole%5:00:00:uninjured:00","synset_id":"unharmed.s.01"},{"index":4,"sense_key":"whole%5:00:00:healthy:00","synset_id":"hale.s.01"},{"index":5,"sense_key":"whole%5:00:00:undiversified:00","synset_id":"solid.s.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"whole%3:00:00::","call_ids":["senseval2.d002.s011.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"whole%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":820,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00176,"input_uncached_usd":0.00164,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0388261249754578}
{"item_id":"senseval2.d002.s011.t001","gold_sense_keys":["notion%1:09:00::"],"candidates":[{"index":1,"sense_key":"notion%1:09:02::","synset_id":"impression.n.01"},{"index":2,"sense_key":"notion%1:09:00::","synset_id":"notion.n.02"},{"index":3,"sense_key":"notion%1:09:01::","synset_id":"notion.n.03"},{"index":4,"sense_key":"notion%1:06:00::","synset_id":"notion.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"notion%1:09:00::","call_ids":["senseval2.d002.s011.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"notion%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":776,"cached_input_tokens":0,"output_tokens":76,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002312,"input_uncached_usd":0.001552,"input_cached_usd":0.0,"output_usd":0.00076,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4564546250039712}
{"item_id":"senseval2.d002.s011.t003","gold_sense_keys":["education%1:04:00::"],"candidates":[{"index":1,"sense_key":"education%1:04:00::","synset_id":"education.n.01"},{"index":2,"sense_key":"education%1:09:00::","synset_id":"education.n.02"},{"index":3,"sense_key":"education%1:09:01::","synset_id":"education.n.03"},{"index":4,"sense_key":"education%1:04:01::","synset_id":"education.n.04"},{"index":5,"sense_key":"education%1:07:01::","synset_id":"education.n.05"},{"index":6,"sense_key":"education%1:14:00::","synset_id":"department_of_education.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"education%1:04:00::","call_ids":["senseval2.d002.s011.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"education%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":904,"cached_input_tokens":0,"output_tokens":147,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003278,"input_uncached_usd":0.001808,"input_cached_usd":0.0,"output_usd":0.0014700000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0733228749595582}
{"item_id":"senseval2.d002.s011.t004","gold_sense_keys":["romantic%5:00:00:impractical:00"],"candidates":[{"index":1,"sense_key":"romantic%3:01:00::","synset_id":"romantic.a.01"},{"index":2,"sense_key":"romantic%5:00:01:loving:00","synset_id":"amatory.s.01"},{"index":3,"sense_key":"romantic%5:00:00:impractical:00","synset_id":"quixotic.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"romantic%5:00:00:impractical:00","call_ids":["senseval2.d002.s011.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"romantic%5:00:00:impractical:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":763,"cached_input_tokens":0,"output_tokens":295,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004476000000000001,"input_uncached_usd":0.001526,"input_cached_usd":0.0,"output_usd":0.0029500000000000004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.0040545420488343}
{"item_id":"senseval2.d002.s011.t005","gold_sense_keys":["rebellion%1:04:01::"],"candidates":[{"index":1,"sense_key":"rebellion%1:04:01::","synset_id":"rebellion.n.01"},{"index":2,"sense_key":"rebellion%1:04:00::","synset_id":"rebellion.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"rebellion%1:04:01::","call_ids":["senseval2.d002.s011.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"rebellion%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":658,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001996,"input_uncached_usd":0.001316,"input_cached_usd":0.0,"output_usd":0.00068,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5075974999926984}
{"item_id":"senseval2.d002.s011.t006","gold_sense_keys":["disciplined%5:00:00:controlled:00"],"candidates":[{"index":1,"sense_key":"disciplined%5:00:00:controlled:00","synset_id":"disciplined.s.01"},{"index":2,"sense_key":"disciplined%5:00:00:trained:00","synset_id":"disciplined.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"disciplined%5:00:00:trained:00","call_ids":["senseval2.d002.s011.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"disciplined%5:00:00:trained:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":627,"cached_input_tokens":0,"output_tokens":228,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0035340000000000002,"input_uncached_usd":0.001254,"input_cached_usd":0.0,"output_usd":0.0022800000000000003,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.357943541952409}
{"item_id":"senseval2.d002.s011.t007","gold_sense_keys":["instruction%1:04:01::"],"candidates":[{"index":1,"sense_key":"instruction%1:10:04::","synset_id":"direction.n.06"},{"index":2,"sense_key":"instruction%1:04:01::","synset_id":"education.n.01"},{"index":3,"sense_key":"instruction%1:04:00::","synset_id":"teaching.n.01"},{"index":4,"sense_key":"instruction%1:10:02::","synset_id":"instruction.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"instruction%1:04:01::","call_ids":["senseval2.d002.s011.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"instruction%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":799,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0017180000000000001,"input_uncached_usd":0.001598,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.920941666001454}
{"item_id":"senseval2.d002.s011.t009","gold_sense_keys":["authoritarian%5:00:00:domineering:00"],"candidates":[{"index":1,"sense_key":"authoritarian%5:00:00:undemocratic:00","synset_id":"authoritarian.s.01"},{"index":2,"sense_key":"authoritarian%5:00:00:domineering:00","synset_id":"authoritarian.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"authoritarian%5:00:00:domineering:00","call_ids":["senseval2.d002.s011.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"authoritarian%5:00:00:domineering:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":722,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002164,"input_uncached_usd":0.001444,"input_cached_usd":0.0,"output_usd":0.00072,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5795198749983683}
{"item_id":"senseval2.d002.s011.t010","gold_sense_keys":["repression%1:04:00::"],"candidates":[{"index":1,"sense_key":"repression%1:26:00::","synset_id":"repression.n.01"},{"index":2,"sense_key":"repression%1:22:00::","synset_id":"repression.n.02"},{"index":3,"sense_key":"repression%1:04:00::","synset_id":"repression.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"repression%1:04:00::","call_ids":["senseval2.d002.s011.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"repression%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":680,"cached_input_tokens":0,"output_tokens":130,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00266,"input_uncached_usd":0.0013599999999999999,"input_cached_usd":0.0,"output_usd":0.0013000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0541445419657975}
{"item_id":"senseval2.d002.s011.t011","gold_sense_keys":["frustration%1:04:02::"],"candidates":[{"index":1,"sense_key":"frustration%1:12:00::","synset_id":"frustration.n.01"},{"index":2,"sense_key":"frustration%1:04:02::","synset_id":"frustration.n.02"},{"index":3,"sense_key":"frustration%1:12:01::","synset_id":"frustration.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"frustration%1:04:02::","call_ids":["senseval2.d002.s011.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"frustration%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":686,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002032,"input_uncached_usd":0.001372,"input_cached_usd":0.0,"output_usd":0.0006600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.408739500096999}
{"item_id":"senseval2.d002.s011.t012","gold_sense_keys":["latent%5:00:00:potential:00"],"candidates":[{"index":1,"sense_key":"latent%5:00:00:potential:00","synset_id":"latent.s.01"},{"index":2,"sense_key":"latent%5:00:00:inactive:02","synset_id":"latent.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"latent%5:00:00:potential:00","call_ids":["senseval2.d002.s011.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"latent%5:00:00:potential:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":623,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001366,"input_uncached_usd":0.001246,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5385480830445886}
{"item_id":"senseval2.d002.s011.t013","gold_sense_keys":["talent%1:09:00::"],"candidates":[{"index":1,"sense_key":"talent%1:09:00::","synset_id":"endowment.n.01"},{"index":2,"sense_key":"talent%1:18:00::","synset_id":"talent.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"talent%1:09:00::","call_ids":["senseval2.d002.s011.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"talent%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":605,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00133,"input_uncached_usd":0.00121,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0383783750003204}
{"item_id":"senseval2.d002.s011.t016","gold_sense_keys":["yet%4:02:02::"],"candidates":[{"index":1,"sense_key":"yet%4:02:02::","synset_id":"yet.r.01"},{"index":2,"sense_key":"yet%4:02:04::","synset_id":"so_far.r.01"},{"index":3,"sense_key":"yet%4:02:03::","synset_id":"even.r.03"},{"index":4,"sense_key":"yet%4:02:00::","synset_id":"yet.r.04"},{"index":5,"sense_key":"yet%4:02:05::","synset_id":"so_far.r.03"},{"index":6,"sense_key":"yet%4:02:06::","synset_id":"however.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"yet%4:02:04::","call_ids":["senseval2.d002.s011.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"yet%4:02:04::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":979,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0025580000000000004,"input_uncached_usd":0.001958,"input_cached_usd":0.0,"output_usd":0.0006000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8379287499701604}
{"item_id":"senseval2.d002.s011.t018","gold_sense_keys":["potentiality%1:09:00::"],"candidates":[{"index":1,"sense_key":"potentiality%1:26:00::","synset_id":"potential.n.01"},{"index":2,"sense_key":"potentiality%1:09:00::","synset_id":"capability.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"potentiality%1:09:00::","call_ids":["senseval2.d002.s011.t018__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"potentiality%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":612,"cached_input_tokens":0,"output_tokens":125,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002474,"input_uncached_usd":0.001224,"input_cached_usd":0.0,"output_usd":0.00125,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7163538750028238}
{"item_id":"senseval2.d002.s011.t019","gold_sense_keys":["inherent%5:00:00:intrinsic:00"],"candidates":[{"index":1,"sense_key":"inherent%5:00:00:intrinsic:00","synset_id":"built-in.s.01"},{"index":2,"sense_key":"inherent%5:00:00:implicit:00","synset_id":"implicit_in.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"inherent%5:00:00:implicit:00","call_ids":["senseval2.d002.s011.t019__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"inherent%5:00:00:implicit:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":702,"cached_input_tokens":0,"output_tokens":104,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002444,"input_uncached_usd":0.0014039999999999999,"input_cached_usd":0.0,"output_usd":0.0010400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5411532080033794}
{"item_id":"senseval2.d002.s011.t020","gold_sense_keys":["soul%1:18:00::"],"candidates":[{"index":1,"sense_key":"soul%1:18:00::","synset_id":"soul.n.01"},{"index":2,"sense_key":"soul%1:03:00::","synset_id":"person.n.01"},{"index":3,"sense_key":"soul%1:12:00::","synset_id":"soul.n.03"},{"index":4,"sense_key":"soul%1:04:00::","synset_id":"soul.n.04"},{"index":5,"sense_key":"soul%1:10:00::","synset_id":"soul.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"soul%1:18:00::","call_ids":["senseval2.d002.s011.t020__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"soul%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":780,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00168,"input_uncached_usd":0.00156,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5767598750535399}
{"item_id":"senseval2.d002.s011.t021","gold_sense_keys":["child%1:18:00::"],"candidates":[{"index":1,"sense_key":"child%1:18:00::","synset_id":"child.n.01"},{"index":2,"sense_key":"child%1:18:01::","synset_id":"child.n.02"},{"index":3,"sense_key":"child%1:18:06::","synset_id":"child.n.03"},{"index":4,"sense_key":"child%1:18:07::","synset_id":"child.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"child%1:18:00::","call_ids":["senseval2.d002.s011.t021__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"child%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":751,"cached_input_tokens":0,"output_tokens":164,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003142,"input_uncached_usd":0.0015019999999999999,"input_cached_usd":0.0,"output_usd":0.0016400000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.150499999988824}
{"item_id":"senseval2.d002.s012.t001","gold_sense_keys":["parent%1:18:00::"],"candidates":[{"index":1,"sense_key":"parent%1:18:00::","synset_id":"parent.n.01"},{"index":2,"sense_key":"parent%1:03:00::","synset_id":"parent.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"parent%1:18:00::","call_ids":["senseval2.d002.s012.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"parent%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":604,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001328,"input_uncached_usd":0.0012079999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0497385000344366}
{"item_id":"senseval2.d002.s012.t002","gold_sense_keys":["find%2:31:10::"],"candidates":[{"index":1,"sense_key":"find%2:40:02::","synset_id":"find.v.01"},{"index":2,"sense_key":"find%2:39:02::","synset_id":"detect.v.01"},{"index":3,"sense_key":"find%2:40:00::","synset_id":"find.v.03"},{"index":4,"sense_key":"find%2:32:00::","synset_id":"determine.v.01"},{"index":5,"sense_key":"find%2:31:10::","synset_id":"find.v.05"},{"index":6,"sense_key":"find%2:39:01::","synset_id":"witness.v.02"},{"index":7,"sense_key":"find%2:40:01::","synset_id":"line_up.v.02"},{"index":8,"sense_key":"find%2:36:00::","synset_id":"discover.v.03"},{"index":9,"sense_key":"find%2:31:09::","synset_id":"discover.v.04"},{"index":10,"sense_key":"find%2:40:03::","synset_id":"find.v.10"},{"index":11,"sense_key":"find%2:32:01::","synset_id":"rule.v.04"},{"index":12,"sense_key":"find%2:30:00::","synset_id":"receive.v.02"},{"index":13,"sense_key":"find%2:39:05::","synset_id":"find.v.13"},{"index":14,"sense_key":"find%2:40:15::","synset_id":"recover.v.01"},{"index":15,"sense_key":"find%2:38:10::","synset_id":"find.v.15"},{"index":16,"sense_key":"find%2:30:13::","synset_id":"find_oneself.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"find%2:31:10::","call_ids":["senseval2.d002.s012.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"find%2:31:10::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1648,"cached_input_tokens":0,"output_tokens":84,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004136,"input_uncached_usd":0.003296,"input_cached_usd":0.0,"output_usd":0.00084,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2789014159934595}
{"item_id":"senseval2.d002.s012.t003","gold_sense_keys":["romantic%5:00:00:impractical:00"],"candidates":[{"index":1,"sense_key":"romantic%3:01:00::","synset_id":"romantic.a.01"},{"index":2,"sense_key":"romantic%5:00:01:loving:00","synset_id":"amatory.s.01"},{"index":3,"sense_key":"romantic%5:00:00:impractical:00","synset_id":"quixotic.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"romantic%5:00:00:impractical:00","call_ids":["senseval2.d002.s012.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"romantic%5:00:00:impractical:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":747,"cached_input_tokens":0,"output_tokens":119,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0026839999999999998,"input_uncached_usd":0.0014939999999999999,"input_cached_usd":0.0,"output_usd":0.00119,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7426327919820324}
{"item_id":"senseval2.d002.s012.t004","gold_sense_keys":["so%4:02:02::"],"candidates":[{"index":1,"sense_key":"so%4:02:02::","synset_id":"so.r.01"},{"index":2,"sense_key":"so%4:02:06::","synset_id":"so.r.02"},{"index":3,"sense_key":"so%4:02:05::","synset_id":"so.r.03"},{"index":4,"sense_key":"so%4:02:07::","synset_id":"so.r.04"},{"index":5,"sense_key":"so%4:02:08::","synset_id":"so.r.05"},{"index":6,"sense_key":"so%4:02:04::","synset_id":"thus.r.02"},{"index":7,"sense_key":"so%4:02:03::","synset_id":"so.r.07"},{"index":8,"sense_key":"so%4:02:09::","synset_id":"then.r.01"},{"index":9,"sense_key":"so%4:02:10::","synset_id":"therefore.r.01"},{"index":10,"sense_key":"so%4:02:00::","synset_id":"indeed.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"so%4:02:02::","call_ids":["senseval2.d002.s012.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"so%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1159,"cached_input_tokens":0,"output_tokens":81,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0031279999999999997,"input_uncached_usd":0.0023179999999999997,"input_cached_usd":0.0,"output_usd":0.0008100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4953307500109076}
{"item_id":"senseval2.d002.s012.t005","gold_sense_keys":["attractive%5:00:00:appealing:00"],"candidates":[{"index":1,"sense_key":"attractive%3:00:01::","synset_id":"attractive.a.01"},{"index":2,"sense_key":"attractive%5:00:00:appealing:00","synset_id":"attractive.s.01"},{"index":3,"sense_key":"attractive%3:00:02::","synset_id":"attractive.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"attractive%5:00:00:appealing:00","call_ids":["senseval2.d002.s012.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"attractive%5:00:00:appealing:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":676,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001472,"input_uncached_usd":0.001352,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.7206731659825891}
{"item_id":"senseval2.d002.s013.t001","gold_sense_keys":["same%3:00:02::"],"candidates":[{"index":1,"sense_key":"same%3:00:02::","synset_id":"same.a.01"},{"index":2,"sense_key":"same%3:00:00::","synset_id":"same.a.02"},{"index":3,"sense_key":"same%3:00:04::","synset_id":"like.a.02"},{"index":4,"sense_key":"same%5:00:00:unchanged:00","synset_id":"same.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"same%3:00:02::","call_ids":["senseval2.d002.s013.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"same%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":726,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002062,"input_uncached_usd":0.001452,"input_cached_usd":0.0,"output_usd":0.0006100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4073783749481663}
{"item_id":"senseval2.d002.s013.t002","gold_sense_keys":["parent%1:18:00::"],"candidates":[{"index":1,"sense_key":"parent%1:18:00::","synset_id":"parent.n.01"},{"index":2,"sense_key":"parent%1:03:00::","synset_id":"parent.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"parent%1:18:00::","call_ids":["senseval2.d002.s013.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"parent%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":611,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001342,"input_uncached_usd":0.001222,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8692833330715075}
{"item_id":"senseval2.d002.s013.t003","gold_sense_keys":["want%2:37:00::"],"candidates":[{"index":1,"sense_key":"want%2:37:00::","synset_id":"desire.v.01"},{"index":2,"sense_key":"want%2:34:00::","synset_id":"want.v.02"},{"index":3,"sense_key":"want%2:35:00::","synset_id":"want.v.03"},{"index":4,"sense_key":"want%2:31:01::","synset_id":"want.v.04"},{"index":5,"sense_key":"want%2:42:01::","synset_id":"want.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"want%2:37:00::","call_ids":["senseval2.d002.s013.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"want%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":772,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0016640000000000001,"input_uncached_usd":0.001544,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.3587322500534356}
{"item_id":"senseval2.d002.s013.t004","gold_sense_keys":["child%1:18:01::"],"candidates":[{"index":1,"sense_key":"child%1:18:00::","synset_id":"child.n.01"},{"index":2,"sense_key":"child%1:18:01::","synset_id":"child.n.02"},{"index":3,"sense_key":"child%1:18:06::","synset_id":"child.n.03"},{"index":4,"sense_key":"child%1:18:07::","synset_id":"child.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"child%1:18:01::","call_ids":["senseval2.d002.s013.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"child%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":742,"cached_input_tokens":0,"output_tokens":133,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0028139999999999997,"input_uncached_usd":0.0014839999999999999,"input_cached_usd":0.0,"output_usd":0.00133,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.3366144170286134}
{"item_id":"senseval2.d002.s013.t005","gold_sense_keys":["get%2:40:00::"],"candidates":[{"index":1,"sense_key":"get%2:40:00::","synset_id":"get.v.01"},{"index":2,"sense_key":"get%2:30:00::","synset_id":"become.v.01"},{"index":3,"sense_key":"get%2:30:02::","synset_id":"get.v.03"},{"index":4,"sense_key":"get%2:30:03::","synset_id":"receive.v.02"},{"index":5,"sense_key":"get%2:38:00::","synset_id":"arrive.v.01"},{"index":6,"sense_key":"get%2:35:03::","synset_id":"bring.v.04"},{"index":7,"sense_key":"get%2:39:00::","synset_id":"experience.v.03"},{"index":8,"sense_key":"get%2:33:00::","synset_id":"pay_back.v.02"},{"index":9,"sense_key":"get%2:40:01::","synset_id":"have.v.17"},{"index":10,"sense_key":"get%2:32:00::","synset_id":"induce.v.02"},{"index":11,"sense_key":"get%2:35:09::","synset_id":"get.v.11"},{"index":12,"sense_key":"get%2:29:01::","synset_id":"grow.v.08"},{"index":13,"sense_key":"get%2:29:00::","synset_id":"contract.v.04"},{"index":14,"sense_key":"get%2:32:03::","synset_id":"get.v.14"},{"index":15,"sense_key":"get%2:30:01::","synset_id":"make.v.02"},{"index":16,"sense_key":"get%2:32:08::","synset_id":"drive.v.11"},{"index":17,"sense_key":"get%2:31:09::","synset_id":"catch.v.18"},{"index":18,"sense_key":"get%2:35:00::","synset_id":"catch.v.07"},{"index":19,"sense_key":"get%2:35:10::","synset_id":"get.v.19"},{"index":20,"sense_key":"get%2:31:03::","synset_id":"get.v.20"},{"index":21,"sense_key":"get%2:40:14::","synset_id":"get.v.21"},{"index":22,"sense_key":"get%2:40:13::","synset_id":"get.v.22"},{"index":23,"sense_key":"get%2:39:13::","synset_id":"catch.v.21"},{"index":24,"sense_key":"get%2:39:09::","synset_id":"catch.v.22"},{"index":25,"sense_key":"get%2:39:14::","synset_id":"get.v.25"},{"index":26,"sense_key":"get%2:38:15::","synset_id":"scram.v.01"},{"index":27,"sense_key":"get%2:38:12::","synset_id":"get.v.27"},{"index":28,"sense_key":"get%2:37:12::","synset_id":"get.v.28"},{"index":29,"sense_key":"get%2:37:11::","synset_id":"get.v.29"},{"index":30,"sense_key":"get%2:36:14::","synset_id":"catch.v.24"},{"index":31,"sense_key":"get%2:36:00::","synset_id":"draw.v.15"},{"index":32,"sense_key":"get%2:35:12::","synset_id":"get.v.32"},{"index":33,"sense_key":"get%2:31:10::","synset_id":"perplex.v.01"},{"index":34,"sense_key":"get%2:30:12::","synset_id":"get_down.v.07"},{"index":35,"sense_key":"get%2:29:11::","synset_id":"suffer.v.02"},{"index":36,"sense_key":"get%2:29:08::","synset_id":"beget.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"get%2:40:00::","call_ids":["senseval2.d002.s013.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"get%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2847,"cached_input_tokens":0,"output_tokens":182,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.007514,"input_uncached_usd":0.005693999999999999,"input_cached_usd":0.0,"output_usd":0.0018200000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.316851459094323}
{"item_id":"senseval2.d002.s013.t006","gold_sense_keys":["decent%5:00:00:sufficient:00"],"candidates":[{"index":1,"sense_key":"decent%5:00:00:respectable:00","synset_id":"decent.s.01"},{"index":2,"sense_key":"decent%5:00:00:proper:00","synset_id":"becoming.s.01"},{"index":3,"sense_key":"decent%3:00:00::","synset_id":"decent.a.03"},{"index":4,"sense_key":"decent%5:00:00:sufficient:00","synset_id":"adequate.s.01"},{"index":5,"sense_key":"decent%5:00:02:modest:01","synset_id":"decent.s.04"},{"index":6,"sense_key":"decent%5:00:00:modest:01","synset_id":"decent.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"decent%5:00:00:sufficient:00","call_ids":["senseval2.d002.s013.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"decent%5:00:00:sufficient:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":909,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002368,"input_uncached_usd":0.001818,"input_cached_usd":0.0,"output_usd":0.00055,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4553263749694452}
{"item_id":"senseval2.d002.s013.t007","gold_sense_keys":["education%1:04:00::"],"candidates":[{"index":1,"sense_key":"education%1:04:00::","synset_id":"education.n.01"},{"index":2,"sense_key":"education%1:09:00::","synset_id":"education.n.02"},{"index":3,"sense_key":"education%1:09:01::","synset_id":"education.n.03"},{"index":4,"sense_key":"education%1:04:01::","synset_id":"education.n.04"},{"index":5,"sense_key":"education%1:07:01::","synset_id":"education.n.05"},{"index":6,"sense_key":"education%1:14:00::","synset_id":"department_of_education.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"education%1:04:00::","call_ids":["senseval2.d002.s013.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"education%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":895,"cached_input_tokens":0,"output_tokens":151,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0033,"input_uncached_usd":0.00179,"input_cached_usd":0.0,"output_usd":0.00151,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.3431198749458417}
{"item_id":"senseval2.d002.s013.t013","gold_sense_keys":["know%2:31:01::"],"candidates":[{"index":1,"sense_key":"know%2:31:01::","synset_id":"know.v.01"},{"index":2,"sense_key":"know%2:31:03::","synset_id":"know.v.02"},{"index":3,"sense_key":"know%2:31:02::","synset_id":"know.v.03"},{"index":4,"sense_key":"know%2:31:00::","synset_id":"know.v.04"},{"index":5,"sense_key":"know%2:31:04::","synset_id":"know.v.05"},{"index":6,"sense_key":"know%2:31:06::","synset_id":"acknowledge.v.06"},{"index":7,"sense_key":"know%2:31:15::","synset_id":"know.v.07"},{"index":8,"sense_key":"know%2:35:00::","synset_id":"sleep_together.v.01"},{"index":9,"sense_key":"know%2:31:11::","synset_id":"know.v.09"},{"index":10,"sense_key":"know%2:31:12::","synset_id":"know.v.10"},{"index":11,"sense_key":"know%2:31:14::","synset_id":"know.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"know%2:31:01::","call_ids":["senseval2.d002.s013.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"know%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1226,"cached_input_tokens":0,"output_tokens":102,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003472,"input_uncached_usd":0.0024519999999999998,"input_cached_usd":0.0,"output_usd":0.00102,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.597962083062157}
{"item_id":"senseval2.d002.s013.t014","gold_sense_keys":["demand%2:42:00::"],"candidates":[{"index":1,"sense_key":"demand%2:32:00::","synset_id":"demand.v.01"},{"index":2,"sense_key":"demand%2:42:00::","synset_id":"necessitate.v.01"},{"index":3,"sense_key":"demand%2:32:02::","synset_id":"demand.v.03"},{"index":4,"sense_key":"demand%2:32:09::","synset_id":"demand.v.04"},{"index":5,"sense_key":"demand%2:32:05::","synset_id":"demand.v.05"},{"index":6,"sense_key":"demand%2:32:06::","synset_id":"demand.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"demand%2:42:00::","call_ids":["senseval2.d002.s013.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"demand%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":794,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002238,"input_uncached_usd":0.001588,"input_cached_usd":0.0,"output_usd":0.0006500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.330018916982226}
{"item_id":"senseval2.d002.s014.t000","gold_sense_keys":["commitment%1:04:00::"],"candidates":[{"index":1,"sense_key":"commitment%1:07:01::","synset_id":"committedness.n.01"},{"index":2,"sense_key":"commitment%1:04:00::","synset_id":"commitment.n.02"},{"index":3,"sense_key":"commitment%1:04:02::","synset_id":"commitment.n.03"},{"index":4,"sense_key":"commitment%1:10:00::","synset_id":"commitment.n.04"},{"index":5,"sense_key":"commitment%1:04:01::","synset_id":"commitment.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"commitment%1:04:00::","call_ids":["senseval2.d002.s014.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"commitment%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":853,"cached_input_tokens":0,"output_tokens":113,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002836,"input_uncached_usd":0.0017059999999999998,"input_cached_usd":0.0,"output_usd":0.0011300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2154677079524845}
{"item_id":"senseval2.d002.s014.t002","gold_sense_keys":["survive%2:42:02::"],"candidates":[{"index":1,"sense_key":"survive%2:42:01::","synset_id":"survive.v.01"},{"index":2,"sense_key":"survive%2:42:02::","synset_id":"survive.v.02"},{"index":3,"sense_key":"survive%2:42:00::","synset_id":"exist.v.02"},{"index":4,"sense_key":"survive%2:42:03::","synset_id":"outlive.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"survive%2:42:02::","call_ids":["senseval2.d002.s014.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"survive%2:42:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":827,"cached_input_tokens":0,"output_tokens":178,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003434,"input_uncached_usd":0.0016539999999999999,"input_cached_usd":0.0,"output_usd":0.0017800000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8780238330364227}
{"item_id":"senseval2.d002.s014.t003","gold_sense_keys":["adolescent%3:01:00::","adolescent%5:00:00:immature:01"],"candidates":[{"index":1,"sense_key":"adolescent%3:01:00::","synset_id":"adolescent.a.01"},{"index":2,"sense_key":"adolescent%5:00:00:young:00","synset_id":"adolescent.s.01"},{"index":3,"sense_key":"adolescent%5:00:00:immature:02","synset_id":"adolescent.s.02"},{"index":4,"sense_key":"adolescent%5:00:00:immature:01","synset_id":"adolescent.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"adolescent%5:00:00:young:00","call_ids":["senseval2.d002.s014.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"adolescent%5:00:00:young:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":808,"cached_input_tokens":0,"output_tokens":280,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004416000000000001,"input_uncached_usd":0.001616,"input_cached_usd":0.0,"output_usd":0.0028000000000000004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.8680619589285925}
{"item_id":"senseval2.d002.s014.t004","gold_sense_keys":["illiteracy%1:09:00::"],"candidates":[{"index":1,"sense_key":"illiteracy%1:09:01::","synset_id":"illiteracy.n.01"},{"index":2,"sense_key":"illiteracy%1:09:00::","synset_id":"illiteracy.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"illiteracy%1:09:00::","call_ids":["senseval2.d002.s014.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"illiteracy%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":626,"cached_input_tokens":0,"output_tokens":106,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002312,"input_uncached_usd":0.0012519999999999999,"input_cached_usd":0.0,"output_usd":0.0010600000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7753974170191213}
{"item_id":"senseval2.d002.s015.t000","gold_sense_keys":["american%3:01:01::"],"candidates":[{"index":1,"sense_key":"american%3:01:01::","synset_id":"american.a.01"},{"index":2,"sense_key":"american%3:01:00::","synset_id":"american.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"american%3:01:01::","call_ids":["senseval2.d002.s015.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"american%3:01:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":635,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00139,"input_uncached_usd":0.0012699999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0903241659980267}
{"item_id":"senseval2.d002.s015.t001","gold_sense_keys":["education%1:04:00::"],"candidates":[{"index":1,"sense_key":"education%1:04:00::","synset_id":"education.n.01"},{"index":2,"sense_key":"education%1:09:00::","synset_id":"education.n.02"},{"index":3,"sense_key":"education%1:09:01::","synset_id":"education.n.03"},{"index":4,"sense_key":"education%1:04:01::","synset_id":"education.n.04"},{"index":5,"sense_key":"education%1:07:01::","synset_id":"education.n.05"},{"index":6,"sense_key":"education%1:14:00::","synset_id":"department_of_education.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"education%1:04:00::","call_ids":["senseval2.d002.s015.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"education%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":915,"cached_input_tokens":0,"output_tokens":109,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00292,"input_uncached_usd":0.0018299999999999998,"input_cached_usd":0.0,"output_usd":0.00109,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1292754160240293}
{"item_id":"senseval2.d002.s015.t002","gold_sense_keys":["future%1:28:00::"],"candidates":[{"index":1,"sense_key":"future%1:28:00::","synset_id":"future.n.01"},{"index":2,"sense_key":"future%1:10:00::","synset_id":"future.n.02"},{"index":3,"sense_key":"future%1:06:00::","synset_id":"future.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"future%1:28:00::","call_ids":["senseval2.d002.s015.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"future%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":662,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001444,"input_uncached_usd":0.0013239999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.118343791924417}
{"item_id":"senseval2.d002.s015.t003","gold_sense_keys":["determine%2:31:02::"],"candidates":[{"index":1,"sense_key":"determine%2:32:00::","synset_id":"determine.v.01"},{"index":2,"sense_key":"determine%2:31:02::","synset_id":"determine.v.02"},{"index":3,"sense_key":"determine%2:31:01::","synset_id":"determine.v.03"},{"index":4,"sense_key":"determine%2:32:03::","synset_id":"specify.v.02"},{"index":5,"sense_key":"determine%2:31:00::","synset_id":"decide.v.01"},{"index":6,"sense_key":"determine%2:31:03::","synset_id":"determine.v.06"},{"index":7,"sense_key":"determine%2:32:05::","synset_id":"settle.v.03"},{"index":8,"sense_key":"determine%2:32:01::","synset_id":"determine.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"determine%2:31:02::","call_ids":["senseval2.d002.s015.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"determine%2:31:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1070,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00277,"input_uncached_usd":0.00214,"input_cached_usd":0.0,"output_usd":0.00063,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2082845410332084}
{"item_id":"senseval2.d002.s015.t004","gold_sense_keys":["degree%1:07:00::","degree%1:26:01::"],"candidates":[{"index":1,"sense_key":"degree%1:07:00::","synset_id":"degree.n.01"},{"index":2,"sense_key":"degree%1:26:01::","synset_id":"degree.n.02"},{"index":3,"sense_key":"degree%1:10:00::","synset_id":"academic_degree.n.01"},{"index":4,"sense_key":"degree%1:23:00::","synset_id":"degree.n.04"},{"index":5,"sense_key":"degree%1:09:00::","synset_id":"degree.n.05"},{"index":6,"sense_key":"degree%1:23:03::","synset_id":"degree.n.06"},{"index":7,"sense_key":"degree%1:07:01::","synset_id":"degree.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"degree%1:07:00::","call_ids":["senseval2.d002.s015.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"degree%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":968,"cached_input_tokens":0,"output_tokens":263,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004566000000000001,"input_uncached_usd":0.001936,"input_cached_usd":0.0,"output_usd":0.0026300000000000004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.7810210419120267}
{"item_id":"senseval2.d002.s015.t005","gold_sense_keys":["allow%2:41:00::","allow%2:42:00::"],"candidates":[{"index":1,"sense_key":"allow%2:41:00::","synset_id":"let.v.01"},{"index":2,"sense_key":"allow%2:32:00::","synset_id":"permit.v.01"},{"index":3,"sense_key":"allow%2:40:00::","synset_id":"allow.v.03"},{"index":4,"sense_key":"allow%2:31:01::","synset_id":"allow.v.04"},{"index":5,"sense_key":"allow%2:42:00::","synset_id":"leave.v.06"},{"index":6,"sense_key":"allow%2:31:00::","synset_id":"allow.v.06"},{"index":7,"sense_key":"allow%2:42:07::","synset_id":"admit.v.05"},{"index":8,"sense_key":"allow%2:41:01::","synset_id":"give_up.v.11"},{"index":9,"sense_key":"allow%2:40:01::","synset_id":"allow.v.09"},{"index":10,"sense_key":"allow%2:32:06::","synset_id":"allow.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"allow%2:41:00::","call_ids":["senseval2.d002.s015.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"allow%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1251,"cached_input_tokens":0,"output_tokens":114,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0036420000000000003,"input_uncached_usd":0.002502,"input_cached_usd":0.0,"output_usd":0.0011400000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6620871250052005}
{"item_id":"senseval2.d002.s015.t007","gold_sense_keys":["prevail%2:33:00::"],"candidates":[{"index":1,"sense_key":"prevail%2:42:02::","synset_id":"predominate.v.01"},{"index":2,"sense_key":"prevail%2:42:01::","synset_id":"prevail.v.02"},{"index":3,"sense_key":"prevail%2:42:00::","synset_id":"prevail.v.03"},{"index":4,"sense_key":"prevail%2:33:00::","synset_id":"prevail.v.04"},{"index":5,"sense_key":"prevail%2:32:02::","synset_id":"prevail.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"prevail%2:33:00::","call_ids":["senseval2.d002.s015.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"prevail%2:33:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":840,"cached_input_tokens":0,"output_tokens":128,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00296,"input_uncached_usd":0.0016799999999999999,"input_cached_usd":0.0,"output_usd":0.00128,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6990948329912499}
{"item_id":"senseval2.d002.s015.t008","gold_sense_keys":["illusion%1:09:00::"],"candidates":[{"index":1,"sense_key":"illusion%1:09:01::","synset_id":"illusion.n.01"},{"index":2,"sense_key":"illusion%1:09:00::","synset_id":"illusion.n.02"},{"index":3,"sense_key":"illusion%1:04:00::","synset_id":"delusion.n.03"},{"index":4,"sense_key":"illusion%1:04:02::","synset_id":"magic_trick.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"illusion%1:09:00::","call_ids":["senseval2.d002.s015.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"illusion%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":762,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001644,"input_uncached_usd":0.001524,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4187677920563146}
{"item_id":"senseval2.d002.s015.t010","gold_sense_keys":["share%2:42:00::"],"candidates":[{"index":1,"sense_key":"share%2:42:00::","synset_id":"share.v.01"},{"index":2,"sense_key":"share%2:40:02::","synset_id":"share.v.02"},{"index":3,"sense_key":"share%2:40:01::","synset_id":"partake.v.02"},{"index":4,"sense_key":"share%2:40:00::","synset_id":"share.v.04"},{"index":5,"sense_key":"share%2:32:00::","synset_id":"share.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"share%2:42:00::","call_ids":["senseval2.d002.s015.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"share%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":739,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002038,"input_uncached_usd":0.001478,"input_cached_usd":0.0,"output_usd":0.0005600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3700647089863196}
{"item_id":"senseval2.d002.s016.t000","gold_sense_keys":["education%1:04:00::","education%1:04:01::"],"candidates":[{"index":1,"sense_key":"education%1:04:00::","synset_id":"education.n.01"},{"index":2,"sense_key":"education%1:09:00::","synset_id":"education.n.02"},{"index":3,"sense_key":"education%1:09:01::","synset_id":"education.n.03"},{"index":4,"sense_key":"education%1:04:01::","synset_id":"education.n.04"},{"index":5,"sense_key":"education%1:07:01::","synset_id":"education.n.05"},{"index":6,"sense_key":"education%1:14:00::","synset_id":"department_of_education.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"education%1:04:01::","call_ids":["senseval2.d002.s016.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"education%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":899,"cached_input_tokens":0,"output_tokens":115,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002948,"input_uncached_usd":0.001798,"input_cached_usd":0.0,"output_usd":0.0011500000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6920543330488726}
{"item_id":"senseval2.d002.s016.t001","gold_sense_keys":["establishment%1:14:00::","establishment%1:14:02::"],"candidates":[{"index":1,"sense_key":"establishment%1:04:00::","synset_id":"constitution.n.02"},{"index":2,"sense_key":"establishment%1:14:00::","synset_id":"institution.n.01"},{"index":3,"sense_key":"establishment%1:14:02::","synset_id":"administration.n.02"},{"index":4,"sense_key":"establishment%1:06:00::","synset_id":"establishment.n.04"},{"index":5,"sense_key":"establishment%1:14:01::","synset_id":"establishment.n.05"},{"index":6,"sense_key":"establishment%1:22:00::","synset_id":"establishment.n.06"},{"index":7,"sense_key":"establishment%1:09:00::","synset_id":"establishment.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"establishment%1:14:02::","call_ids":["senseval2.d002.s016.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"establishment%1:14:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":927,"cached_input_tokens":0,"output_tokens":131,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003164,"input_uncached_usd":0.001854,"input_cached_usd":0.0,"output_usd":0.0013100000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7308382500195876}
{"item_id":"senseval2.d002.s016.t002","gold_sense_keys":["fight%2:33:01::"],"candidates":[{"index":1,"sense_key":"fight%2:33:00::","synset_id":"contend.v.06"},{"index":2,"sense_key":"fight%2:33:01::","synset_id":"fight.v.02"},{"index":3,"sense_key":"fight%2:41:00::","synset_id":"fight.v.03"},{"index":4,"sense_key":"fight%2:41:10::","synset_id":"crusade.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"fight%2:33:01::","call_ids":["senseval2.d002.s016.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"fight%2:33:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":842,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002224,"input_uncached_usd":0.001684,"input_cached_usd":0.0,"output_usd":0.00054,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1321456669829786}
{"item_id":"senseval2.d002.s017.t000","gold_sense_keys":["reason%1:09:01::","reason%1:10:00::","reason%1:10:01::","reason%1:16:00::"],"candidates":[{"index":1,"sense_key":"reason%1:16:00::","synset_id":"reason.n.01"},{"index":2,"sense_key":"reason%1:10:01::","synset_id":"reason.n.02"},{"index":3,"sense_key":"reason%1:09:00::","synset_id":"reason.n.03"},{"index":4,"sense_key":"reason%1:26:00::","synset_id":"rationality.n.01"},{"index":5,"sense_key":"reason%1:10:00::","synset_id":"cause.n.02"},{"index":6,"sense_key":"reason%1:09:01::","synset_id":"reason.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"reason%1:10:01::","call_ids":["senseval2.d002.s017.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"reason%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":860,"cached_input_tokens":0,"output_tokens":260,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00432,"input_uncached_usd":0.00172,"input_cached_usd":0.0,"output_usd":0.0026000000000000003,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.62060595897492}
{"item_id":"senseval2.d002.s017.t002","gold_sense_keys":["simple%3:00:02::","simple%5:00:00:easy:01"],"candidates":[{"index":1,"sense_key":"simple%3:00:02::","synset_id":"simple.a.01"},{"index":2,"sense_key":"simple%5:00:00:easy:01","synset_id":"elementary.s.01"},{"index":3,"sense_key":"simple%5:00:02:plain:01","synset_id":"bare.s.04"},{"index":4,"sense_key":"simple%5:00:00:naive:00","synset_id":"childlike.s.02"},{"index":5,"sense_key":"simple%5:00:00:retarded:00","synset_id":"dim-witted.s.01"},{"index":6,"sense_key":"simple%3:00:01::","synset_id":"simple.a.06"},{"index":7,"sense_key":"simple%5:00:00:plain:01","synset_id":"simple.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"simple%3:00:02::","call_ids":["senseval2.d002.s017.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"simple%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":920,"cached_input_tokens":0,"output_tokens":109,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00293,"input_uncached_usd":0.0018399999999999998,"input_cached_usd":0.0,"output_usd":0.00109,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9363841660087928}
{"item_id":"senseval2.d002.s017.t003","gold_sense_keys":["reason%1:09:01::","reason%1:10:00::","reason%1:10:01::","reason%1:16:00::"],"candidates":[{"index":1,"sense_key":"reason%1:16:00::","synset_id":"reason.n.01"},{"index":2,"sense_key":"reason%1:10:01::","synset_id":"reason.n.02"},{"index":3,"sense_key":"reason%1:09:00::","synset_id":"reason.n.03"},{"index":4,"sense_key":"reason%1:26:00::","synset_id":"rationality.n.01"},{"index":5,"sense_key":"reason%1:10:00::","synset_id":"cause.n.02"},{"index":6,"sense_key":"reason%1:09:01::","synset_id":"reason.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"reason%1:10:01::","call_ids":["senseval2.d002.s017.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"reason%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":859,"cached_input_tokens":0,"output_tokens":198,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003698,"input_uncached_usd":0.001718,"input_cached_usd":0.0,"output_usd":0.00198,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.687080000061542}
{"item_id":"senseval2.d002.s018.t000","gold_sense_keys":["progressive%3:00:01::"],"candidates":[{"index":1,"sense_key":"progressive%3:00:01::","synset_id":"progressive.a.01"},{"index":2,"sense_key":"progressive%5:00:00:liberal:00","synset_id":"progressive.s.01"},{"index":3,"sense_key":"progressive%3:00:02::","synset_id":"progressive.a.03"},{"index":4,"sense_key":"progressive%5:00:00:increasing:00","synset_id":"progressive.s.02"},{"index":5,"sense_key":"progressive%5:00:00:ordered:00","synset_id":"progressive.s.03"},{"index":6,"sense_key":"progressive%5:00:00:active:02","synset_id":"progressive.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"progressive%3:00:01::","call_ids":["senseval2.d002.s018.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"progressive%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":773,"cached_input_tokens":0,"output_tokens":50,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002046,"input_uncached_usd":0.0015459999999999998,"input_cached_usd":0.0,"output_usd":0.0005,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":4.910188458976336}
{"item_id":"senseval2.d002.s018.t001","gold_sense_keys":["education%1:04:00::"],"candidates":[{"index":1,"sense_key":"education%1:04:00::","synset_id":"education.n.01"},{"index":2,"sense_key":"education%1:09:00::","synset_id":"education.n.02"},{"index":3,"sense_key":"education%1:09:01::","synset_id":"education.n.03"},{"index":4,"sense_key":"education%1:04:01::","synset_id":"education.n.04"},{"index":5,"sense_key":"education%1:07:01::","synset_id":"education.n.05"},{"index":6,"sense_key":"education%1:14:00::","synset_id":"department_of_education.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"education%1:04:00::","call_ids":["senseval2.d002.s018.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"education%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":856,"cached_input_tokens":0,"output_tokens":103,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002742,"input_uncached_usd":0.001712,"input_cached_usd":0.0,"output_usd":0.00103,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6786982079502195}
{"item_id":"senseval2.d002.s018.t002","gold_sense_keys":["once%4:02:01::"],"candidates":[{"index":1,"sense_key":"once%4:02:00::","synset_id":"once.r.01"},{"index":2,"sense_key":"once%4:02:02::","synset_id":"once.r.02"},{"index":3,"sense_key":"once%4:02:01::","synset_id":"once.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"once%4:02:01::","call_ids":["senseval2.d002.s018.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"once%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":619,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001358,"input_uncached_usd":0.001238,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1825603749603033}
{"item_id":"senseval2.d002.s018.t003","gold_sense_keys":["call%2:32:02::"],"candidates":[{"index":1,"sense_key":"call%2:32:02::","synset_id":"name.v.01"},{"index":2,"sense_key":"call%2:32:00::","synset_id":"call.v.02"},{"index":3,"sense_key":"call%2:32:01::","synset_id":"call.v.03"},{"index":4,"sense_key":"call%2:32:09::","synset_id":"shout.v.02"},{"index":5,"sense_key":"call%2:32:05::","synset_id":"call.v.05"},{"index":6,"sense_key":"call%2:41:00::","synset_id":"visit.v.03"},{"index":7,"sense_key":"call%2:41:04::","synset_id":"call.v.07"},{"index":8,"sense_key":"call%2:32:13::","synset_id":"call.v.08"},{"index":9,"sense_key":"call%2:32:04::","synset_id":"call.v.09"},{"index":10,"sense_key":"call%2:32:03::","synset_id":"call.v.10"},{"index":11,"sense_key":"call%2:42:09::","synset_id":"call.v.11"},{"index":12,"sense_key":"call%2:41:14::","synset_id":"address.v.06"},{"index":13,"sense_key":"call%2:41:12::","synset_id":"call.v.13"},{"index":14,"sense_key":"call%2:40:09::","synset_id":"call.v.14"},{"index":15,"sense_key":"call%2:40:05::","synset_id":"bid.v.04"},{"index":16,"sense_key":"call%2:38:00::","synset_id":"call.v.16"},{"index":17,"sense_key":"call%2:32:06::","synset_id":"call.v.17"},{"index":18,"sense_key":"call%2:32:10::","synset_id":"predict.v.01"},{"index":19,"sense_key":"call%2:32:12::","synset_id":"call.v.19"},{"index":20,"sense_key":"call%2:32:07::","synset_id":"call.v.20"},{"index":21,"sense_key":"call%2:32:08::","synset_id":"call.v.21"},{"index":22,"sense_key":"call%2:32:11::","synset_id":"call.v.22"},{"index":23,"sense_key":"call%2:32:15::","synset_id":"call.v.23"},{"index":24,"sense_key":"call%2:32:14::","synset_id":"call.v.24"},{"index":25,"sense_key":"call%2:31:13::","synset_id":"call.v.25"},{"index":26,"sense_key":"call%2:31:00::","synset_id":"call.v.26"},{"index":27,"sense_key":"call%2:31:05::","synset_id":"call.v.27"},{"index":28,"sense_key":"call%2:29:05::","synset_id":"call.v.28"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"call%2:32:02::","call_ids":["senseval2.d002.s018.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"call%2:32:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2088,"cached_input_tokens":0,"output_tokens":187,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.006046,"input_uncached_usd":0.004176,"input_cached_usd":0.0,"output_usd":0.0018700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.122302458039485}
{"item_id":"senseval2.d002.s018.t004","gold_sense_keys":["far%4:02:02::"],"candidates":[{"index":1,"sense_key":"far%4:02:02::","synset_id":"far.r.01"},{"index":2,"sense_key":"far%4:02:00::","synset_id":"far.r.02"},{"index":3,"sense_key":"far%4:02:04::","synset_id":"far.r.03"},{"index":4,"sense_key":"far%4:02:01::","synset_id":"far.r.04"},{"index":5,"sense_key":"far%4:02:05::","synset_id":"far.r.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"far%4:02:02::","call_ids":["senseval2.d002.s018.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"far%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":717,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001554,"input_uncached_usd":0.001434,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0436990830348805}
{"item_id":"senseval2.d002.s018.t005","gold_sense_keys":["more%4:02:00::"],"candidates":[{"index":1,"sense_key":"more%4:02:00::","synset_id":"more.r.01"},{"index":2,"sense_key":"more%4:02:01::","synset_id":"more.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"more%4:02:00::","call_ids":["senseval2.d002.s018.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"more%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":582,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001824,"input_uncached_usd":0.0011639999999999999,"input_cached_usd":0.0,"output_usd":0.0006600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2937089999904856}
{"item_id":"senseval2.d002.s018.t007","gold_sense_keys":["agreeable%3:00:00::"],"candidates":[{"index":1,"sense_key":"agreeable%3:00:00::","synset_id":"agreeable.a.01"},{"index":2,"sense_key":"agreeable%5:00:00:consistent:00","synset_id":"accordant.s.01"},{"index":3,"sense_key":"agreeable%5:00:00:accordant:00","synset_id":"agreeable.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"agreeable%3:00:00::","call_ids":["senseval2.d002.s018.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"agreeable%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":678,"cached_input_tokens":0,"output_tokens":119,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002546,"input_uncached_usd":0.001356,"input_cached_usd":0.0,"output_usd":0.00119,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7584032920422032}
{"item_id":"senseval2.d002.s018.t008","gold_sense_keys":["teacher%1:18:00::"],"candidates":[{"index":1,"sense_key":"teacher%1:18:00::","synset_id":"teacher.n.01"},{"index":2,"sense_key":"teacher%1:09:00::","synset_id":"teacher.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"teacher%1:18:00::","call_ids":["senseval2.d002.s018.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"teacher%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":567,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0012540000000000001,"input_uncached_usd":0.001134,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9994361250428483}
{"item_id":"senseval2.d002.s018.t009","gold_sense_keys":["disciplined%5:00:00:controlled:00"],"candidates":[{"index":1,"sense_key":"disciplined%5:00:00:controlled:00","synset_id":"disciplined.s.01"},{"index":2,"sense_key":"disciplined%5:00:00:trained:00","synset_id":"disciplined.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"disciplined%5:00:00:trained:00","call_ids":["senseval2.d002.s018.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"disciplined%5:00:00:trained:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":579,"cached_input_tokens":0,"output_tokens":148,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002638,"input_uncached_usd":0.001158,"input_cached_usd":0.0,"output_usd":0.0014800000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8541001670528203}
{"item_id":"senseval2.d002.s018.t010","gold_sense_keys":["instruction%1:04:01::"],"candidates":[{"index":1,"sense_key":"instruction%1:10:04::","synset_id":"direction.n.06"},{"index":2,"sense_key":"instruction%1:04:01::","synset_id":"education.n.01"},{"index":3,"sense_key":"instruction%1:04:00::","synset_id":"teaching.n.01"},{"index":4,"sense_key":"instruction%1:10:02::","synset_id":"instruction.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"instruction%1:04:01::","call_ids":["senseval2.d002.s018.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"instruction%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":751,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001622,"input_uncached_usd":0.0015019999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8302546249469742}
{"item_id":"senseval2.d002.s019.t000","gold_sense_keys":["nice%3:00:00::"],"candidates":[{"index":1,"sense_key":"nice%3:00:00::","synset_id":"nice.a.01"},{"index":2,"sense_key":"nice%5:00:00:respectable:00","synset_id":"decent.s.01"},{"index":3,"sense_key":"nice%5:00:02:precise:00","synset_id":"nice.s.02"},{"index":4,"sense_key":"nice%5:00:00:fastidious:00","synset_id":"dainty.s.04"},{"index":5,"sense_key":"nice%5:00:00:polite:00","synset_id":"courteous.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"nice%3:00:00::","call_ids":["senseval2.d002.s019.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"nice%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":852,"cached_input_tokens":0,"output_tokens":93,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002634,"input_uncached_usd":0.001704,"input_cached_usd":0.0,"output_usd":0.00093,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.665836124913767}
{"item_id":"senseval2.d002.s019.t001","gold_sense_keys":["teacher%1:18:00::"],"candidates":[{"index":1,"sense_key":"teacher%1:18:00::","synset_id":"teacher.n.01"},{"index":2,"sense_key":"teacher%1:09:00::","synset_id":"teacher.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"teacher%1:18:00::","call_ids":["senseval2.d002.s019.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"teacher%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":568,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001256,"input_uncached_usd":0.0011359999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4732099999673665}
{"item_id":"senseval2.d002.s019.t002","gold_sense_keys":["think%2:31:03::"],"candidates":[{"index":1,"sense_key":"think%2:31:01::","synset_id":"think.v.01"},{"index":2,"sense_key":"think%2:31:03::","synset_id":"think.v.02"},{"index":3,"sense_key":"think%2:31:00::","synset_id":"think.v.03"},{"index":4,"sense_key":"think%2:31:02::","synset_id":"remember.v.01"},{"index":5,"sense_key":"think%2:36:00::","synset_id":"think.v.05"},{"index":6,"sense_key":"think%2:31:09::","synset_id":"think.v.06"},{"index":7,"sense_key":"think%2:31:06::","synset_id":"intend.v.01"},{"index":8,"sense_key":"think%2:31:12::","synset_id":"think.v.08"},{"index":9,"sense_key":"think%2:31:13::","synset_id":"think.v.09"},{"index":10,"sense_key":"think%2:31:15::","synset_id":"think.v.10"},{"index":11,"sense_key":"think%2:31:14::","synset_id":"think.v.11"},{"index":12,"sense_key":"think%2:31:08::","synset_id":"think.v.12"},{"index":13,"sense_key":"think%2:30:00::","synset_id":"think.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"think%2:31:03::","call_ids":["senseval2.d002.s019.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"think%2:31:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1256,"cached_input_tokens":0,"output_tokens":198,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0044919999999999995,"input_uncached_usd":0.002512,"input_cached_usd":0.0,"output_usd":0.00198,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.745626708958298}
{"item_id":"senseval2.d002.s019.t003","gold_sense_keys":["personality%1:07:00::"],"candidates":[{"index":1,"sense_key":"personality%1:07:00::","synset_id":"personality.n.01"},{"index":2,"sense_key":"personality%1:18:00::","synset_id":"personality.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"personality%1:07:00::","call_ids":["senseval2.d002.s019.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"personality%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":598,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0013160000000000001,"input_uncached_usd":0.001196,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3649147920077667}
{"item_id":"senseval2.d002.s019.t004","gold_sense_keys":["development%1:22:02::"],"candidates":[{"index":1,"sense_key":"development%1:04:01::","synset_id":"development.n.01"},{"index":2,"sense_key":"development%1:22:02::","synset_id":"development.n.02"},{"index":3,"sense_key":"development%1:22:01::","synset_id":"growth.n.01"},{"index":4,"sense_key":"development%1:11:01::","synset_id":"development.n.04"},{"index":5,"sense_key":"development%1:04:02::","synset_id":"exploitation.n.01"},{"index":6,"sense_key":"development%1:15:00::","synset_id":"development.n.06"},{"index":7,"sense_key":"development%1:26:00::","synset_id":"development.n.07"},{"index":8,"sense_key":"development%1:22:00::","synset_id":"development.n.08"},{"index":9,"sense_key":"development%1:10:00::","synset_id":"development.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"development%1:22:02::","call_ids":["senseval2.d002.s019.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"development%1:22:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1131,"cached_input_tokens":0,"output_tokens":136,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003622,"input_uncached_usd":0.0022619999999999997,"input_cached_usd":0.0,"output_usd":0.00136,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.232870624982752}
{"item_id":"senseval2.d002.s019.t005","gold_sense_keys":["even%4:02:01::"],"candidates":[{"index":1,"sense_key":"even%4:02:00::","synset_id":"even.r.01"},{"index":2,"sense_key":"even%4:02:03::","synset_id":"even.r.02"},{"index":3,"sense_key":"even%4:02:01::","synset_id":"even.r.03"},{"index":4,"sense_key":"even%4:02:04::","synset_id":"even.r.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"even%4:02:01::","call_ids":["senseval2.d002.s019.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"even%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":694,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002108,"input_uncached_usd":0.001388,"input_cached_usd":0.0,"output_usd":0.00072,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.628499332931824}
{"item_id":"senseval2.d002.s019.t006","gold_sense_keys":["nice%3:00:00::"],"candidates":[{"index":1,"sense_key":"nice%3:00:00::","synset_id":"nice.a.01"},{"index":2,"sense_key":"nice%5:00:00:respectable:00","synset_id":"decent.s.01"},{"index":3,"sense_key":"nice%5:00:02:precise:00","synset_id":"nice.s.02"},{"index":4,"sense_key":"nice%5:00:00:fastidious:00","synset_id":"dainty.s.04"},{"index":5,"sense_key":"nice%5:00:00:polite:00","synset_id":"courteous.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"nice%3:00:00::","call_ids":["senseval2.d002.s019.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"nice%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":853,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001826,"input_uncached_usd":0.0017059999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8217827499611303}
{"item_id":"senseval2.d002.s019.t007","gold_sense_keys":["minimize%2:30:00::"],"candidates":[{"index":1,"sense_key":"minimize%2:30:00::","synset_id":"minimize.v.01"},{"index":2,"sense_key":"minimize%2:32:01::","synset_id":"understate.v.01"},{"index":3,"sense_key":"minimize%2:32:00::","synset_id":"minimize.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"minimize%2:32:01::","call_ids":["senseval2.d002.s019.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"minimize%2:32:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":634,"cached_input_tokens":0,"output_tokens":212,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0033880000000000004,"input_uncached_usd":0.001268,"input_cached_usd":0.0,"output_usd":0.0021200000000000004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.650761416996829}
{"item_id":"senseval2.d002.s019.t009","gold_sense_keys":["test%1:10:00::"],"candidates":[{"index":1,"sense_key":"test%1:09:02::","synset_id":"trial.n.02"},{"index":2,"sense_key":"test%1:04:02::","synset_id":"test.n.02"},{"index":3,"sense_key":"test%1:10:00::","synset_id":"examination.n.02"},{"index":4,"sense_key":"test%1:04:03::","synset_id":"test.n.04"},{"index":5,"sense_key":"test%1:04:00::","synset_id":"test.n.05"},{"index":6,"sense_key":"test%1:05:00::","synset_id":"test.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"test%1:10:00::","call_ids":["senseval2.d002.s019.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"test%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":891,"cached_input_tokens":0,"output_tokens":75,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002532,"input_uncached_usd":0.001782,"input_cached_usd":0.0,"output_usd":0.00075,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.801127374987118}
{"item_id":"senseval2.d002.s019.t010","gold_sense_keys":["often%4:02:02::"],"candidates":[{"index":1,"sense_key":"often%4:02:00::","synset_id":"frequently.r.01"},{"index":2,"sense_key":"often%4:02:01::","synset_id":"much.r.05"},{"index":3,"sense_key":"often%4:02:02::","synset_id":"often.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"often%4:02:02::","call_ids":["senseval2.d002.s019.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"often%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":620,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00201,"input_uncached_usd":0.00124,"input_cached_usd":0.0,"output_usd":0.0007700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8931884169578552}
{"item_id":"senseval2.d002.s019.t012","gold_sense_keys":["result%1:11:00::"],"candidates":[{"index":1,"sense_key":"result%1:19:00::","synset_id":"consequence.n.01"},{"index":2,"sense_key":"result%1:10:00::","synset_id":"solution.n.02"},{"index":3,"sense_key":"result%1:11:00::","synset_id":"result.n.03"},{"index":4,"sense_key":"result%1:10:01::","synset_id":"resultant_role.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"result%1:11:00::","call_ids":["senseval2.d002.s019.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"result%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":784,"cached_input_tokens":0,"output_tokens":137,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002938,"input_uncached_usd":0.001568,"input_cached_usd":0.0,"output_usd":0.0013700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.548992000054568}
{"item_id":"senseval2.d002.s020.t001","gold_sense_keys":["provide%2:40:00::"],"candidates":[{"index":1,"sense_key":"provide%2:40:00::","synset_id":"supply.v.01"},{"index":2,"sense_key":"provide%2:34:00::","synset_id":"provide.v.02"},{"index":3,"sense_key":"provide%2:32:00::","synset_id":"provide.v.03"},{"index":4,"sense_key":"provide%2:41:00::","synset_id":"put_up.v.02"},{"index":5,"sense_key":"provide%2:42:00::","synset_id":"leave.v.06"},{"index":6,"sense_key":"provide%2:40:08::","synset_id":"provide.v.06"},{"index":7,"sense_key":"provide%2:30:05::","synset_id":"provide.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"provide%2:40:00::","call_ids":["senseval2.d002.s020.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"provide%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":974,"cached_input_tokens":0,"output_tokens":90,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002848,"input_uncached_usd":0.0019479999999999999,"input_cached_usd":0.0,"output_usd":0.0009000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6459673750214279}
{"item_id":"senseval2.d002.s020.t002","gold_sense_keys":["teacher%1:18:00::"],"candidates":[{"index":1,"sense_key":"teacher%1:18:00::","synset_id":"teacher.n.01"},{"index":2,"sense_key":"teacher%1:09:00::","synset_id":"teacher.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"teacher%1:18:00::","call_ids":["senseval2.d002.s020.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"teacher%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":568,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001256,"input_uncached_usd":0.0011359999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9590574579779059}
{"item_id":"senseval2.d002.s020.t003","gold_sense_keys":["profession%1:04:00::","profession%1:14:00::"],"candidates":[{"index":1,"sense_key":"profession%1:14:00::","synset_id":"profession.n.01"},{"index":2,"sense_key":"profession%1:04:00::","synset_id":"profession.n.02"},{"index":3,"sense_key":"profession%1:10:00::","synset_id":"profession.n.03"},{"index":4,"sense_key":"profession%1:10:01::","synset_id":"profession.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"profession%1:04:00::","call_ids":["senseval2.d002.s020.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"profession%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":685,"cached_input_tokens":0,"output_tokens":81,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00218,"input_uncached_usd":0.00137,"input_cached_usd":0.0,"output_usd":0.0008100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4341960840392858}
{"item_id":"senseval2.d002.s020.t004","gold_sense_keys":["pass%2:41:08::"],"candidates":[{"index":1,"sense_key":"pass%2:38:00::","synset_id":"pass.v.01"},{"index":2,"sense_key":"pass%2:38:05::","synset_id":"travel_by.v.01"},{"index":3,"sense_key":"pass%2:41:02::","synset_id":"legislate.v.01"},{"index":4,"sense_key":"pass%2:38:03::","synset_id":"elapse.v.01"},{"index":5,"sense_key":"pass%2:40:00::","synset_id":"pass.v.05"},{"index":6,"sense_key":"pass%2:42:08::","synset_id":"run.v.03"},{"index":7,"sense_key":"pass%2:38:01::","synset_id":"pass.v.07"},{"index":8,"sense_key":"pass%2:30:00::","synset_id":"happen.v.01"},{"index":9,"sense_key":"pass%2:41:04::","synset_id":"pass.v.09"},{"index":10,"sense_key":"pass%2:42:00::","synset_id":"spend.v.01"},{"index":11,"sense_key":"pass%2:35:00::","synset_id":"guide.v.05"},{"index":12,"sense_key":"pass%2:32:01::","synset_id":"communicate.v.01"},{"index":13,"sense_key":"pass%2:30:01::","synset_id":"evanesce.v.01"},{"index":14,"sense_key":"pass%2:41:08::","synset_id":"pass.v.14"},{"index":15,"sense_key":"pass%2:42:04::","synset_id":"exceed.v.02"},{"index":16,"sense_key":"pass%2:41:07::","synset_id":"pass.v.16"},{"index":17,"sense_key":"pass%2:41:11::","synset_id":"pass.v.17"},{"index":18,"sense_key":"pass%2:40:02::","synset_id":"pass.v.18"},{"index":19,"sense_key":"pass%2:38:02::","synset_id":"sink.v.03"},{"index":20,"sense_key":"pass%2:40:01::","synset_id":"pass.v.20"},{"index":21,"sense_key":"pass%2:40:13::","synset_id":"fall.v.21"},{"index":22,"sense_key":"pass%2:38:04::","synset_id":"pass.v.22"},{"index":23,"sense_key":"pass%2:32:00::","synset_id":"authorize.v.01"},{"index":24,"sense_key":"pass%2:30:11::","synset_id":"die.v.01"},{"index":25,"sense_key":"pass%2:29:00::","synset_id":"excrete.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":14,"chosen_sense_key":"pass%2:41:08::","call_ids":["senseval2.d002.s020.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":14,"predicted_sense_key":"pass%2:41:08::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2084,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004288,"input_uncached_usd":0.004168,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6016027909936383}
{"item_id":"senseval2.d002.s020.t005","gold_sense_keys":["course%1:04:01::"],"candidates":[{"index":1,"sense_key":"course%1:04:01::","synset_id":"course.n.01"},{"index":2,"sense_key":"course%1:14:00::","synset_id":"course.n.02"},{"index":3,"sense_key":"course%1:15:00::","synset_id":"course.n.03"},{"index":4,"sense_key":"course%1:04:00::","synset_id":"course.n.04"},{"index":5,"sense_key":"course%1:17:00::","synset_id":"path.n.04"},{"index":6,"sense_key":"course%1:14:01::","synset_id":"class.n.02"},{"index":7,"sense_key":"course%1:13:00::","synset_id":"course.n.07"},{"index":8,"sense_key":"course%1:06:01::","synset_id":"course.n.08"},{"index":9,"sense_key":"course%1:06:00::","synset_id":"course.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"course%1:04:01::","call_ids":["senseval2.d002.s020.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"course%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1039,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002198,"input_uncached_usd":0.002078,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1698137499624863}
{"item_id":"senseval2.d002.s020.t006","gold_sense_keys":["educational%3:01:00::"],"candidates":[{"index":1,"sense_key":"educational%3:01:00::","synset_id":"educational.a.01"},{"index":2,"sense_key":"educational%5:00:00:instructive:00","synset_id":"educational.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"educational%3:01:00::","call_ids":["senseval2.d002.s020.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"educational%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":547,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001214,"input_uncached_usd":0.001094,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4879233749816194}
{"item_id":"senseval2.d002.s020.t008","gold_sense_keys":["educational%3:01:00::"],"candidates":[{"index":1,"sense_key":"educational%3:01:00::","synset_id":"educational.a.01"},{"index":2,"sense_key":"educational%5:00:00:instructive:00","synset_id":"educational.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"educational%3:01:00::","call_ids":["senseval2.d002.s020.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"educational%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":547,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001214,"input_uncached_usd":0.001094,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0923497918993235}
{"item_id":"senseval2.d002.s020.t009","gold_sense_keys":["philosophy%1:09:00::"],"candidates":[{"index":1,"sense_key":"philosophy%1:09:01::","synset_id":"doctrine.n.01"},{"index":2,"sense_key":"philosophy%1:09:00::","synset_id":"philosophy.n.02"},{"index":3,"sense_key":"philosophy%1:09:02::","synset_id":"philosophy.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"philosophy%1:09:00::","call_ids":["senseval2.d002.s020.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"philosophy%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":670,"cached_input_tokens":0,"output_tokens":130,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00264,"input_uncached_usd":0.00134,"input_cached_usd":0.0,"output_usd":0.0013000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6352106250124052}
{"item_id":"senseval2.d002.s021.t000","gold_sense_keys":["take%2:31:02::"],"candidates":[{"index":1,"sense_key":"take%2:41:04::","synset_id":"take.v.01"},{"index":2,"sense_key":"take%2:40:06::","synset_id":"take.v.02"},{"index":3,"sense_key":"take%2:38:09::","synset_id":"lead.v.01"},{"index":4,"sense_key":"take%2:35:00::","synset_id":"take.v.04"},{"index":5,"sense_key":"take%2:30:01::","synset_id":"assume.v.03"},{"index":6,"sense_key":"take%2:31:07::","synset_id":"take.v.06"},{"index":7,"sense_key":"take%2:38:10::","synset_id":"bring.v.01"},{"index":8,"sense_key":"take%2:40:00::","synset_id":"take.v.08"},{"index":9,"sense_key":"take%2:38:02::","synset_id":"take.v.09"},{"index":10,"sense_key":"take%2:31:01::","synset_id":"choose.v.01"},{"index":11,"sense_key":"take%2:40:05::","synset_id":"accept.v.02"},{"index":12,"sense_key":"take%2:41:00::","synset_id":"fill.v.04"},{"index":13,"sense_key":"take%2:31:03::","synset_id":"consider.v.03"},{"index":14,"sense_key":"take%2:42:00::","synset_id":"necessitate.v.01"},{"index":15,"sense_key":"take%2:39:00::","synset_id":"take.v.15"},{"index":16,"sense_key":"take%2:32:02::","synset_id":"film.v.01"},{"index":17,"sense_key":"take%2:30:00::","synset_id":"remove.v.01"},{"index":18,"sense_key":"take%2:34:00::","synset_id":"consume.v.02"},{"index":19,"sense_key":"take%2:31:09::","synset_id":"take.v.19"},{"index":20,"sense_key":"take%2:40:15::","synset_id":"take.v.20"},{"index":21,"sense_key":"take%2:40:01::","synset_id":"take.v.21"},{"index":22,"sense_key":"take%2:38:00::","synset_id":"assume.v.05"},{"index":23,"sense_key":"take%2:40:07::","synset_id":"accept.v.05"},{"index":24,"sense_key":"take%2:30:08::","synset_id":"take.v.24"},{"index":25,"sense_key":"take%2:31:02::","synset_id":"learn.v.04"},{"index":26,"sense_key":"take%2:32:00::","synset_id":"claim.v.05"},{"index":27,"sense_key":"take%2:38:05::","synset_id":"take.v.27"},{"index":28,"sense_key":"take%2:33:09::","synset_id":"aim.v.01"},{"index":29,"sense_key":"take%2:30:09::","synset_id":"take.v.29"},{"index":30,"sense_key":"take%2:42:10::","synset_id":"carry.v.02"},{"index":31,"sense_key":"take%2:40:03::","synset_id":"lease.v.04"},{"index":32,"sense_key":"take%2:40:04::","synset_id":"subscribe.v.05"},{"index":33,"sense_key":"take%2:40:02::","synset_id":"take.v.33"},{"index":34,"sense_key":"take%2:40:08::","synset_id":"take.v.34"},{"index":35,"sense_key":"take%2:35:14::","synset_id":"take.v.35"},{"index":36,"sense_key":"take%2:32:09::","synset_id":"claim.v.04"},{"index":37,"sense_key":"take%2:42:03::","synset_id":"accept.v.08"},{"index":38,"sense_key":"take%2:42:15::","synset_id":"contain.v.05"},{"index":39,"sense_key":"take%2:41:13::","synset_id":"take.v.39"},{"index":40,"sense_key":"take%2:38:11::","synset_id":"drive.v.16"},{"index":41,"sense_key":"take%2:33:08::","synset_id":"take.v.41"},{"index":42,"sense_key":"take%2:29:08::","synset_id":"contract.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":25,"chosen_sense_key":"take%2:31:02::","call_ids":["senseval2.d002.s021.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":25,"predicted_sense_key":"take%2:31:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":3042,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.006204,"input_uncached_usd":0.006084,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4143862089840695}
{"item_id":"senseval2.d002.s021.t002","gold_sense_keys":["course%1:04:01::"],"candidates":[{"index":1,"sense_key":"course%1:04:01::","synset_id":"course.n.01"},{"index":2,"sense_key":"course%1:14:00::","synset_id":"course.n.02"},{"index":3,"sense_key":"course%1:15:00::","synset_id":"course.n.03"},{"index":4,"sense_key":"course%1:04:00::","synset_id":"course.n.04"},{"index":5,"sense_key":"course%1:17:00::","synset_id":"path.n.04"},{"index":6,"sense_key":"course%1:14:01::","synset_id":"class.n.02"},{"index":7,"sense_key":"course%1:13:00::","synset_id":"course.n.07"},{"index":8,"sense_key":"course%1:06:01::","synset_id":"course.n.08"},{"index":9,"sense_key":"course%1:06:00::","synset_id":"course.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"course%1:04:01::","call_ids":["senseval2.d002.s021.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"course%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1013,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002146,"input_uncached_usd":0.002026,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.013554000062868}
{"item_id":"senseval2.d002.s021.t003","gold_sense_keys":["college%1:14:01::"],"candidates":[{"index":1,"sense_key":"college%1:14:00::","synset_id":"college.n.01"},{"index":2,"sense_key":"college%1:14:01::","synset_id":"college.n.02"},{"index":3,"sense_key":"college%1:06:00::","synset_id":"college.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"college%1:14:01::","call_ids":["senseval2.d002.s021.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"college%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":557,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001234,"input_uncached_usd":0.001114,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8377813749248162}
{"item_id":"senseval2.d002.s021.t004","gold_sense_keys":["think%2:31:03::","think%2:36:00::"],"candidates":[{"index":1,"sense_key":"think%2:31:01::","synset_id":"think.v.01"},{"index":2,"sense_key":"think%2:31:03::","synset_id":"think.v.02"},{"index":3,"sense_key":"think%2:31:00::","synset_id":"think.v.03"},{"index":4,"sense_key":"think%2:31:02::","synset_id":"remember.v.01"},{"index":5,"sense_key":"think%2:36:00::","synset_id":"think.v.05"},{"index":6,"sense_key":"think%2:31:09::","synset_id":"think.v.06"},{"index":7,"sense_key":"think%2:31:06::","synset_id":"intend.v.01"},{"index":8,"sense_key":"think%2:31:12::","synset_id":"think.v.08"},{"index":9,"sense_key":"think%2:31:13::","synset_id":"think.v.09"},{"index":10,"sense_key":"think%2:31:15::","synset_id":"think.v.10"},{"index":11,"sense_key":"think%2:31:14::","synset_id":"think.v.11"},{"index":12,"sense_key":"think%2:31:08::","synset_id":"think.v.12"},{"index":13,"sense_key":"think%2:30:00::","synset_id":"think.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"think%2:31:03::","call_ids":["senseval2.d002.s021.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"think%2:31:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1232,"cached_input_tokens":0,"output_tokens":110,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0035640000000000003,"input_uncached_usd":0.002464,"input_cached_usd":0.0,"output_usd":0.0011,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8129105829866603}
{"item_id":"senseval2.d002.s022.t001","gold_sense_keys":["fairly%4:02:00::"],"candidates":[{"index":1,"sense_key":"fairly%4:02:02::","synset_id":"reasonably.r.01"},{"index":2,"sense_key":"fairly%4:02:00::","synset_id":"fairly.r.02"},{"index":3,"sense_key":"fairly%4:02:03::","synset_id":"fairly.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"fairly%4:02:00::","call_ids":["senseval2.d002.s022.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"fairly%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":663,"cached_input_tokens":0,"output_tokens":127,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0025960000000000002,"input_uncached_usd":0.001326,"input_cached_usd":0.0,"output_usd":0.00127,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7889195419847965}
{"item_id":"senseval2.d002.s022.t002","gold_sense_keys":["describe%2:32:00::"],"candidates":[{"index":1,"sense_key":"describe%2:32:00::","synset_id":"describe.v.01"},{"index":2,"sense_key":"describe%2:32:01::","synset_id":"report.v.01"},{"index":3,"sense_key":"describe%2:35:00::","synset_id":"trace.v.02"},{"index":4,"sense_key":"describe%2:31:00::","synset_id":"identify.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"describe%2:32:01::","call_ids":["senseval2.d002.s022.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"describe%2:32:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":694,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002038,"input_uncached_usd":0.001388,"input_cached_usd":0.0,"output_usd":0.0006500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.754958083969541}
{"item_id":"senseval2.d002.s022.t003","gold_sense_keys":["pap%1:10:00::"],"candidates":[{"index":1,"sense_key":"pap%1:10:00::","synset_id":"pap.n.01"},{"index":2,"sense_key":"pap%1:13:00::","synset_id":"soft_diet.n.01"},{"index":3,"sense_key":"pap%1:08:00::","synset_id":"nipple.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"pap%1:10:00::","call_ids":["senseval2.d002.s022.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"pap%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":610,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00134,"input_uncached_usd":0.00122,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0051254580030218}
{"item_id":"senseval2.d002.s022.t004","gold_sense_keys":["course%1:04:01::"],"candidates":[{"index":1,"sense_key":"course%1:04:01::","synset_id":"course.n.01"},{"index":2,"sense_key":"course%1:14:00::","synset_id":"course.n.02"},{"index":3,"sense_key":"course%1:15:00::","synset_id":"course.n.03"},{"index":4,"sense_key":"course%1:04:00::","synset_id":"course.n.04"},{"index":5,"sense_key":"course%1:17:00::","synset_id":"path.n.04"},{"index":6,"sense_key":"course%1:14:01::","synset_id":"class.n.02"},{"index":7,"sense_key":"course%1:13:00::","synset_id":"course.n.07"},{"index":8,"sense_key":"course%1:06:01::","synset_id":"course.n.08"},{"index":9,"sense_key":"course%1:06:00::","synset_id":"course.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"course%1:04:01::","call_ids":["senseval2.d002.s022.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"course%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1020,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0021599999999999996,"input_uncached_usd":0.0020399999999999997,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5418998330133036}
{"item_id":"senseval2.d002.s023.t002","gold_sense_keys":["teacher%1:18:00::"],"candidates":[{"index":1,"sense_key":"teacher%1:18:00::","synset_id":"teacher.n.01"},{"index":2,"sense_key":"teacher%1:09:00::","synset_id":"teacher.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"teacher%1:18:00::","call_ids":["senseval2.d002.s023.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"teacher%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":557,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001234,"input_uncached_usd":0.001114,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8423429590184242}
{"item_id":"senseval2.d002.s023.t003","gold_sense_keys":["distinct%5:00:00:different:00"],"candidates":[{"index":1,"sense_key":"distinct%5:00:00:different:00","synset_id":"distinct.s.01"},{"index":2,"sense_key":"distinct%3:00:00::","synset_id":"distinct.a.02"},{"index":3,"sense_key":"distinct%5:00:00:separate:00","synset_id":"discrete.s.01"},{"index":4,"sense_key":"distinct%5:00:00:definite:00","synset_id":"distinct.s.03"},{"index":5,"sense_key":"distinct%5:00:00:clear:00","synset_id":"clear-cut.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"distinct%5:00:00:different:00","call_ids":["senseval2.d002.s023.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"distinct%5:00:00:different:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":847,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0022240000000000003,"input_uncached_usd":0.001694,"input_cached_usd":0.0,"output_usd":0.0005300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.292282292037271}
{"item_id":"senseval2.d002.s023.t004","gold_sense_keys":["educational%3:01:00::"],"candidates":[{"index":1,"sense_key":"educational%3:01:00::","synset_id":"educational.a.01"},{"index":2,"sense_key":"educational%5:00:00:instructive:00","synset_id":"educational.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"educational%3:01:00::","call_ids":["senseval2.d002.s023.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"educational%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":536,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0011920000000000001,"input_uncached_usd":0.001072,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1794144170125946}
{"item_id":"senseval2.d002.s023.t005","gold_sense_keys":["establishment%1:14:00::","establishment%1:14:02::"],"candidates":[{"index":1,"sense_key":"establishment%1:04:00::","synset_id":"constitution.n.02"},{"index":2,"sense_key":"establishment%1:14:00::","synset_id":"institution.n.01"},{"index":3,"sense_key":"establishment%1:14:02::","synset_id":"administration.n.02"},{"index":4,"sense_key":"establishment%1:06:00::","synset_id":"establishment.n.04"},{"index":5,"sense_key":"establishment%1:14:01::","synset_id":"establishment.n.05"},{"index":6,"sense_key":"establishment%1:22:00::","synset_id":"establishment.n.06"},{"index":7,"sense_key":"establishment%1:09:00::","synset_id":"establishment.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"establishment%1:14:02::","call_ids":["senseval2.d002.s023.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"establishment%1:14:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":874,"cached_input_tokens":0,"output_tokens":166,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0034080000000000004,"input_uncached_usd":0.001748,"input_cached_usd":0.0,"output_usd":0.0016600000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0586403750348836}
{"item_id":"senseval2.d002.s024.t000","gold_sense_keys":["know%2:31:00::"],"candidates":[{"index":1,"sense_key":"know%2:31:01::","synset_id":"know.v.01"},{"index":2,"sense_key":"know%2:31:03::","synset_id":"know.v.02"},{"index":3,"sense_key":"know%2:31:02::","synset_id":"know.v.03"},{"index":4,"sense_key":"know%2:31:00::","synset_id":"know.v.04"},{"index":5,"sense_key":"know%2:31:04::","synset_id":"know.v.05"},{"index":6,"sense_key":"know%2:31:06::","synset_id":"acknowledge.v.06"},{"index":7,"sense_key":"know%2:31:15::","synset_id":"know.v.07"},{"index":8,"sense_key":"know%2:35:00::","synset_id":"sleep_together.v.01"},{"index":9,"sense_key":"know%2:31:11::","synset_id":"know.v.09"},{"index":10,"sense_key":"know%2:31:12::","synset_id":"know.v.10"},{"index":11,"sense_key":"know%2:31:14::","synset_id":"know.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"know%2:31:00::","call_ids":["senseval2.d002.s024.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"know%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1168,"cached_input_tokens":0,"output_tokens":45,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002786,"input_uncached_usd":0.002336,"input_cached_usd":0.0,"output_usd":0.00045000000000000004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0478152089053765}
{"item_id":"senseval2.d002.s024.t004","gold_sense_keys":["seriously%4:02:00::"],"candidates":[{"index":1,"sense_key":"seriously%4:02:00::","synset_id":"seriously.r.01"},{"index":2,"sense_key":"seriously%4:02:01::","synset_id":"badly.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"seriously%4:02:00::","call_ids":["senseval2.d002.s024.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"seriously%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":591,"cached_input_tokens":0,"output_tokens":75,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001932,"input_uncached_usd":0.0011819999999999999,"input_cached_usd":0.0,"output_usd":0.00075,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5213172920048237}
{"item_id":"senseval2.d002.s024.t005","gold_sense_keys":["committed%3:00:00::"],"candidates":[{"index":1,"sense_key":"committed%3:00:00::","synset_id":"committed.a.01"},{"index":2,"sense_key":"committed%3:00:04::","synset_id":"attached.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"committed%3:00:00::","call_ids":["senseval2.d002.s024.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"committed%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":556,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001232,"input_uncached_usd":0.001112,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1063797499518842}
{"item_id":"senseval2.d002.s024.t007","gold_sense_keys":["teaching%1:04:00::","teaching%1:04:01::"],"candidates":[{"index":1,"sense_key":"teaching%1:04:00::","synset_id":"teaching.n.01"},{"index":2,"sense_key":"teaching%1:09:00::","synset_id":"teaching.n.02"},{"index":3,"sense_key":"teaching%1:04:01::","synset_id":"education.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"teaching%1:04:01::","call_ids":["senseval2.d002.s024.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"teaching%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":688,"cached_input_tokens":0,"output_tokens":107,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002446,"input_uncached_usd":0.0013759999999999998,"input_cached_usd":0.0,"output_usd":0.00107,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1692065419629216}
{"item_id":"senseval2.d002.s025.t000","gold_sense_keys":["best%3:00:00::"],"candidates":[{"index":1,"sense_key":"best%3:00:00::","synset_id":"best.a.01"},{"index":2,"sense_key":"best%5:00:00:advisable:00","synset_id":"better.s.01"},{"index":3,"sense_key":"good%3:00:01::","synset_id":"good.a.01"},{"index":4,"sense_key":"full%5:00:00:ample:00","synset_id":"full.s.04"},{"index":5,"sense_key":"good%3:00:02::","synset_id":"good.a.03"},{"index":6,"sense_key":"estimable%5:00:00:reputable:00","synset_id":"estimable.s.01"},{"index":7,"sense_key":"beneficial%5:00:00:advantageous:00","synset_id":"beneficial.s.01"},{"index":8,"sense_key":"good%5:00:00:nice:00","synset_id":"good.s.04"},{"index":9,"sense_key":"good%5:00:00:righteous:00","synset_id":"good.s.05"},{"index":10,"sense_key":"adept%5:00:00:skilled:00","synset_id":"adept.s.01"},{"index":11,"sense_key":"good%5:00:00:complete:00","synset_id":"good.s.07"},{"index":12,"sense_key":"dear%5:00:00:close:02","synset_id":"dear.s.02"},{"index":13,"sense_key":"dependable%5:00:00:sound:00","synset_id":"dependable.s.03"},{"index":14,"sense_key":"good%5:00:00:opportune:00","synset_id":"good.s.10"},{"index":15,"sense_key":"good%5:00:00:fortunate:00","synset_id":"good.s.11"},{"index":16,"sense_key":"effective%5:00:00:operative:00","synset_id":"effective.s.03"},{"index":17,"sense_key":"good%5:00:00:pleasing:00","synset_id":"good.s.13"},{"index":18,"sense_key":"good%5:00:00:intellectual:00","synset_id":"good.s.14"},{"index":19,"sense_key":"good%5:00:00:healthy:00","synset_id":"good.s.15"},{"index":20,"sense_key":"good%5:00:00:healthful:00","synset_id":"good.s.16"},{"index":21,"sense_key":"good%5:00:00:genuine:00","synset_id":"good.s.17"},{"index":22,"sense_key":"good%5:00:00:fresh:01","synset_id":"good.s.18"},{"index":23,"sense_key":"good%5:00:00:discriminating:00","synset_id":"good.s.19"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"best%3:00:00::","call_ids":["senseval2.d002.s025.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"best%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1884,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0043479999999999994,"input_uncached_usd":0.0037679999999999996,"input_cached_usd":0.0,"output_usd":0.00058,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.46285966609139}
{"item_id":"senseval2.d002.s025.t001","gold_sense_keys":["bright%5:00:00:intelligent:00"],"candidates":[{"index":1,"sense_key":"bright%3:00:00::","synset_id":"bright.a.01"},{"index":2,"sense_key":"bright%5:00:00:colorful:00","synset_id":"bright.s.01"},{"index":3,"sense_key":"bright%5:00:00:intelligent:00","synset_id":"bright.s.02"},{"index":4,"sense_key":"bright%5:00:00:light:06","synset_id":"bright.s.03"},{"index":5,"sense_key":"bright%5:00:00:polished:00","synset_id":"bright.s.04"},{"index":6,"sense_key":"bright%5:00:00:glorious:00","synset_id":"bright.s.05"},{"index":7,"sense_key":"bright%3:00:02::","synset_id":"undimmed.a.01"},{"index":8,"sense_key":"bright%5:00:00:reverberant:00","synset_id":"bright.s.06"},{"index":9,"sense_key":"bright%5:00:00:happy:00","synset_id":"bright.s.07"},{"index":10,"sense_key":"bright%5:00:00:auspicious:00","synset_id":"bright.s.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"bright%5:00:00:intelligent:00","call_ids":["senseval2.d002.s025.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"bright%5:00:00:intelligent:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1117,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002784,"input_uncached_usd":0.002234,"input_cached_usd":0.0,"output_usd":0.00055,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4108115409035236}
{"item_id":"senseval2.d002.s025.t002","gold_sense_keys":["generation%1:14:01::"],"candidates":[{"index":1,"sense_key":"generation%1:14:01::","synset_id":"coevals.n.01"},{"index":2,"sense_key":"generation%1:14:00::","synset_id":"generation.n.02"},{"index":3,"sense_key":"generation%1:28:00::","synset_id":"generation.n.03"},{"index":4,"sense_key":"generation%1:28:01::","synset_id":"generation.n.04"},{"index":5,"sense_key":"generation%1:11:00::","synset_id":"genesis.n.01"},{"index":6,"sense_key":"generation%1:04:01::","synset_id":"generation.n.06"},{"index":7,"sense_key":"generation%1:04:00::","synset_id":"generation.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"generation%1:14:01::","call_ids":["senseval2.d002.s025.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"generation%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":744,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001608,"input_uncached_usd":0.001488,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4876598749542609}
{"item_id":"senseval2.d002.s025.t003","gold_sense_keys":["very%4:02:00::"],"candidates":[{"index":1,"sense_key":"very%4:02:00::","synset_id":"very.r.01"},{"index":2,"sense_key":"very%4:02:01::","synset_id":"very.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"very%4:02:00::","call_ids":["senseval2.d002.s025.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"very%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":539,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001198,"input_uncached_usd":0.001078,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0188412080751732}
{"item_id":"senseval2.d002.s025.t006","gold_sense_keys":["people%1:14:00::"],"candidates":[{"index":1,"sense_key":"people%1:14:00::","synset_id":"people.n.01"},{"index":2,"sense_key":"people%1:14:01::","synset_id":"citizenry.n.01"},{"index":3,"sense_key":"people%1:14:02::","synset_id":"people.n.03"},{"index":4,"sense_key":"people%1:14:03::","synset_id":"multitude.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"people%1:14:00::","call_ids":["senseval2.d002.s025.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"people%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":649,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001418,"input_uncached_usd":0.001298,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0978139579528943}
{"item_id":"senseval2.d002.s026.t000","gold_sense_keys":["job%1:04:00::"],"candidates":[{"index":1,"sense_key":"job%1:04:00::","synset_id":"occupation.n.01"},{"index":2,"sense_key":"job%1:04:01::","synset_id":"job.n.02"},{"index":3,"sense_key":"job%1:06:00::","synset_id":"job.n.03"},{"index":4,"sense_key":"job%1:06:01::","synset_id":"job.n.04"},{"index":5,"sense_key":"job%1:04:04::","synset_id":"job.n.05"},{"index":6,"sense_key":"job%1:04:05::","synset_id":"job.n.06"},{"index":7,"sense_key":"job%1:04:03::","synset_id":"job.n.07"},{"index":8,"sense_key":"job%1:26:00::","synset_id":"problem.n.01"},{"index":9,"sense_key":"job%1:18:00::","synset_id":"job.n.09"},{"index":10,"sense_key":"job%1:18:01::","synset_id":"job.n.10"},{"index":11,"sense_key":"job%1:10:01::","synset_id":"job.n.11"},{"index":12,"sense_key":"job%1:10:00::","synset_id":"job.n.12"},{"index":13,"sense_key":"job%1:04:02::","synset_id":"caper.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"job%1:04:04::","call_ids":["senseval2.d002.s026.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"job%1:04:04::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1140,"cached_input_tokens":0,"output_tokens":159,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00387,"input_uncached_usd":0.00228,"input_cached_usd":0.0,"output_usd":0.00159,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.7251223750645295}
{"item_id":"senseval2.d002.s026.t001","gold_sense_keys":["well%4:02:00::"],"candidates":[{"index":1,"sense_key":"well%4:02:00::","synset_id":"well.r.01"},{"index":2,"sense_key":"well%4:02:05::","synset_id":"well.r.02"},{"index":3,"sense_key":"well%4:02:03::","synset_id":"well.r.03"},{"index":4,"sense_key":"well%4:02:15::","synset_id":"well.r.04"},{"index":5,"sense_key":"well%4:02:08::","synset_id":"well.r.05"},{"index":6,"sense_key":"well%4:02:07::","synset_id":"well.r.06"},{"index":7,"sense_key":"well%4:02:11::","synset_id":"well.r.07"},{"index":8,"sense_key":"well%4:02:14::","synset_id":"well.r.08"},{"index":9,"sense_key":"well%4:02:13::","synset_id":"well.r.09"},{"index":10,"sense_key":"well%4:02:12::","synset_id":"well.r.10"},{"index":11,"sense_key":"well%4:02:10::","synset_id":"well.r.11"},{"index":12,"sense_key":"well%4:02:09::","synset_id":"well.r.12"},{"index":13,"sense_key":"well%4:02:01::","synset_id":"well.r.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"well%4:02:00::","call_ids":["senseval2.d002.s026.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"well%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1205,"cached_input_tokens":0,"output_tokens":91,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00332,"input_uncached_usd":0.00241,"input_cached_usd":0.0,"output_usd":0.0009100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.898920041974634}
{"item_id":"senseval2.d002.s027.t000","gold_sense_keys":["remember%2:31:00::"],"candidates":[{"index":1,"sense_key":"remember%2:31:00::","synset_id":"remember.v.01"},{"index":2,"sense_key":"remember%2:31:01::","synset_id":"remember.v.02"},{"index":3,"sense_key":"remember%2:31:02::","synset_id":"remember.v.03"},{"index":4,"sense_key":"remember%2:40:00::","synset_id":"remember.v.04"},{"index":5,"sense_key":"remember%2:32:07::","synset_id":"remember.v.05"},{"index":6,"sense_key":"remember%2:32:00::","synset_id":"commend.v.05"},{"index":7,"sense_key":"remember%2:31:08::","synset_id":"remember.v.07"},{"index":8,"sense_key":"remember%2:31:09::","synset_id":"commemorate.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"remember%2:31:00::","call_ids":["senseval2.d002.s027.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"remember%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":917,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001954,"input_uncached_usd":0.001834,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.450691416976042}
{"item_id":"senseval2.d002.s027.t001","gold_sense_keys":["truly%4:02:00::","truly%4:02:04::"],"candidates":[{"index":1,"sense_key":"truly%4:02:00::","synset_id":"truly.r.01"},{"index":2,"sense_key":"truly%4:02:01::","synset_id":"rightfully.r.01"},{"index":3,"sense_key":"truly%4:02:03::","synset_id":"sincerely.r.01"},{"index":4,"sense_key":"truly%4:02:04::","synset_id":"in_truth.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"truly%4:02:00::","call_ids":["senseval2.d002.s027.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"truly%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":692,"cached_input_tokens":0,"output_tokens":179,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003174,"input_uncached_usd":0.001384,"input_cached_usd":0.0,"output_usd":0.0017900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.7419057079823688}
{"item_id":"senseval2.d002.s027.t003","gold_sense_keys":["teacher%1:18:00::"],"candidates":[{"index":1,"sense_key":"teacher%1:18:00::","synset_id":"teacher.n.01"},{"index":2,"sense_key":"teacher%1:09:00::","synset_id":"teacher.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"teacher%1:18:00::","call_ids":["senseval2.d002.s027.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"teacher%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":501,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001122,"input_uncached_usd":0.0010019999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.423358083004132}
{"item_id":"senseval2.d002.s028.t000","gold_sense_keys":["education%1:09:01::"],"candidates":[{"index":1,"sense_key":"education%1:04:00::","synset_id":"education.n.01"},{"index":2,"sense_key":"education%1:09:00::","synset_id":"education.n.02"},{"index":3,"sense_key":"education%1:09:01::","synset_id":"education.n.03"},{"index":4,"sense_key":"education%1:04:01::","synset_id":"education.n.04"},{"index":5,"sense_key":"education%1:07:01::","synset_id":"education.n.05"},{"index":6,"sense_key":"education%1:14:00::","synset_id":"department_of_education.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"education%1:04:00::","call_ids":["senseval2.d002.s028.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"education%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":795,"cached_input_tokens":0,"output_tokens":139,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00298,"input_uncached_usd":0.0015899999999999998,"input_cached_usd":0.0,"output_usd":0.0013900000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6638077499810606}
{"item_id":"senseval2.d002.s028.t001","gold_sense_keys":["proceed%2:38:00::"],"candidates":[{"index":1,"sense_key":"proceed%2:32:00::","synset_id":"continue.v.02"},{"index":2,"sense_key":"proceed%2:38:00::","synset_id":"proceed.v.02"},{"index":3,"sense_key":"proceed%2:41:00::","synset_id":"go.v.02"},{"index":4,"sense_key":"proceed%2:30:00::","synset_id":"proceed.v.04"},{"index":5,"sense_key":"proceed%2:42:00::","synset_id":"continue.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"proceed%2:42:00::","call_ids":["senseval2.d002.s028.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"proceed%2:42:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":742,"cached_input_tokens":0,"output_tokens":122,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002704,"input_uncached_usd":0.0014839999999999999,"input_cached_usd":0.0,"output_usd":0.0012200000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.252523875096813}
{"item_id":"senseval2.d002.s028.t003","gold_sense_keys":["competent%3:00:00::"],"candidates":[{"index":1,"sense_key":"competent%3:00:00::","synset_id":"competent.a.01"},{"index":2,"sense_key":"competent%5:00:00:adequate:00","synset_id":"competent.s.01"},{"index":3,"sense_key":"competent%3:00:02::","synset_id":"competent.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"competent%3:00:00::","call_ids":["senseval2.d002.s028.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"competent%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":545,"cached_input_tokens":0,"output_tokens":122,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0023100000000000004,"input_uncached_usd":0.00109,"input_cached_usd":0.0,"output_usd":0.0012200000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1428595420438796}
{"item_id":"senseval2.d002.s029.t000","gold_sense_keys":["sense%1:10:00::"],"candidates":[{"index":1,"sense_key":"sense%1:09:05::","synset_id":"sense.n.01"},{"index":2,"sense_key":"sense%1:10:00::","synset_id":"sense.n.02"},{"index":3,"sense_key":"sense%1:09:02::","synset_id":"sense.n.03"},{"index":4,"sense_key":"sense%1:09:04::","synset_id":"common_sense.n.01"},{"index":5,"sense_key":"sense%1:09:06::","synset_id":"sense.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"sense%1:10:00::","call_ids":["senseval2.d002.s029.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"sense%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":765,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00209,"input_uncached_usd":0.00153,"input_cached_usd":0.0,"output_usd":0.0005600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5948526250431314}
{"item_id":"senseval2.d002.s029.t001","gold_sense_keys":["teacher%1:18:00::"],"candidates":[{"index":1,"sense_key":"teacher%1:18:00::","synset_id":"teacher.n.01"},{"index":2,"sense_key":"teacher%1:09:00::","synset_id":"teacher.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"teacher%1:18:00::","call_ids":["senseval2.d002.s029.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"teacher%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":504,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001128,"input_uncached_usd":0.001008,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.074509207974188}
{"item_id":"senseval2.d002.s029.t002","gold_sense_keys":["compare%2:31:01::"],"candidates":[{"index":1,"sense_key":"compare%2:31:00::","synset_id":"compare.v.01"},{"index":2,"sense_key":"compare%2:42:00::","synset_id":"compare.v.02"},{"index":3,"sense_key":"compare%2:31:01::","synset_id":"compare.v.03"},{"index":4,"sense_key":"compare%2:32:00::","synset_id":"compare.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"compare%2:31:01::","call_ids":["senseval2.d002.s029.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"compare%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":641,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001402,"input_uncached_usd":0.001282,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8899824589025229}
{"item_id":"senseval2.d002.s029.t003","gold_sense_keys":["one%1:09:00::"],"candidates":[{"index":1,"sense_key":"one%1:23:00::","synset_id":"one.n.01"},{"index":2,"sense_key":"one%1:09:00::","synset_id":"one.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"one%1:09:00::","call_ids":["senseval2.d002.s029.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"one%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":529,"cached_input_tokens":0,"output_tokens":80,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0018579999999999998,"input_uncached_usd":0.001058,"input_cached_usd":0.0,"output_usd":0.0008,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5275116249686107}
{"item_id":"senseval2.d002.s030.t000","gold_sense_keys":["brilliant%5:00:00:intelligent:00"],"candidates":[{"index":1,"sense_key":"brilliant%5:00:00:superior:02","synset_id":"brilliant.s.01"},{"index":2,"sense_key":"brilliant%5:00:00:intelligent:00","synset_id":"brainy.s.01"},{"index":3,"sense_key":"brilliant%5:00:00:impressive:00","synset_id":"brilliant.s.03"},{"index":4,"sense_key":"brilliant%5:00:00:colorful:00","synset_id":"bright.s.01"},{"index":5,"sense_key":"brilliant%5:00:00:bright:00","synset_id":"brilliant.s.05"},{"index":6,"sense_key":"brilliant%5:00:00:reverberant:00","synset_id":"bright.s.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"brilliant%5:00:00:intelligent:00","call_ids":["senseval2.d002.s030.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"brilliant%5:00:00:intelligent:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":813,"cached_input_tokens":0,"output_tokens":99,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002616,"input_uncached_usd":0.0016259999999999998,"input_cached_usd":0.0,"output_usd":0.00099,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5695979580050334}
{"item_id":"senseval2.d002.s030.t001","gold_sense_keys":["probably%4:02:00::"],"candidates":[{"index":1,"sense_key":"probably%4:02:00::","synset_id":"probably.r.01"},{"index":2,"sense_key":"probably%4:02:01::","synset_id":"credibly.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"probably%4:02:00::","call_ids":["senseval2.d002.s030.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"probably%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":548,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001216,"input_uncached_usd":0.001096,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7659412920475006}
{"item_id":"senseval2.d002.s031.t000","gold_sense_keys":["competent%3:00:00::"],"candidates":[{"index":1,"sense_key":"competent%3:00:00::","synset_id":"competent.a.01"},{"index":2,"sense_key":"competent%5:00:00:adequate:00","synset_id":"competent.s.01"},{"index":3,"sense_key":"competent%3:00:02::","synset_id":"competent.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"competent%3:00:00::","call_ids":["senseval2.d002.s031.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"competent%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":517,"cached_input_tokens":0,"output_tokens":132,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002354,"input_uncached_usd":0.001034,"input_cached_usd":0.0,"output_usd":0.0013200000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.3324142090277746}
{"item_id":"senseval2.d002.s031.t002","gold_sense_keys":["serve%2:42:02::","serve%2:42:03::"],"candidates":[{"index":1,"sense_key":"serve%2:42:03::","synset_id":"serve.v.01"},{"index":2,"sense_key":"serve%2:33:00::","synset_id":"serve.v.02"},{"index":3,"sense_key":"serve%2:42:01::","synset_id":"serve.v.03"},{"index":4,"sense_key":"serve%2:41:00::","synset_id":"service.v.01"},{"index":5,"sense_key":"serve%2:34:01::","synset_id":"serve.v.05"},{"index":6,"sense_key":"serve%2:34:00::","synset_id":"serve.v.06"},{"index":7,"sense_key":"serve%2:41:02::","synset_id":"serve.v.07"},{"index":8,"sense_key":"serve%2:42:02::","synset_id":"serve.v.08"},{"index":9,"sense_key":"serve%2:42:00::","synset_id":"serve.v.09"},{"index":10,"sense_key":"serve%2:41:01::","synset_id":"serve.v.10"},{"index":11,"sense_key":"serve%2:35:00::","synset_id":"serve.v.11"},{"index":12,"sense_key":"serve%2:42:12::","synset_id":"suffice.v.01"},{"index":13,"sense_key":"serve%2:41:13::","synset_id":"serve.v.13"},{"index":14,"sense_key":"serve%2:35:01::","synset_id":"serve.v.14"},{"index":15,"sense_key":"serve%2:33:01::","synset_id":"serve.v.15"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"serve%2:42:02::","call_ids":["senseval2.d002.s031.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"serve%2:42:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1303,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003156,"input_uncached_usd":0.002606,"input_cached_usd":0.0,"output_usd":0.00055,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4048955410253257}
{"item_id":"senseval2.d002.s031.t003","gold_sense_keys":["well%4:02:00::"],"candidates":[{"index":1,"sense_key":"well%4:02:00::","synset_id":"well.r.01"},{"index":2,"sense_key":"well%4:02:05::","synset_id":"well.r.02"},{"index":3,"sense_key":"well%4:02:03::","synset_id":"well.r.03"},{"index":4,"sense_key":"well%4:02:15::","synset_id":"well.r.04"},{"index":5,"sense_key":"well%4:02:08::","synset_id":"well.r.05"},{"index":6,"sense_key":"well%4:02:07::","synset_id":"well.r.06"},{"index":7,"sense_key":"well%4:02:11::","synset_id":"well.r.07"},{"index":8,"sense_key":"well%4:02:14::","synset_id":"well.r.08"},{"index":9,"sense_key":"well%4:02:13::","synset_id":"well.r.09"},{"index":10,"sense_key":"well%4:02:12::","synset_id":"well.r.10"},{"index":11,"sense_key":"well%4:02:10::","synset_id":"well.r.11"},{"index":12,"sense_key":"well%4:02:09::","synset_id":"well.r.12"},{"index":13,"sense_key":"well%4:02:01::","synset_id":"well.r.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"well%4:02:00::","call_ids":["senseval2.d002.s031.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"well%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1184,"cached_input_tokens":0,"output_tokens":113,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0034980000000000002,"input_uncached_usd":0.002368,"input_cached_usd":0.0,"output_usd":0.0011300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.814363749930635}
{"item_id":"senseval2.d002.s032.t000","gold_sense_keys":["teacher%1:18:00::"],"candidates":[{"index":1,"sense_key":"teacher%1:18:00::","synset_id":"teacher.n.01"},{"index":2,"sense_key":"teacher%1:09:00::","synset_id":"teacher.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"teacher%1:18:00::","call_ids":["senseval2.d002.s032.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"teacher%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":494,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001108,"input_uncached_usd":0.000988,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9504042080370709}
{"item_id":"senseval2.d002.s032.t001","gold_sense_keys":["important%3:00:00::","important%3:00:02::","important%3:00:04::"],"candidates":[{"index":1,"sense_key":"important%3:00:00::","synset_id":"important.a.01"},{"index":2,"sense_key":"important%3:00:04::","synset_id":"significant.a.01"},{"index":3,"sense_key":"important%3:00:02::","synset_id":"crucial.a.01"},{"index":4,"sense_key":"important%5:00:00:influential:00","synset_id":"authoritative.s.01"},{"index":5,"sense_key":"important%5:00:00:immodest:02","synset_id":"important.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"important%3:00:00::","call_ids":["senseval2.d002.s032.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"important%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":701,"cached_input_tokens":0,"output_tokens":137,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002772,"input_uncached_usd":0.001402,"input_cached_usd":0.0,"output_usd":0.0013700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.063208582927473}
{"item_id":"senseval2.d002.s032.t002","gold_sense_keys":["factor%1:09:00::","factor%1:11:00::"],"candidates":[{"index":1,"sense_key":"factor%1:11:00::","synset_id":"factor.n.01"},{"index":2,"sense_key":"factor%1:09:00::","synset_id":"component.n.01"},{"index":3,"sense_key":"factor%1:23:00::","synset_id":"divisor.n.01"},{"index":4,"sense_key":"factor%1:18:00::","synset_id":"agent.n.04"},{"index":5,"sense_key":"factor%1:23:01::","synset_id":"factor.n.05"},{"index":6,"sense_key":"factor%1:09:01::","synset_id":"factor.n.06"},{"index":7,"sense_key":"factor%1:08:00::","synset_id":"gene.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"factor%1:11:00::","call_ids":["senseval2.d002.s032.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"factor%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":827,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001774,"input_uncached_usd":0.0016539999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.954794832970947}
{"item_id":"senseval2.d002.s032.t003","gold_sense_keys":["educational%3:01:00::"],"candidates":[{"index":1,"sense_key":"educational%3:01:00::","synset_id":"educational.a.01"},{"index":2,"sense_key":"educational%5:00:00:instructive:00","synset_id":"educational.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"educational%3:01:00::","call_ids":["senseval2.d002.s032.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"educational%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":473,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001546,"input_uncached_usd":0.000946,"input_cached_usd":0.0,"output_usd":0.0006000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3046408329391852}
{"item_id":"senseval2.d002.s032.t004","gold_sense_keys":["crisis%1:26:00::"],"candidates":[{"index":1,"sense_key":"crisis%1:26:00::","synset_id":"crisis.n.01"},{"index":2,"sense_key":"crisis%1:11:00::","synset_id":"crisis.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"crisis%1:26:00::","call_ids":["senseval2.d002.s032.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"crisis%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":511,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001142,"input_uncached_usd":0.001022,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9771230419864878}
{"item_id":"senseval2.d002.s033.t000","gold_sense_keys":["problem%1:26:00::"],"candidates":[{"index":1,"sense_key":"problem%1:26:00::","synset_id":"problem.n.01"},{"index":2,"sense_key":"problem%1:10:00::","synset_id":"problem.n.02"},{"index":3,"sense_key":"problem%1:09:00::","synset_id":"trouble.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"problem%1:09:00::","call_ids":["senseval2.d002.s033.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"problem%1:09:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":582,"cached_input_tokens":0,"output_tokens":295,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0041140000000000005,"input_uncached_usd":0.0011639999999999999,"input_cached_usd":0.0,"output_usd":0.0029500000000000004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.8608043330023065}
{"item_id":"senseval2.d002.s033.t001","gold_sense_keys":["equity%1:07:00::"],"candidates":[{"index":1,"sense_key":"equity%1:21:01::","synset_id":"equity.n.01"},{"index":2,"sense_key":"equity%1:21:00::","synset_id":"equity.n.02"},{"index":3,"sense_key":"equity%1:07:00::","synset_id":"fairness.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"equity%1:07:00::","call_ids":["senseval2.d002.s033.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"equity%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":539,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001198,"input_uncached_usd":0.001078,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3942382079549134}
{"item_id":"senseval2.d002.s033.t002","gold_sense_keys":["educational%3:01:00::"],"candidates":[{"index":1,"sense_key":"educational%3:01:00::","synset_id":"educational.a.01"},{"index":2,"sense_key":"educational%5:00:00:instructive:00","synset_id":"educational.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"educational%3:01:00::","call_ids":["senseval2.d002.s033.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"educational%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":483,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001526,"input_uncached_usd":0.000966,"input_cached_usd":0.0,"output_usd":0.0005600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3179973340593278}
{"item_id":"senseval2.d002.s033.t003","gold_sense_keys":["problem%1:09:00::","problem%1:26:00::"],"candidates":[{"index":1,"sense_key":"problem%1:26:00::","synset_id":"problem.n.01"},{"index":2,"sense_key":"problem%1:10:00::","synset_id":"problem.n.02"},{"index":3,"sense_key":"problem%1:09:00::","synset_id":"trouble.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"problem%1:26:00::","call_ids":["senseval2.d002.s033.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"problem%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":582,"cached_input_tokens":0,"output_tokens":229,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0034540000000000005,"input_uncached_usd":0.0011639999999999999,"input_cached_usd":0.0,"output_usd":0.0022900000000000004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.91689212503843}
{"item_id":"senseval2.d002.s034.t002","gold_sense_keys":["teacher%1:18:00::"],"candidates":[{"index":1,"sense_key":"teacher%1:18:00::","synset_id":"teacher.n.01"},{"index":2,"sense_key":"teacher%1:09:00::","synset_id":"teacher.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"teacher%1:18:00::","call_ids":["senseval2.d002.s034.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"teacher%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":510,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00114,"input_uncached_usd":0.0010199999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0850245419424027}
{"item_id":"senseval2.d002.s034.t003","gold_sense_keys":["think%2:31:03::"],"candidates":[{"index":1,"sense_key":"think%2:31:01::","synset_id":"think.v.01"},{"index":2,"sense_key":"think%2:31:03::","synset_id":"think.v.02"},{"index":3,"sense_key":"think%2:31:00::","synset_id":"think.v.03"},{"index":4,"sense_key":"think%2:31:02::","synset_id":"remember.v.01"},{"index":5,"sense_key":"think%2:36:00::","synset_id":"think.v.05"},{"index":6,"sense_key":"think%2:31:09::","synset_id":"think.v.06"},{"index":7,"sense_key":"think%2:31:06::","synset_id":"intend.v.01"},{"index":8,"sense_key":"think%2:31:12::","synset_id":"think.v.08"},{"index":9,"sense_key":"think%2:31:13::","synset_id":"think.v.09"},{"index":10,"sense_key":"think%2:31:15::","synset_id":"think.v.10"},{"index":11,"sense_key":"think%2:31:14::","synset_id":"think.v.11"},{"index":12,"sense_key":"think%2:31:08::","synset_id":"think.v.12"},{"index":13,"sense_key":"think%2:30:00::","synset_id":"think.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"think%2:31:03::","call_ids":["senseval2.d002.s034.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"think%2:31:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1198,"cached_input_tokens":0,"output_tokens":132,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0037159999999999997,"input_uncached_usd":0.0023959999999999997,"input_cached_usd":0.0,"output_usd":0.0013200000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7575706250499934}
{"item_id":"senseval2.d002.s034.t004","gold_sense_keys":["educate%2:41:00::"],"candidates":[{"index":1,"sense_key":"educate%2:41:00::","synset_id":"educate.v.01"},{"index":2,"sense_key":"educate%2:31:00::","synset_id":"train.v.01"},{"index":3,"sense_key":"educate%2:41:01::","synset_id":"educate.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"educate%2:41:00::","call_ids":["senseval2.d002.s034.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"educate%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":604,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001328,"input_uncached_usd":0.0012079999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.7798273749649525}
{"item_id":"senseval2.d002.s034.t005","gold_sense_keys":["child%1:18:00::"],"candidates":[{"index":1,"sense_key":"child%1:18:00::","synset_id":"child.n.01"},{"index":2,"sense_key":"child%1:18:01::","synset_id":"child.n.02"},{"index":3,"sense_key":"child%1:18:06::","synset_id":"child.n.03"},{"index":4,"sense_key":"child%1:18:07::","synset_id":"child.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"child%1:18:00::","call_ids":["senseval2.d002.s034.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"child%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":646,"cached_input_tokens":0,"output_tokens":102,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002312,"input_uncached_usd":0.001292,"input_cached_usd":0.0,"output_usd":0.00102,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8934911659453064}
{"item_id":"senseval2.d002.s034.t006","gold_sense_keys":["better%4:02:00::"],"candidates":[{"index":1,"sense_key":"better%4:02:00::","synset_id":"better.r.01"},{"index":2,"sense_key":"better%4:02:01::","synset_id":"better.r.02"},{"index":3,"sense_key":"well%4:02:00::","synset_id":"well.r.01"},{"index":4,"sense_key":"well%4:02:05::","synset_id":"well.r.02"},{"index":5,"sense_key":"well%4:02:03::","synset_id":"well.r.03"},{"index":6,"sense_key":"well%4:02:15::","synset_id":"well.r.04"},{"index":7,"sense_key":"well%4:02:08::","synset_id":"well.r.05"},{"index":8,"sense_key":"well%4:02:07::","synset_id":"well.r.06"},{"index":9,"sense_key":"well%4:02:11::","synset_id":"well.r.07"},{"index":10,"sense_key":"well%4:02:14::","synset_id":"well.r.08"},{"index":11,"sense_key":"well%4:02:13::","synset_id":"well.r.09"},{"index":12,"sense_key":"well%4:02:12::","synset_id":"well.r.10"},{"index":13,"sense_key":"well%4:02:10::","synset_id":"well.r.11"},{"index":14,"sense_key":"well%4:02:09::","synset_id":"well.r.12"},{"index":15,"sense_key":"well%4:02:01::","synset_id":"well.r.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"better%4:02:00::","call_ids":["senseval2.d002.s034.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"better%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1405,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00293,"input_uncached_usd":0.00281,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.44170045806095}
{"item_id":"senseval2.d002.s034.t008","gold_sense_keys":["get%2:40:00::"],"candidates":[{"index":1,"sense_key":"get%2:40:00::","synset_id":"get.v.01"},{"index":2,"sense_key":"get%2:30:00::","synset_id":"become.v.01"},{"index":3,"sense_key":"get%2:30:02::","synset_id":"get.v.03"},{"index":4,"sense_key":"get%2:30:03::","synset_id":"receive.v.02"},{"index":5,"sense_key":"get%2:38:00::","synset_id":"arrive.v.01"},{"index":6,"sense_key":"get%2:35:03::","synset_id":"bring.v.04"},{"index":7,"sense_key":"get%2:39:00::","synset_id":"experience.v.03"},{"index":8,"sense_key":"get%2:33:00::","synset_id":"pay_back.v.02"},{"index":9,"sense_key":"get%2:40:01::","synset_id":"have.v.17"},{"index":10,"sense_key":"get%2:32:00::","synset_id":"induce.v.02"},{"index":11,"sense_key":"get%2:35:09::","synset_id":"get.v.11"},{"index":12,"sense_key":"get%2:29:01::","synset_id":"grow.v.08"},{"index":13,"sense_key":"get%2:29:00::","synset_id":"contract.v.04"},{"index":14,"sense_key":"get%2:32:03::","synset_id":"get.v.14"},{"index":15,"sense_key":"get%2:30:01::","synset_id":"make.v.02"},{"index":16,"sense_key":"get%2:32:08::","synset_id":"drive.v.11"},{"index":17,"sense_key":"get%2:31:09::","synset_id":"catch.v.18"},{"index":18,"sense_key":"get%2:35:00::","synset_id":"catch.v.07"},{"index":19,"sense_key":"get%2:35:10::","synset_id":"get.v.19"},{"index":20,"sense_key":"get%2:31:03::","synset_id":"get.v.20"},{"index":21,"sense_key":"get%2:40:14::","synset_id":"get.v.21"},{"index":22,"sense_key":"get%2:40:13::","synset_id":"get.v.22"},{"index":23,"sense_key":"get%2:39:13::","synset_id":"catch.v.21"},{"index":24,"sense_key":"get%2:39:09::","synset_id":"catch.v.22"},{"index":25,"sense_key":"get%2:39:14::","synset_id":"get.v.25"},{"index":26,"sense_key":"get%2:38:15::","synset_id":"scram.v.01"},{"index":27,"sense_key":"get%2:38:12::","synset_id":"get.v.27"},{"index":28,"sense_key":"get%2:37:12::","synset_id":"get.v.28"},{"index":29,"sense_key":"get%2:37:11::","synset_id":"get.v.29"},{"index":30,"sense_key":"get%2:36:14::","synset_id":"catch.v.24"},{"index":31,"sense_key":"get%2:36:00::","synset_id":"draw.v.15"},{"index":32,"sense_key":"get%2:35:12::","synset_id":"get.v.32"},{"index":33,"sense_key":"get%2:31:10::","synset_id":"perplex.v.01"},{"index":34,"sense_key":"get%2:30:12::","synset_id":"get_down.v.07"},{"index":35,"sense_key":"get%2:29:11::","synset_id":"suffer.v.02"},{"index":36,"sense_key":"get%2:29:08::","synset_id":"beget.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"get%2:40:00::","call_ids":["senseval2.d002.s034.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"get%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2751,"cached_input_tokens":0,"output_tokens":150,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0070019999999999995,"input_uncached_usd":0.005502,"input_cached_usd":0.0,"output_usd":0.0015,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1460242919856682}
{"item_id":"senseval2.d002.s034.t010","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["senseval2.d002.s034.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":668,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001456,"input_uncached_usd":0.001336,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3939391250023618}
{"item_id":"senseval2.d002.s034.t011","gold_sense_keys":["more%4:02:01::"],"candidates":[{"index":1,"sense_key":"more%4:02:00::","synset_id":"more.r.01"},{"index":2,"sense_key":"more%4:02:01::","synset_id":"more.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"more%4:02:01::","call_ids":["senseval2.d002.s034.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"more%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":525,"cached_input_tokens":0,"output_tokens":87,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00192,"input_uncached_usd":0.00105,"input_cached_usd":0.0,"output_usd":0.0008700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4945345830637962}
{"item_id":"senseval2.d002.s035.t002","gold_sense_keys":["teacher%1:18:00::"],"candidates":[{"index":1,"sense_key":"teacher%1:18:00::","synset_id":"teacher.n.01"},{"index":2,"sense_key":"teacher%1:09:00::","synset_id":"teacher.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"teacher%1:18:00::","call_ids":["senseval2.d002.s035.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"teacher%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":529,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001178,"input_uncached_usd":0.001058,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4716430419357494}
{"item_id":"senseval2.d002.s035.t003","gold_sense_keys":["union%1:14:01::"],"candidates":[{"index":1,"sense_key":"union%1:14:01::","synset_id":"union.n.01"},{"index":2,"sense_key":"union%1:15:00::","synset_id":"union.n.02"},{"index":3,"sense_key":"union%1:04:02::","synset_id":"coupling.n.03"},{"index":4,"sense_key":"union%1:26:00::","synset_id":"union.n.04"},{"index":5,"sense_key":"union%1:26:02::","synset_id":"marriage.n.01"},{"index":6,"sense_key":"union%1:22:00::","synset_id":"union.n.06"},{"index":7,"sense_key":"union%1:14:02::","synset_id":"union.n.07"},{"index":8,"sense_key":"union%1:14:03::","synset_id":"union.n.08"},{"index":9,"sense_key":"union%1:11:00::","synset_id":"union.n.09"},{"index":10,"sense_key":"union%1:06:01::","synset_id":"union.n.10"},{"index":11,"sense_key":"union%1:04:00::","synset_id":"union.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"union%1:14:01::","call_ids":["senseval2.d002.s035.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"union%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1179,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0024779999999999997,"input_uncached_usd":0.002358,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.334703208063729}
{"item_id":"senseval2.d002.s035.t004","gold_sense_keys":["mind%2:32:00::"],"candidates":[{"index":1,"sense_key":"mind%2:32:00::","synset_id":"mind.v.01"},{"index":2,"sense_key":"mind%2:31:02::","synset_id":"mind.v.02"},{"index":3,"sense_key":"mind%2:41:01::","synset_id":"take_care.v.02"},{"index":4,"sense_key":"mind%2:41:00::","synset_id":"heed.v.01"},{"index":5,"sense_key":"mind%2:31:03::","synset_id":"beware.v.01"},{"index":6,"sense_key":"mind%2:31:00::","synset_id":"mind.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"mind%2:32:00::","call_ids":["senseval2.d002.s035.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"mind%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":735,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00206,"input_uncached_usd":0.00147,"input_cached_usd":0.0,"output_usd":0.00059,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.782769790966995}
{"item_id":"senseval2.d002.s035.t005","gold_sense_keys":["spread%2:32:00::","spread%2:35:00::"],"candidates":[{"index":1,"sense_key":"spread%2:35:00::","synset_id":"spread.v.01"},{"index":2,"sense_key":"spread%2:35:07::","synset_id":"spread.v.02"},{"index":3,"sense_key":"spread%2:42:00::","synset_id":"spread.v.03"},{"index":4,"sense_key":"spread%2:35:03::","synset_id":"unfold.v.04"},{"index":5,"sense_key":"spread%2:32:00::","synset_id":"circulate.v.02"},{"index":6,"sense_key":"spread%2:32:02::","synset_id":"go_around.v.02"},{"index":7,"sense_key":"spread%2:38:00::","synset_id":"spread.v.07"},{"index":8,"sense_key":"spread%2:38:02::","synset_id":"diffuse.v.01"},{"index":9,"sense_key":"spread%2:35:12::","synset_id":"spread.v.09"},{"index":10,"sense_key":"spread%2:35:13::","synset_id":"spread.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"spread%2:32:00::","call_ids":["senseval2.d002.s035.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"spread%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1010,"cached_input_tokens":0,"output_tokens":87,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00289,"input_uncached_usd":0.00202,"input_cached_usd":0.0,"output_usd":0.0008700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.253134125028737}
{"item_id":"senseval2.d002.s035.t007","gold_sense_keys":["narrow%5:00:00:limited:00"],"candidates":[{"index":1,"sense_key":"narrow%3:00:00::","synset_id":"narrow.a.01"},{"index":2,"sense_key":"narrow%5:00:00:limited:00","synset_id":"narrow.s.01"},{"index":3,"sense_key":"narrow%3:00:06::","synset_id":"narrow-minded.a.02"},{"index":4,"sense_key":"narrow%3:00:02::","synset_id":"narrow.a.04"},{"index":5,"sense_key":"narrow%5:00:00:careful:00","synset_id":"minute.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"narrow%5:00:00:limited:00","call_ids":["senseval2.d002.s035.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"narrow%5:00:00:limited:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":725,"cached_input_tokens":0,"output_tokens":186,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00331,"input_uncached_usd":0.00145,"input_cached_usd":0.0,"output_usd":0.00186,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.77343658299651}
{"item_id":"senseval2.d002.s035.t008","gold_sense_keys":["purpose%1:09:00::"],"candidates":[{"index":1,"sense_key":"purpose%1:09:00::","synset_id":"purpose.n.01"},{"index":2,"sense_key":"purpose%1:07:00::","synset_id":"function.n.02"},{"index":3,"sense_key":"purpose%1:07:01::","synset_id":"determination.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"purpose%1:09:00::","call_ids":["senseval2.d002.s035.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"purpose%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":661,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0014420000000000001,"input_uncached_usd":0.001322,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4159911669557914}
{"item_id":"senseval2.d002.s036.t003","gold_sense_keys":["politician%1:18:00::","politician%1:18:01::"],"candidates":[{"index":1,"sense_key":"politician%1:18:01::","synset_id":"politician.n.01"},{"index":2,"sense_key":"politician%1:18:00::","synset_id":"politician.n.02"},{"index":3,"sense_key":"politician%1:18:02::","synset_id":"politician.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"politician%1:18:00::","call_ids":["senseval2.d002.s036.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"politician%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":564,"cached_input_tokens":0,"output_tokens":79,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001918,"input_uncached_usd":0.0011279999999999999,"input_cached_usd":0.0,"output_usd":0.00079,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5606652910355479}
{"item_id":"senseval2.d002.s036.t004","gold_sense_keys":["find%2:31:10::"],"candidates":[{"index":1,"sense_key":"find%2:40:02::","synset_id":"find.v.01"},{"index":2,"sense_key":"find%2:39:02::","synset_id":"detect.v.01"},{"index":3,"sense_key":"find%2:40:00::","synset_id":"find.v.03"},{"index":4,"sense_key":"find%2:32:00::","synset_id":"determine.v.01"},{"index":5,"sense_key":"find%2:31:10::","synset_id":"find.v.05"},{"index":6,"sense_key":"find%2:39:01::","synset_id":"witness.v.02"},{"index":7,"sense_key":"find%2:40:01::","synset_id":"line_up.v.02"},{"index":8,"sense_key":"find%2:36:00::","synset_id":"discover.v.03"},{"index":9,"sense_key":"find%2:31:09::","synset_id":"discover.v.04"},{"index":10,"sense_key":"find%2:40:03::","synset_id":"find.v.10"},{"index":11,"sense_key":"find%2:32:01::","synset_id":"rule.v.04"},{"index":12,"sense_key":"find%2:30:00::","synset_id":"receive.v.02"},{"index":13,"sense_key":"find%2:39:05::","synset_id":"find.v.13"},{"index":14,"sense_key":"find%2:40:15::","synset_id":"recover.v.01"},{"index":15,"sense_key":"find%2:38:10::","synset_id":"find.v.15"},{"index":16,"sense_key":"find%2:30:13::","synset_id":"find_oneself.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"find%2:31:10::","call_ids":["senseval2.d002.s036.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"find%2:31:10::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1586,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003902,"input_uncached_usd":0.003172,"input_cached_usd":0.0,"output_usd":0.0007300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3251279999967664}
{"item_id":"senseval2.d002.s036.t006","gold_sense_keys":["help%2:41:00::"],"candidates":[{"index":1,"sense_key":"help%2:41:00::","synset_id":"help.v.01"},{"index":2,"sense_key":"help%2:29:00::","synset_id":"help.v.02"},{"index":3,"sense_key":"help%2:42:00::","synset_id":"help.v.03"},{"index":4,"sense_key":"help%2:42:08::","synset_id":"help_oneself.v.01"},{"index":5,"sense_key":"help%2:34:00::","synset_id":"serve.v.05"},{"index":6,"sense_key":"help%2:41:12::","synset_id":"help.v.06"},{"index":7,"sense_key":"help%2:34:04::","synset_id":"avail.v.03"},{"index":8,"sense_key":"help%2:30:13::","synset_id":"help.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"help%2:42:00::","call_ids":["senseval2.d002.s036.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"help%2:42:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":880,"cached_input_tokens":0,"output_tokens":86,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00262,"input_uncached_usd":0.0017599999999999998,"input_cached_usd":0.0,"output_usd":0.0008600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0595333750825375}
{"item_id":"senseval2.d002.s036.t007","gold_sense_keys":["strike%2:38:08::","strike%2:41:02::"],"candidates":[{"index":1,"sense_key":"strike%2:35:00::","synset_id":"strike.v.01"},{"index":2,"sense_key":"strike%2:37:00::","synset_id":"affect.v.05"},{"index":3,"sense_key":"strike%2:35:01::","synset_id":"hit.v.02"},{"index":4,"sense_key":"strike%2:33:00::","synset_id":"strike.v.04"},{"index":5,"sense_key":"strike%2:32:00::","synset_id":"strike.v.05"},{"index":6,"sense_key":"strike%2:35:02::","synset_id":"hit.v.05"},{"index":7,"sense_key":"strike%2:41:00::","synset_id":"strike.v.07"},{"index":8,"sense_key":"strike%2:42:00::","synset_id":"fall.v.08"},{"index":9,"sense_key":"strike%2:41:02::","synset_id":"come_to.v.03"},{"index":10,"sense_key":"strike%2:35:03::","synset_id":"strike.v.10"},{"index":11,"sense_key":"strike%2:30:00::","synset_id":"strike.v.11"},{"index":12,"sense_key":"strike%2:40:00::","synset_id":"fall_upon.v.01"},{"index":13,"sense_key":"strike%2:36:00::","synset_id":"strike.v.13"},{"index":14,"sense_key":"strike%2:35:08::","synset_id":"strike.v.14"},{"index":15,"sense_key":"strike%2:39:00::","synset_id":"hit.v.09"},{"index":16,"sense_key":"strike%2:38:02::","synset_id":"hit.v.12"},{"index":17,"sense_key":"strike%2:38:08::","synset_id":"assume.v.05"},{"index":18,"sense_key":"strike%2:36:02::","synset_id":"mint.v.01"},{"index":19,"sense_key":"strike%2:35:10::","synset_id":"strickle.v.02"},{"index":20,"sense_key":"strike%2:35:09::","synset_id":"strike.v.20"},{"index":21,"sense_key":"strike%2:31:00::","synset_id":"strike.v.21"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":17,"chosen_sense_key":"strike%2:38:08::","call_ids":["senseval2.d002.s036.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":17,"predicted_sense_key":"strike%2:38:08::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1842,"cached_input_tokens":0,"output_tokens":71,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0043939999999999995,"input_uncached_usd":0.0036839999999999998,"input_cached_usd":0.0,"output_usd":0.00071,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3872589999809861}
{"item_id":"senseval2.d002.s036.t008","gold_sense_keys":["friendly%5:00:00:amicable:00"],"candidates":[{"index":1,"sense_key":"friendly%3:00:01::","synset_id":"friendly.a.01"},{"index":2,"sense_key":"friendly%5:00:00:amicable:00","synset_id":"friendly.s.01"},{"index":3,"sense_key":"friendly%3:00:03::","synset_id":"friendly.a.03"},{"index":4,"sense_key":"friendly%3:00:02::","synset_id":"friendly.a.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"friendly%5:00:00:amicable:00","call_ids":["senseval2.d002.s036.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"friendly%5:00:00:amicable:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":693,"cached_input_tokens":0,"output_tokens":120,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0025859999999999998,"input_uncached_usd":0.0013859999999999999,"input_cached_usd":0.0,"output_usd":0.0012000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6494977090042084}
{"item_id":"senseval2.d002.s036.t009","gold_sense_keys":["posture%1:09:00::"],"candidates":[{"index":1,"sense_key":"posture%1:07:00::","synset_id":"position.n.04"},{"index":2,"sense_key":"posture%1:07:01::","synset_id":"carriage.n.03"},{"index":3,"sense_key":"posture%1:09:00::","synset_id":"position.n.12"},{"index":4,"sense_key":"posture%1:07:02::","synset_id":"military_capability.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"posture%1:09:00::","call_ids":["senseval2.d002.s036.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"posture%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":690,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0015,"input_uncached_usd":0.00138,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0792953330092132}
{"item_id":"senseval2.d002.s036.t010","gold_sense_keys":["behalf%1:04:00::"],"candidates":[{"index":1,"sense_key":"behalf%1:04:00::","synset_id":"behalf.n.01"},{"index":2,"sense_key":"behalf%1:07:00::","synset_id":"behalf.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"behalf%1:04:00::","call_ids":["senseval2.d002.s036.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"behalf%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":582,"cached_input_tokens":0,"output_tokens":110,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002264,"input_uncached_usd":0.0011639999999999999,"input_cached_usd":0.0,"output_usd":0.0011,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7479099580086768}
{"item_id":"senseval2.d002.s036.t011","gold_sense_keys":["important%3:00:00::","important%3:00:02::","important%3:00:04::"],"candidates":[{"index":1,"sense_key":"important%3:00:00::","synset_id":"important.a.01"},{"index":2,"sense_key":"important%3:00:04::","synset_id":"significant.a.01"},{"index":3,"sense_key":"important%3:00:02::","synset_id":"crucial.a.01"},{"index":4,"sense_key":"important%5:00:00:influential:00","synset_id":"authoritative.s.01"},{"index":5,"sense_key":"important%5:00:00:immodest:02","synset_id":"important.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"important%5:00:00:influential:00","call_ids":["senseval2.d002.s036.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"important%5:00:00:influential:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":744,"cached_input_tokens":0,"output_tokens":115,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002638,"input_uncached_usd":0.001488,"input_cached_usd":0.0,"output_usd":0.0011500000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":4.550304958946072}
{"item_id":"senseval2.d002.s037.t000","gold_sense_keys":["shred%1:23:00::"],"candidates":[{"index":1,"sense_key":"shred%1:23:00::","synset_id":"shred.n.01"},{"index":2,"sense_key":"shred%1:06:00::","synset_id":"rag.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"shred%1:23:00::","call_ids":["senseval2.d002.s037.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"shred%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":560,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00124,"input_uncached_usd":0.00112,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.7700376670109108}
{"item_id":"senseval2.d002.s037.t001","gold_sense_keys":["evidence%1:09:00::","evidence%1:10:01::"],"candidates":[{"index":1,"sense_key":"evidence%1:09:00::","synset_id":"evidence.n.01"},{"index":2,"sense_key":"evidence%1:10:01::","synset_id":"evidence.n.02"},{"index":3,"sense_key":"evidence%1:10:00::","synset_id":"evidence.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"evidence%1:09:00::","call_ids":["senseval2.d002.s037.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"evidence%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":629,"cached_input_tokens":0,"output_tokens":89,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002148,"input_uncached_usd":0.001258,"input_cached_usd":0.0,"output_usd":0.0008900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9574742499971762}
{"item_id":"senseval2.d002.s037.t002","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":2,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":3,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"other%3:00:00::","call_ids":["senseval2.d002.s037.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":674,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0014680000000000001,"input_uncached_usd":0.001348,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2340696669416502}
{"item_id":"senseval2.d002.s037.t003","gold_sense_keys":["thing%1:09:02::"],"candidates":[{"index":1,"sense_key":"thing%1:26:00::","synset_id":"thing.n.01"},{"index":2,"sense_key":"thing%1:04:00::","synset_id":"thing.n.02"},{"index":3,"sense_key":"thing%1:09:00::","synset_id":"thing.n.03"},{"index":4,"sense_key":"thing%1:06:00::","synset_id":"thing.n.04"},{"index":5,"sense_key":"thing%1:11:00::","synset_id":"thing.n.05"},{"index":6,"sense_key":"thing%1:09:02::","synset_id":"matter.n.01"},{"index":7,"sense_key":"thing%1:10:00::","synset_id":"thing.n.07"},{"index":8,"sense_key":"thing%1:06:01::","synset_id":"thing.n.08"},{"index":9,"sense_key":"thing%1:07:00::","synset_id":"thing.n.09"},{"index":10,"sense_key":"thing%1:09:01::","synset_id":"thing.n.10"},{"index":11,"sense_key":"thing%1:12:00::","synset_id":"thing.n.11"},{"index":12,"sense_key":"thing%1:03:00::","synset_id":"thing.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"thing%1:26:00::","call_ids":["senseval2.d002.s037.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"thing%1:26:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":996,"cached_input_tokens":0,"output_tokens":174,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003732,"input_uncached_usd":0.001992,"input_cached_usd":0.0,"output_usd":0.0017400000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.23757229198236}
{"item_id":"senseval2.d002.s037.t004","gold_sense_keys":["equal%3:00:00::"],"candidates":[{"index":1,"sense_key":"equal%3:00:00::","synset_id":"equal.a.01"},{"index":2,"sense_key":"equal%3:00:03::","synset_id":"adequate.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"equal%3:00:00::","call_ids":["senseval2.d002.s037.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"equal%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":559,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0018479999999999998,"input_uncached_usd":0.0011179999999999999,"input_cached_usd":0.0,"output_usd":0.0007300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5398417499382049}
{"item_id":"senseval2.d002.s037.t006","gold_sense_keys":["differential%1:07:00::"],"candidates":[{"index":1,"sense_key":"differential%1:09:00::","synset_id":"derived_function.n.01"},{"index":2,"sense_key":"differential%1:07:00::","synset_id":"differential.n.02"},{"index":3,"sense_key":"differential%1:06:00::","synset_id":"differential_gear.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"differential%1:07:00::","call_ids":["senseval2.d002.s037.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"differential%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":644,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001878,"input_uncached_usd":0.0012879999999999999,"input_cached_usd":0.0,"output_usd":0.00059,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.781132375006564}
{"item_id":"senseval2.d002.s037.t007","gold_sense_keys":["result%2:42:00::"],"candidates":[{"index":1,"sense_key":"result%2:42:00::","synset_id":"result.v.01"},{"index":2,"sense_key":"result%2:42:02::","synset_id":"leave.v.07"},{"index":3,"sense_key":"result%2:30:00::","synset_id":"result.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"result%2:42:00::","call_ids":["senseval2.d002.s037.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"result%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":613,"cached_input_tokens":0,"output_tokens":131,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002536,"input_uncached_usd":0.0012259999999999999,"input_cached_usd":0.0,"output_usd":0.0013100000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0737810420105234}
{"item_id":"senseval2.d002.s037.t008","gold_sense_keys":["educational%3:01:00::"],"candidates":[{"index":1,"sense_key":"educational%3:01:00::","synset_id":"educational.a.01"},{"index":2,"sense_key":"educational%5:00:00:instructive:00","synset_id":"educational.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"educational%3:01:00::","call_ids":["senseval2.d002.s037.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"educational%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":520,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0017000000000000001,"input_uncached_usd":0.00104,"input_cached_usd":0.0,"output_usd":0.0006600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9111097909044474}
{"item_id":"senseval2.d002.s037.t009","gold_sense_keys":["differential%1:07:00::"],"candidates":[{"index":1,"sense_key":"differential%1:09:00::","synset_id":"derived_function.n.01"},{"index":2,"sense_key":"differential%1:07:00::","synset_id":"differential.n.02"},{"index":3,"sense_key":"differential%1:06:00::","synset_id":"differential_gear.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"differential%1:07:00::","call_ids":["senseval2.d002.s037.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"differential%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":644,"cached_input_tokens":0,"output_tokens":79,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002078,"input_uncached_usd":0.0012879999999999999,"input_cached_usd":0.0,"output_usd":0.00079,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0080603749956936}
{"item_id":"senseval2.d002.s038.t001","gold_sense_keys":["evidence%1:09:00::","evidence%1:10:01::"],"candidates":[{"index":1,"sense_key":"evidence%1:09:00::","synset_id":"evidence.n.01"},{"index":2,"sense_key":"evidence%1:10:01::","synset_id":"evidence.n.02"},{"index":3,"sense_key":"evidence%1:10:00::","synset_id":"evidence.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"evidence%1:09:00::","call_ids":["senseval2.d002.s038.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"evidence%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":643,"cached_input_tokens":0,"output_tokens":83,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0021160000000000003,"input_uncached_usd":0.001286,"input_cached_usd":0.0,"output_usd":0.0008300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6278487920062616}
{"item_id":"senseval2.d002.s038.t002","gold_sense_keys":["sure%3:00:00::"],"candidates":[{"index":1,"sense_key":"sure%3:00:00::","synset_id":"certain.a.02"},{"index":2,"sense_key":"sure%5:00:00:careful:00","synset_id":"certain.s.03"},{"index":3,"sense_key":"sure%3:00:04::","synset_id":"certain.a.04"},{"index":4,"sense_key":"sure%5:00:00:secure:02","synset_id":"sure.s.02"},{"index":5,"sense_key":"sure%5:00:00:reliable:00","synset_id":"certain.s.02"},{"index":6,"sense_key":"sure%5:00:00:trustworthy:00","synset_id":"sure.s.04"},{"index":7,"sense_key":"sure%5:00:00:true:00","synset_id":"sure.s.05"},{"index":8,"sense_key":"sure%5:00:00:steady:00","synset_id":"sure.s.06"},{"index":9,"sense_key":"sure%5:00:02:certain:01","synset_id":"indisputable.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"sure%3:00:00::","call_ids":["senseval2.d002.s038.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"sure%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1032,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002674,"input_uncached_usd":0.002064,"input_cached_usd":0.0,"output_usd":0.0006100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8467741661006585}
{"item_id":"senseval2.d002.s039.t000","gold_sense_keys":["wish%2:37:02::"],"candidates":[{"index":1,"sense_key":"wish%2:37:00::","synset_id":"wish.v.01"},{"index":2,"sense_key":"wish%2:37:02::","synset_id":"wish.v.02"},{"index":3,"sense_key":"wish%2:32:02::","synset_id":"wish.v.03"},{"index":4,"sense_key":"wish%2:37:01::","synset_id":"wish.v.04"},{"index":5,"sense_key":"wish%2:32:01::","synset_id":"wish.v.05"},{"index":6,"sense_key":"wish%2:32:00::","synset_id":"wish.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"wish%2:37:02::","call_ids":["senseval2.d002.s039.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"wish%2:37:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":754,"cached_input_tokens":0,"output_tokens":80,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002308,"input_uncached_usd":0.001508,"input_cached_usd":0.0,"output_usd":0.0008,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.468755541020073}
{"item_id":"senseval2.d002.s039.t001","gold_sense_keys":["serious%3:00:00::"],"candidates":[{"index":1,"sense_key":"serious%3:00:00::","synset_id":"serious.a.01"},{"index":2,"sense_key":"serious%5:00:00:important:00","synset_id":"serious.s.01"},{"index":3,"sense_key":"serious%5:00:00:critical:03","synset_id":"dangerous.s.01"},{"index":4,"sense_key":"serious%5:00:00:intellectual:00","synset_id":"good.s.14"},{"index":5,"sense_key":"serious%3:00:02::","synset_id":"unplayful.a.01"},{"index":6,"sense_key":"serious%5:00:00:difficult:00","synset_id":"serious.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"serious%3:00:00::","call_ids":["senseval2.d002.s039.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"serious%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":832,"cached_input_tokens":0,"output_tokens":91,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002574,"input_uncached_usd":0.001664,"input_cached_usd":0.0,"output_usd":0.0009100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4107524589635432}
{"item_id":"senseval2.d002.s039.t002","gold_sense_keys":["american%3:01:01::"],"candidates":[{"index":1,"sense_key":"american%3:01:01::","synset_id":"american.a.01"},{"index":2,"sense_key":"american%3:01:00::","synset_id":"american.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"american%3:01:01::","call_ids":["senseval2.d002.s039.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"american%3:01:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":578,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001276,"input_uncached_usd":0.001156,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0301113749155775}
{"item_id":"senseval2.d002.s039.t003","gold_sense_keys":["education%1:04:00::"],"candidates":[{"index":1,"sense_key":"education%1:04:00::","synset_id":"education.n.01"},{"index":2,"sense_key":"education%1:09:00::","synset_id":"education.n.02"},{"index":3,"sense_key":"education%1:09:01::","synset_id":"education.n.03"},{"index":4,"sense_key":"education%1:04:01::","synset_id":"education.n.04"},{"index":5,"sense_key":"education%1:07:01::","synset_id":"education.n.05"},{"index":6,"sense_key":"education%1:14:00::","synset_id":"department_of_education.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"education%1:04:00::","call_ids":["senseval2.d002.s039.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"education%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":857,"cached_input_tokens":0,"output_tokens":128,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002994,"input_uncached_usd":0.001714,"input_cached_usd":0.0,"output_usd":0.00128,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.661179875023663}
{"item_id":"senseval2.d002.s039.t004","gold_sense_keys":["know%2:31:01::"],"candidates":[{"index":1,"sense_key":"know%2:31:01::","synset_id":"know.v.01"},{"index":2,"sense_key":"know%2:31:03::","synset_id":"know.v.02"},{"index":3,"sense_key":"know%2:31:02::","synset_id":"know.v.03"},{"index":4,"sense_key":"know%2:31:00::","synset_id":"know.v.04"},{"index":5,"sense_key":"know%2:31:04::","synset_id":"know.v.05"},{"index":6,"sense_key":"know%2:31:06::","synset_id":"acknowledge.v.06"},{"index":7,"sense_key":"know%2:31:15::","synset_id":"know.v.07"},{"index":8,"sense_key":"know%2:35:00::","synset_id":"sleep_together.v.01"},{"index":9,"sense_key":"know%2:31:11::","synset_id":"know.v.09"},{"index":10,"sense_key":"know%2:31:12::","synset_id":"know.v.10"},{"index":11,"sense_key":"know%2:31:14::","synset_id":"know.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"know%2:31:01::","call_ids":["senseval2.d002.s039.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"know%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1188,"cached_input_tokens":0,"output_tokens":150,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003876,"input_uncached_usd":0.002376,"input_cached_usd":0.0,"output_usd":0.0015,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9572153330082074}
{"item_id":"senseval2.d002.s039.t005","gold_sense_keys":["exactly%4:02:01::"],"candidates":[{"index":1,"sense_key":"exactly%4:02:01::","synset_id":"precisely.r.01"},{"index":2,"sense_key":"exactly%4:02:04::","synset_id":"precisely.r.03"},{"index":3,"sense_key":"exactly%4:02:02::","synset_id":"precisely.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"exactly%4:02:01::","call_ids":["senseval2.d002.s039.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"exactly%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":678,"cached_input_tokens":0,"output_tokens":119,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002546,"input_uncached_usd":0.001356,"input_cached_usd":0.0,"output_usd":0.00119,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6197377920616418}
{"item_id":"senseval2.d002.s039.t006","gold_sense_keys":["just%4:02:05::"],"candidates":[{"index":1,"sense_key":"just%4:02:00::","synset_id":"merely.r.01"},{"index":2,"sense_key":"just%4:02:05::","synset_id":"precisely.r.01"},{"index":3,"sense_key":"just%4:02:01::","synset_id":"just.r.03"},{"index":4,"sense_key":"just%4:02:03::","synset_id":"just.r.04"},{"index":5,"sense_key":"just%4:02:04::","synset_id":"barely.r.01"},{"index":6,"sense_key":"just%4:02:06::","synset_id":"just.r.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"just%4:02:05::","call_ids":["senseval2.d002.s039.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"just%4:02:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":863,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002316,"input_uncached_usd":0.0017259999999999999,"input_cached_usd":0.0,"output_usd":0.00059,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3815826670033857}
{"item_id":"senseval2.d002.s039.t007","gold_sense_keys":["important%3:00:00::","important%3:00:02::","important%3:00:04::"],"candidates":[{"index":1,"sense_key":"important%3:00:00::","synset_id":"important.a.01"},{"index":2,"sense_key":"important%3:00:04::","synset_id":"significant.a.01"},{"index":3,"sense_key":"important%3:00:02::","synset_id":"crucial.a.01"},{"index":4,"sense_key":"important%5:00:00:influential:00","synset_id":"authoritative.s.01"},{"index":5,"sense_key":"important%5:00:00:immodest:02","synset_id":"important.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"important%3:00:00::","call_ids":["senseval2.d002.s039.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"important%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":775,"cached_input_tokens":0,"output_tokens":96,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00251,"input_uncached_usd":0.00155,"input_cached_usd":0.0,"output_usd":0.0009600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.010037334053777}
{"item_id":"senseval2.d002.s040.t002","gold_sense_keys":["school%1:14:00::"],"candidates":[{"index":1,"sense_key":"school%1:14:00::","synset_id":"school.n.01"},{"index":2,"sense_key":"school%1:06:00::","synset_id":"school.n.02"},{"index":3,"sense_key":"school%1:09:00::","synset_id":"school.n.03"},{"index":4,"sense_key":"school%1:14:02::","synset_id":"school.n.04"},{"index":5,"sense_key":"school%1:28:00::","synset_id":"school.n.05"},{"index":6,"sense_key":"school%1:14:03::","synset_id":"school.n.06"},{"index":7,"sense_key":"school%1:14:01::","synset_id":"school.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"school%1:14:00::","call_ids":["senseval2.d002.s040.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"school%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":871,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002372,"input_uncached_usd":0.0017419999999999998,"input_cached_usd":0.0,"output_usd":0.00063,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7252195830224082}
{"item_id":"senseval2.d002.s040.t003","gold_sense_keys":["scattered%5:00:00:distributed:00"],"candidates":[{"index":1,"sense_key":"scattered%5:00:00:distributed:00","synset_id":"scattered.s.01"},{"index":2,"sense_key":"scattered%5:00:00:incoherent:00","synset_id":"confused.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"scattered%5:00:00:distributed:00","call_ids":["senseval2.d002.s040.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"scattered%5:00:00:distributed:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":634,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0013880000000000001,"input_uncached_usd":0.001268,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2125227079959586}
{"item_id":"senseval2.d002.s040.t005","gold_sense_keys":["poor%3:00:00::","poor%3:00:03::"],"candidates":[{"index":1,"sense_key":"poor%5:00:00:unfortunate:00","synset_id":"hapless.s.01"},{"index":2,"sense_key":"poor%3:00:00::","synset_id":"poor.a.02"},{"index":3,"sense_key":"poor%3:00:03::","synset_id":"poor.a.03"},{"index":4,"sense_key":"poor%3:00:02::","synset_id":"poor.a.04"},{"index":5,"sense_key":"poor%5:00:00:insufficient:00","synset_id":"inadequate.s.01"},{"index":6,"sense_key":"poor%5:00:00:bad:00","synset_id":"poor.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"poor%3:00:03::","call_ids":["senseval2.d002.s040.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"poor%3:00:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":866,"cached_input_tokens":0,"output_tokens":82,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002552,"input_uncached_usd":0.0017319999999999998,"input_cached_usd":0.0,"output_usd":0.0008200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5958186249481514}
{"item_id":"senseval2.d002.s040.t006","gold_sense_keys":["same%3:00:00::"],"candidates":[{"index":1,"sense_key":"same%3:00:02::","synset_id":"same.a.01"},{"index":2,"sense_key":"same%3:00:00::","synset_id":"same.a.02"},{"index":3,"sense_key":"same%3:00:04::","synset_id":"like.a.02"},{"index":4,"sense_key":"same%5:00:00:unchanged:00","synset_id":"same.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"same%3:00:00::","call_ids":["senseval2.d002.s040.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"same%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":693,"cached_input_tokens":0,"output_tokens":202,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003406,"input_uncached_usd":0.0013859999999999999,"input_cached_usd":0.0,"output_usd":0.00202,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.6501092079561204}
{"item_id":"senseval2.d002.s041.t001","gold_sense_keys":["majority%1:07:00::"],"candidates":[{"index":1,"sense_key":"majority%1:07:00::","synset_id":"majority.n.01"},{"index":2,"sense_key":"majority%1:23:00::","synset_id":"majority.n.02"},{"index":3,"sense_key":"majority%1:28:00::","synset_id":"majority.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"majority%1:07:00::","call_ids":["senseval2.d002.s041.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"majority%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":644,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001408,"input_uncached_usd":0.0012879999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.865257625002414}
{"item_id":"senseval2.d002.s041.t003","gold_sense_keys":["school%1:14:00::"],"candidates":[{"index":1,"sense_key":"school%1:14:00::","synset_id":"school.n.01"},{"index":2,"sense_key":"school%1:06:00::","synset_id":"school.n.02"},{"index":3,"sense_key":"school%1:09:00::","synset_id":"school.n.03"},{"index":4,"sense_key":"school%1:14:02::","synset_id":"school.n.04"},{"index":5,"sense_key":"school%1:28:00::","synset_id":"school.n.05"},{"index":6,"sense_key":"school%1:14:03::","synset_id":"school.n.06"},{"index":7,"sense_key":"school%1:14:01::","synset_id":"school.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"school%1:14:00::","call_ids":["senseval2.d002.s041.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"school%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":867,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002264,"input_uncached_usd":0.0017339999999999999,"input_cached_usd":0.0,"output_usd":0.0005300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.527979000005871}
{"item_id":"senseval2.d002.s041.t004","gold_sense_keys":["know%2:31:01::","know%2:31:02::","know%2:31:12::"],"candidates":[{"index":1,"sense_key":"know%2:31:01::","synset_id":"know.v.01"},{"index":2,"sense_key":"know%2:31:03::","synset_id":"know.v.02"},{"index":3,"sense_key":"know%2:31:02::","synset_id":"know.v.03"},{"index":4,"sense_key":"know%2:31:00::","synset_id":"know.v.04"},{"index":5,"sense_key":"know%2:31:04::","synset_id":"know.v.05"},{"index":6,"sense_key":"know%2:31:06::","synset_id":"acknowledge.v.06"},{"index":7,"sense_key":"know%2:31:15::","synset_id":"know.v.07"},{"index":8,"sense_key":"know%2:35:00::","synset_id":"sleep_together.v.01"},{"index":9,"sense_key":"know%2:31:11::","synset_id":"know.v.09"},{"index":10,"sense_key":"know%2:31:12::","synset_id":"know.v.10"},{"index":11,"sense_key":"know%2:31:14::","synset_id":"know.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"know%2:31:01::","call_ids":["senseval2.d002.s041.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"know%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1189,"cached_input_tokens":0,"output_tokens":120,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003578,"input_uncached_usd":0.002378,"input_cached_usd":0.0,"output_usd":0.0012000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1611095840344205}
{"item_id":"senseval2.d002.s041.t005","gold_sense_keys":["effort%1:04:00::"],"candidates":[{"index":1,"sense_key":"effort%1:04:00::","synset_id":"attempt.n.01"},{"index":2,"sense_key":"effort%1:04:01::","synset_id":"effort.n.02"},{"index":3,"sense_key":"effort%1:04:02::","synset_id":"feat.n.01"},{"index":4,"sense_key":"effort%1:04:03::","synset_id":"campaign.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"effort%1:04:03::","call_ids":["senseval2.d002.s041.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"effort%1:04:03::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":788,"cached_input_tokens":0,"output_tokens":169,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003266,"input_uncached_usd":0.001576,"input_cached_usd":0.0,"output_usd":0.00169,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9137651670025662}
{"item_id":"senseval2.d002.s041.t006","gold_sense_keys":["educational%3:01:00::"],"candidates":[{"index":1,"sense_key":"educational%3:01:00::","synset_id":"educational.a.01"},{"index":2,"sense_key":"educational%5:00:00:instructive:00","synset_id":"educational.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"educational%3:01:00::","call_ids":["senseval2.d002.s041.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"educational%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":548,"cached_input_tokens":0,"output_tokens":76,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001856,"input_uncached_usd":0.001096,"input_cached_usd":0.0,"output_usd":0.00076,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3438882089685649}
{"item_id":"senseval2.d002.s041.t007","gold_sense_keys":["reform%1:04:00::"],"candidates":[{"index":1,"sense_key":"reform%1:04:00::","synset_id":"reform.n.01"},{"index":2,"sense_key":"reform%1:04:01::","synset_id":"reform.n.02"},{"index":3,"sense_key":"reform%1:04:02::","synset_id":"reform.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"reform%1:04:01::","call_ids":["senseval2.d002.s041.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"reform%1:04:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":648,"cached_input_tokens":0,"output_tokens":88,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002176,"input_uncached_usd":0.0012959999999999998,"input_cached_usd":0.0,"output_usd":0.00088,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5136983749689534}
{"item_id":"senseval2.d002.s041.t008","gold_sense_keys":["doomed%5:00:00:unfortunate:00"],"candidates":[{"index":1,"sense_key":"doomed%5:00:00:dead:01","synset_id":"doomed.s.01"},{"index":2,"sense_key":"doomed%5:00:00:lost:02","synset_id":"cursed.s.01"},{"index":3,"sense_key":"doomed%5:00:00:unfortunate:00","synset_id":"doomed.s.03"},{"index":4,"sense_key":"doomed%5:00:00:certain:03","synset_id":"doomed.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"doomed%5:00:00:unfortunate:00","call_ids":["senseval2.d002.s041.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"doomed%5:00:00:unfortunate:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":791,"cached_input_tokens":0,"output_tokens":130,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002882,"input_uncached_usd":0.0015819999999999999,"input_cached_usd":0.0,"output_usd":0.0013000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0180420839460567}
{"item_id":"senseval2.d002.s042.t000","gold_sense_keys":["really%4:02:02::"],"candidates":[{"index":1,"sense_key":"really%4:02:03::","synset_id":"truly.r.01"},{"index":2,"sense_key":"really%4:02:04::","synset_id":"actually.r.01"},{"index":3,"sense_key":"really%4:02:02::","synset_id":"in_truth.r.01"},{"index":4,"sense_key":"really%4:02:00::","synset_id":"very.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"really%4:02:02::","call_ids":["senseval2.d002.s042.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"really%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":757,"cached_input_tokens":0,"output_tokens":158,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003094,"input_uncached_usd":0.001514,"input_cached_usd":0.0,"output_usd":0.00158,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6245061251102015}
{"item_id":"senseval2.d002.s042.t001","gold_sense_keys":["know%2:31:01::"],"candidates":[{"index":1,"sense_key":"know%2:31:01::","synset_id":"know.v.01"},{"index":2,"sense_key":"know%2:31:03::","synset_id":"know.v.02"},{"index":3,"sense_key":"know%2:31:02::","synset_id":"know.v.03"},{"index":4,"sense_key":"know%2:31:00::","synset_id":"know.v.04"},{"index":5,"sense_key":"know%2:31:04::","synset_id":"know.v.05"},{"index":6,"sense_key":"know%2:31:06::","synset_id":"acknowledge.v.06"},{"index":7,"sense_key":"know%2:31:15::","synset_id":"know.v.07"},{"index":8,"sense_key":"know%2:35:00::","synset_id":"sleep_together.v.01"},{"index":9,"sense_key":"know%2:31:11::","synset_id":"know.v.09"},{"index":10,"sense_key":"know%2:31:12::","synset_id":"know.v.10"},{"index":11,"sense_key":"know%2:31:14::","synset_id":"know.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"know%2:31:01::","call_ids":["senseval2.d002.s042.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"know%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1183,"cached_input_tokens":0,"output_tokens":158,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003946,"input_uncached_usd":0.002366,"input_cached_usd":0.0,"output_usd":0.00158,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.800035082967952}
{"item_id":"senseval2.d002.s042.t002","gold_sense_keys":["need%2:42:00::"],"candidates":[{"index":1,"sense_key":"need%2:42:00::","synset_id":"necessitate.v.01"},{"index":2,"sense_key":"need%2:34:00::","synset_id":"want.v.02"},{"index":3,"sense_key":"need%2:34:01::","synset_id":"need.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"need%2:34:01::","call_ids":["senseval2.d002.s042.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"need%2:34:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":631,"cached_input_tokens":0,"output_tokens":240,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0036620000000000003,"input_uncached_usd":0.0012619999999999999,"input_cached_usd":0.0,"output_usd":0.0024000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.122735625016503}
{"item_id":"senseval2.d002.s042.t003","gold_sense_keys":["know%2:31:01::"],"candidates":[{"index":1,"sense_key":"know%2:31:01::","synset_id":"know.v.01"},{"index":2,"sense_key":"know%2:31:03::","synset_id":"know.v.02"},{"index":3,"sense_key":"know%2:31:02::","synset_id":"know.v.03"},{"index":4,"sense_key":"know%2:31:00::","synset_id":"know.v.04"},{"index":5,"sense_key":"know%2:31:04::","synset_id":"know.v.05"},{"index":6,"sense_key":"know%2:31:06::","synset_id":"acknowledge.v.06"},{"index":7,"sense_key":"know%2:31:15::","synset_id":"know.v.07"},{"index":8,"sense_key":"know%2:35:00::","synset_id":"sleep_together.v.01"},{"index":9,"sense_key":"know%2:31:11::","synset_id":"know.v.09"},{"index":10,"sense_key":"know%2:31:12::","synset_id":"know.v.10"},{"index":11,"sense_key":"know%2:31:14::","synset_id":"know.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"know%2:31:01::","call_ids":["senseval2.d002.s042.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"know%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1183,"cached_input_tokens":0,"output_tokens":103,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003396,"input_uncached_usd":0.002366,"input_cached_usd":0.0,"output_usd":0.00103,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.007346459082328}
{"item_id":"senseval2.d002.s042.t005","gold_sense_keys":["assimilate%2:31:00::"],"candidates":[{"index":1,"sense_key":"assimilate%2:31:00::","synset_id":"absorb.v.02"},{"index":2,"sense_key":"assimilate%2:30:00::","synset_id":"assimilate.v.02"},{"index":3,"sense_key":"assimilate%2:30:01::","synset_id":"assimilate.v.03"},{"index":4,"sense_key":"assimilate%2:35:00::","synset_id":"assimilate.v.04"},{"index":5,"sense_key":"assimilate%2:30:02::","synset_id":"assimilate.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"assimilate%2:31:00::","call_ids":["senseval2.d002.s042.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"assimilate%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":728,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001576,"input_uncached_usd":0.0014559999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0668246670393273}
{"item_id":"senseval2.d002.s043.t000","gold_sense_keys":["respect%1:09:00::"],"candidates":[{"index":1,"sense_key":"respect%1:09:00::","synset_id":"respect.n.01"},{"index":2,"sense_key":"respect%1:26:00::","synset_id":"esteem.n.01"},{"index":3,"sense_key":"respect%1:09:01::","synset_id":"respect.n.03"},{"index":4,"sense_key":"respect%1:04:03::","synset_id":"deference.n.01"},{"index":5,"sense_key":"respect%1:04:02::","synset_id":"obedience.n.03"},{"index":6,"sense_key":"respect%1:12:00::","synset_id":"regard.n.06"},{"index":7,"sense_key":"respect%1:07:00::","synset_id":"deference.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"respect%1:09:00::","call_ids":["senseval2.d002.s043.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"respect%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":941,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002002,"input_uncached_usd":0.001882,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4645757919643074}
{"item_id":"senseval2.d002.s043.t005","gold_sense_keys":["rather%4:02:02::"],"candidates":[{"index":1,"sense_key":"rather%4:02:02::","synset_id":"rather.r.01"},{"index":2,"sense_key":"rather%4:02:04::","synset_id":"rather.r.02"},{"index":3,"sense_key":"rather%4:02:01::","synset_id":"preferably.r.01"},{"index":4,"sense_key":"rather%4:02:00::","synset_id":"quite.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"rather%4:02:02::","call_ids":["senseval2.d002.s043.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"rather%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":741,"cached_input_tokens":0,"output_tokens":145,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002932,"input_uncached_usd":0.001482,"input_cached_usd":0.0,"output_usd":0.0014500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9061092080082744}
{"item_id":"senseval2.d002.s043.t007","gold_sense_keys":["concerned%3:00:00::","concerned%5:00:00:involved:00"],"candidates":[{"index":1,"sense_key":"concerned%3:00:00::","synset_id":"concerned.a.01"},{"index":2,"sense_key":"concerned%5:00:00:involved:00","synset_id":"concerned.s.01"},{"index":3,"sense_key":"concerned%5:00:01:involved:00","synset_id":"implicated.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"concerned%3:00:00::","call_ids":["senseval2.d002.s043.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"concerned%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":678,"cached_input_tokens":0,"output_tokens":70,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002056,"input_uncached_usd":0.001356,"input_cached_usd":0.0,"output_usd":0.0007000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.571626125019975}
{"item_id":"senseval2.d002.s044.t001","gold_sense_keys":["inclined%3:00:02::"],"candidates":[{"index":1,"sense_key":"inclined%3:00:02::","synset_id":"inclined.a.01"},{"index":2,"sense_key":"inclined%3:00:01::","synset_id":"inclined.a.02"},{"index":3,"sense_key":"inclined%5:00:00:willing:00","synset_id":"disposed.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"inclined%3:00:02::","call_ids":["senseval2.d002.s044.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"inclined%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":680,"cached_input_tokens":0,"output_tokens":76,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00212,"input_uncached_usd":0.0013599999999999999,"input_cached_usd":0.0,"output_usd":0.00076,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3252555419458076}
{"item_id":"senseval2.d002.s044.t002","gold_sense_keys":["echo%2:32:00::"],"candidates":[{"index":1,"sense_key":"echo%2:32:00::","synset_id":"repeat.v.04"},{"index":2,"sense_key":"echo%2:39:00::","synset_id":"resound.v.01"},{"index":3,"sense_key":"echo%2:42:00::","synset_id":"echo.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"echo%2:32:00::","call_ids":["senseval2.d002.s044.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"echo%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":667,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001454,"input_uncached_usd":0.001334,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1423536670627072}
{"item_id":"senseval2.d002.s044.t003","gold_sense_keys":["conventional%3:00:00::","conventional%3:00:01::","conventional%5:00:00:orthodox:00"],"candidates":[{"index":1,"sense_key":"conventional%3:00:00::","synset_id":"conventional.a.01"},{"index":2,"sense_key":"conventional%5:00:00:orthodox:00","synset_id":"conventional.s.01"},{"index":3,"sense_key":"conventional%3:00:02::","synset_id":"conventional.a.03"},{"index":4,"sense_key":"conventional%3:00:01::","synset_id":"conventional.a.04"},{"index":5,"sense_key":"conventional%5:00:00:nonrepresentational:00","synset_id":"conventional.s.02"},{"index":6,"sense_key":"conventional%5:00:00:traditional:00","synset_id":"conventional.s.03"},{"index":7,"sense_key":"conventional%5:00:00:formal:01","synset_id":"ceremonious.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"conventional%5:00:00:orthodox:00","call_ids":["senseval2.d002.s044.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"conventional%5:00:00:orthodox:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":945,"cached_input_tokens":0,"output_tokens":194,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00383,"input_uncached_usd":0.00189,"input_cached_usd":0.0,"output_usd":0.00194,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1593936249846593}
{"item_id":"senseval2.d002.s044.t004","gold_sense_keys":["pap%1:10:00::"],"candidates":[{"index":1,"sense_key":"pap%1:10:00::","synset_id":"pap.n.01"},{"index":2,"sense_key":"pap%1:13:00::","synset_id":"soft_diet.n.01"},{"index":3,"sense_key":"pap%1:08:00::","synset_id":"nipple.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"pap%1:10:00::","call_ids":["senseval2.d002.s044.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"pap%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":662,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001444,"input_uncached_usd":0.0013239999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0278028750326484}
{"item_id":"senseval2.d002.s045.t000","gold_sense_keys":["thus%4:02:00::"],"candidates":[{"index":1,"sense_key":"thus%4:02:00::","synset_id":"therefore.r.01"},{"index":2,"sense_key":"thus%4:02:01::","synset_id":"thus.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"thus%4:02:01::","call_ids":["senseval2.d002.s045.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"thus%4:02:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":668,"cached_input_tokens":0,"output_tokens":266,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0039959999999999996,"input_uncached_usd":0.001336,"input_cached_usd":0.0,"output_usd":0.00266,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.381079790997319}
{"item_id":"senseval2.d002.s045.t001","gold_sense_keys":["recent%5:00:00:past:00"],"candidates":[{"index":1,"sense_key":"recent%5:00:00:new:00","synset_id":"recent.s.01"},{"index":2,"sense_key":"recent%5:00:00:past:00","synset_id":"late.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"recent%5:00:00:past:00","call_ids":["senseval2.d002.s045.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"recent%5:00:00:past:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":600,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0019199999999999998,"input_uncached_usd":0.0012,"input_cached_usd":0.0,"output_usd":0.00072,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6064693339867517}
{"item_id":"senseval2.d002.s045.t002","gold_sense_keys":["governor%1:18:00::"],"candidates":[{"index":1,"sense_key":"governor%1:18:00::","synset_id":"governor.n.01"},{"index":2,"sense_key":"governor%1:06:00::","synset_id":"governor.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"governor%1:18:00::","call_ids":["senseval2.d002.s045.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"governor%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":584,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001288,"input_uncached_usd":0.001168,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0339343330124393}
{"item_id":"senseval2.d002.s045.t003","gold_sense_keys":["conference%1:10:00::","conference%1:14:00::"],"candidates":[{"index":1,"sense_key":"conference%1:14:00::","synset_id":"conference.n.01"},{"index":2,"sense_key":"conference%1:14:01::","synset_id":"league.n.01"},{"index":3,"sense_key":"conference%1:10:00::","synset_id":"conference.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"conference%1:14:00::","call_ids":["senseval2.d002.s045.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"conference%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":648,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001416,"input_uncached_usd":0.0012959999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.227353124995716}
{"item_id":"senseval2.d002.s045.t004","gold_sense_keys":["education%1:04:00::"],"candidates":[{"index":1,"sense_key":"education%1:04:00::","synset_id":"education.n.01"},{"index":2,"sense_key":"education%1:09:00::","synset_id":"education.n.02"},{"index":3,"sense_key":"education%1:09:01::","synset_id":"education.n.03"},{"index":4,"sense_key":"education%1:04:01::","synset_id":"education.n.04"},{"index":5,"sense_key":"education%1:07:01::","synset_id":"education.n.05"},{"index":6,"sense_key":"education%1:14:00::","synset_id":"department_of_education.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"education%1:04:00::","call_ids":["senseval2.d002.s045.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"education%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":885,"cached_input_tokens":0,"output_tokens":110,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00287,"input_uncached_usd":0.0017699999999999999,"input_cached_usd":0.0,"output_usd":0.0011,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1945412079803646}
{"item_id":"senseval2.d002.s045.t005","gold_sense_keys":["announce%2:32:02::"],"candidates":[{"index":1,"sense_key":"announce%2:32:00::","synset_id":"announce.v.01"},{"index":2,"sense_key":"announce%2:32:02::","synset_id":"announce.v.02"},{"index":3,"sense_key":"announce%2:32:07::","synset_id":"announce.v.03"},{"index":4,"sense_key":"announce%2:32:01::","synset_id":"announce.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"announce%2:32:02::","call_ids":["senseval2.d002.s045.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"announce%2:32:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":691,"cached_input_tokens":0,"output_tokens":80,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002182,"input_uncached_usd":0.001382,"input_cached_usd":0.0,"output_usd":0.0008,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5283586250152439}
{"item_id":"senseval2.d002.s045.t006","gold_sense_keys":["country%1:14:00::","country%1:14:01::"],"candidates":[{"index":1,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":2,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":3,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":4,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":5,"sense_key":"country%1:15:02::","synset_id":"area.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"country%1:14:00::","call_ids":["senseval2.d002.s045.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"country%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":846,"cached_input_tokens":0,"output_tokens":126,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0029519999999999998,"input_uncached_usd":0.001692,"input_cached_usd":0.0,"output_usd":0.00126,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6974684589076787}
{"item_id":"senseval2.d002.s045.t007","gold_sense_keys":["need%2:34:00::"],"candidates":[{"index":1,"sense_key":"need%2:42:00::","synset_id":"necessitate.v.01"},{"index":2,"sense_key":"need%2:34:00::","synset_id":"want.v.02"},{"index":3,"sense_key":"need%2:34:01::","synset_id":"need.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"need%2:34:00::","call_ids":["senseval2.d002.s045.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"need%2:34:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":665,"cached_input_tokens":0,"output_tokens":196,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0032900000000000004,"input_uncached_usd":0.00133,"input_cached_usd":0.0,"output_usd":0.0019600000000000004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.8322954170871526}
{"item_id":"senseval2.d002.s045.t009","gold_sense_keys":["policy%1:09:00::"],"candidates":[{"index":1,"sense_key":"policy%1:09:00::","synset_id":"policy.n.01"},{"index":2,"sense_key":"policy%1:10:00::","synset_id":"policy.n.02"},{"index":3,"sense_key":"policy%1:10:02::","synset_id":"policy.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"policy%1:09:00::","call_ids":["senseval2.d002.s045.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"policy%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":683,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0014860000000000001,"input_uncached_usd":0.001366,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9600402500946075}
{"item_id":"senseval2.d002.s045.t010","gold_sense_keys":["child%1:18:00::"],"candidates":[{"index":1,"sense_key":"child%1:18:00::","synset_id":"child.n.01"},{"index":2,"sense_key":"child%1:18:01::","synset_id":"child.n.02"},{"index":3,"sense_key":"child%1:18:06::","synset_id":"child.n.03"},{"index":4,"sense_key":"child%1:18:07::","synset_id":"child.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"child%1:18:00::","call_ids":["senseval2.d002.s045.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"child%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":732,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0015840000000000001,"input_uncached_usd":0.001464,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.537867083097808}
{"item_id":"senseval2.d002.s046.t001","gold_sense_keys":["development%1:04:01::","development%1:22:02::"],"candidates":[{"index":1,"sense_key":"development%1:04:01::","synset_id":"development.n.01"},{"index":2,"sense_key":"development%1:22:02::","synset_id":"development.n.02"},{"index":3,"sense_key":"development%1:22:01::","synset_id":"growth.n.01"},{"index":4,"sense_key":"development%1:11:01::","synset_id":"development.n.04"},{"index":5,"sense_key":"development%1:04:02::","synset_id":"exploitation.n.01"},{"index":6,"sense_key":"development%1:15:00::","synset_id":"development.n.06"},{"index":7,"sense_key":"development%1:26:00::","synset_id":"development.n.07"},{"index":8,"sense_key":"development%1:22:00::","synset_id":"development.n.08"},{"index":9,"sense_key":"development%1:10:00::","synset_id":"development.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"development%1:22:01::","call_ids":["senseval2.d002.s046.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"development%1:22:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1155,"cached_input_tokens":0,"output_tokens":97,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00328,"input_uncached_usd":0.00231,"input_cached_usd":0.0,"output_usd":0.00097,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4195174999767914}
{"item_id":"senseval2.d002.s046.t002","gold_sense_keys":["policy%1:09:00::"],"candidates":[{"index":1,"sense_key":"policy%1:09:00::","synset_id":"policy.n.01"},{"index":2,"sense_key":"policy%1:10:00::","synset_id":"policy.n.02"},{"index":3,"sense_key":"policy%1:10:02::","synset_id":"policy.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"policy%1:09:00::","call_ids":["senseval2.d002.s046.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"policy%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":679,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001478,"input_uncached_usd":0.0013579999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5204908340238035}
{"item_id":"senseval2.d002.s046.t003","gold_sense_keys":["governor%1:18:00::"],"candidates":[{"index":1,"sense_key":"governor%1:18:00::","synset_id":"governor.n.01"},{"index":2,"sense_key":"governor%1:06:00::","synset_id":"governor.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"governor%1:18:00::","call_ids":["senseval2.d002.s046.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"governor%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":580,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00128,"input_uncached_usd":0.00116,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4936510419938713}
{"item_id":"senseval2.d002.s046.t004","gold_sense_keys":["seem%2:39:00::"],"candidates":[{"index":1,"sense_key":"seem%2:39:00::","synset_id":"look.v.02"},{"index":2,"sense_key":"seem%2:39:01::","synset_id":"appear.v.04"},{"index":3,"sense_key":"seem%2:42:00::","synset_id":"seem.v.03"},{"index":4,"sense_key":"seem%2:39:02::","synset_id":"seem.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"seem%2:39:00::","call_ids":["senseval2.d002.s046.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"seem%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":751,"cached_input_tokens":0,"output_tokens":145,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0029519999999999998,"input_uncached_usd":0.0015019999999999999,"input_cached_usd":0.0,"output_usd":0.0014500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.0151267499895766}
{"item_id":"senseval2.d002.s046.t005","gold_sense_keys":["more%4:02:00::"],"candidates":[{"index":1,"sense_key":"more%4:02:00::","synset_id":"more.r.01"},{"index":2,"sense_key":"more%4:02:01::","synset_id":"more.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"more%4:02:00::","call_ids":["senseval2.d002.s046.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"more%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":607,"cached_input_tokens":0,"output_tokens":76,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001974,"input_uncached_usd":0.001214,"input_cached_usd":0.0,"output_usd":0.00076,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4394633329939097}
{"item_id":"senseval2.d002.s046.t007","gold_sense_keys":["need%1:17:00::","need%1:26:00::"],"candidates":[{"index":1,"sense_key":"need%1:26:00::","synset_id":"need.n.01"},{"index":2,"sense_key":"need%1:17:00::","synset_id":"need.n.02"},{"index":3,"sense_key":"need%1:03:00::","synset_id":"motivation.n.01"},{"index":4,"sense_key":"need%1:26:01::","synset_id":"indigence.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"need%1:17:00::","call_ids":["senseval2.d002.s046.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"need%1:17:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":804,"cached_input_tokens":0,"output_tokens":133,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0029379999999999996,"input_uncached_usd":0.0016079999999999998,"input_cached_usd":0.0,"output_usd":0.00133,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2139389590593055}
{"item_id":"senseval2.d002.s047.t002","gold_sense_keys":["extend%2:30:01::"],"candidates":[{"index":1,"sense_key":"extend%2:30:01::","synset_id":"widen.v.04"},{"index":2,"sense_key":"extend%2:42:03::","synset_id":"run.v.03"},{"index":3,"sense_key":"extend%2:42:01::","synset_id":"cover.v.03"},{"index":4,"sense_key":"extend%2:40:05::","synset_id":"extend.v.04"},{"index":5,"sense_key":"extend%2:29:01::","synset_id":"exsert.v.01"},{"index":6,"sense_key":"extend%2:42:02::","synset_id":"extend.v.06"},{"index":7,"sense_key":"extend%2:40:04::","synset_id":"offer.v.05"},{"index":8,"sense_key":"extend%2:29:00::","synset_id":"stretch.v.02"},{"index":9,"sense_key":"extend%2:30:09::","synset_id":"extend.v.09"},{"index":10,"sense_key":"extend%2:30:02::","synset_id":"prolong.v.01"},{"index":11,"sense_key":"extend%2:38:10::","synset_id":"unfold.v.03"},{"index":12,"sense_key":"extend%2:38:14::","synset_id":"gallop.v.03"},{"index":13,"sense_key":"extend%2:35:10::","synset_id":"extend.v.13"},{"index":14,"sense_key":"extend%2:34:00::","synset_id":"strain.v.03"},{"index":15,"sense_key":"extend%2:30:15::","synset_id":"extend.v.15"},{"index":16,"sense_key":"extend%2:30:06::","synset_id":"carry.v.09"},{"index":17,"sense_key":"extend%2:30:12::","synset_id":"extend.v.17"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"extend%2:30:01::","call_ids":["senseval2.d002.s047.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"extend%2:30:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1613,"cached_input_tokens":0,"output_tokens":133,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004555999999999999,"input_uncached_usd":0.0032259999999999997,"input_cached_usd":0.0,"output_usd":0.00133,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.0597292080055922}
{"item_id":"senseval2.d002.s047.t003","gold_sense_keys":["educational%3:01:00::"],"candidates":[{"index":1,"sense_key":"educational%3:01:00::","synset_id":"educational.a.01"},{"index":2,"sense_key":"educational%5:00:00:instructive:00","synset_id":"educational.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"educational%3:01:00::","call_ids":["senseval2.d002.s047.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"educational%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":553,"cached_input_tokens":0,"output_tokens":79,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0018960000000000001,"input_uncached_usd":0.001106,"input_cached_usd":0.0,"output_usd":0.00079,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3448385000228882}
{"item_id":"senseval2.d002.s047.t004","gold_sense_keys":["system%1:14:00::"],"candidates":[{"index":1,"sense_key":"system%1:06:00::","synset_id":"system.n.01"},{"index":2,"sense_key":"system%1:14:00::","synset_id":"system.n.02"},{"index":3,"sense_key":"system%1:27:00::","synset_id":"system.n.03"},{"index":4,"sense_key":"system%1:09:00::","synset_id":"system.n.04"},{"index":5,"sense_key":"system%1:09:02::","synset_id":"arrangement.n.03"},{"index":6,"sense_key":"system%1:08:00::","synset_id":"system.n.06"},{"index":7,"sense_key":"system%1:09:01::","synset_id":"system.n.07"},{"index":8,"sense_key":"system%1:08:01::","synset_id":"system.n.08"},{"index":9,"sense_key":"system%1:07:00::","synset_id":"organization.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"system%1:14:00::","call_ids":["senseval2.d002.s047.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"system%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1137,"cached_input_tokens":0,"output_tokens":142,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0036940000000000002,"input_uncached_usd":0.002274,"input_cached_usd":0.0,"output_usd":0.00142,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8623482920229435}
{"item_id":"senseval2.d002.s048.t000","gold_sense_keys":["desirable%3:00:00::"],"candidates":[{"index":1,"sense_key":"desirable%3:00:00::","synset_id":"desirable.a.01"},{"index":2,"sense_key":"desirable%5:00:00:eligible:00","synset_id":"desirable.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"desirable%3:00:00::","call_ids":["senseval2.d002.s048.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"desirable%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":609,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001338,"input_uncached_usd":0.001218,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3142647499917075}
{"item_id":"senseval2.d002.s048.t002","gold_sense_keys":["program%1:09:01::"],"candidates":[{"index":1,"sense_key":"program%1:09:00::","synset_id":"plan.n.01"},{"index":2,"sense_key":"program%1:09:01::","synset_id":"program.n.02"},{"index":3,"sense_key":"program%1:10:01::","synset_id":"broadcast.n.02"},{"index":4,"sense_key":"program%1:10:04::","synset_id":"platform.n.02"},{"index":5,"sense_key":"program%1:10:00::","synset_id":"program.n.05"},{"index":6,"sense_key":"program%1:10:05::","synset_id":"course_of_study.n.01"},{"index":7,"sense_key":"program%1:10:02::","synset_id":"program.n.07"},{"index":8,"sense_key":"program%1:04:00::","synset_id":"program.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"program%1:09:01::","call_ids":["senseval2.d002.s048.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"program%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1037,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0026939999999999998,"input_uncached_usd":0.002074,"input_cached_usd":0.0,"output_usd":0.00062,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.5295673749642447}
{"item_id":"senseval2.d002.s048.t003","gold_sense_keys":["educational%3:01:00::"],"candidates":[{"index":1,"sense_key":"educational%3:01:00::","synset_id":"educational.a.01"},{"index":2,"sense_key":"educational%5:00:00:instructive:00","synset_id":"educational.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"educational%3:01:00::","call_ids":["senseval2.d002.s048.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"educational%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":561,"cached_input_tokens":0,"output_tokens":108,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002202,"input_uncached_usd":0.001122,"input_cached_usd":0.0,"output_usd":0.00108,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6717602090211585}
{"item_id":"senseval2.d002.s048.t004","gold_sense_keys":["program%1:09:01::"],"candidates":[{"index":1,"sense_key":"program%1:09:00::","synset_id":"plan.n.01"},{"index":2,"sense_key":"program%1:09:01::","synset_id":"program.n.02"},{"index":3,"sense_key":"program%1:10:01::","synset_id":"broadcast.n.02"},{"index":4,"sense_key":"program%1:10:04::","synset_id":"platform.n.02"},{"index":5,"sense_key":"program%1:10:00::","synset_id":"program.n.05"},{"index":6,"sense_key":"program%1:10:05::","synset_id":"course_of_study.n.01"},{"index":7,"sense_key":"program%1:10:02::","synset_id":"program.n.07"},{"index":8,"sense_key":"program%1:04:00::","synset_id":"program.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"program%1:09:01::","call_ids":["senseval2.d002.s048.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"program%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1037,"cached_input_tokens":0,"output_tokens":125,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0033239999999999997,"input_uncached_usd":0.002074,"input_cached_usd":0.0,"output_usd":0.00125,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.084871875005774}
{"item_id":"senseval2.d002.s049.t000","gold_sense_keys":["know%2:31:01::","know%2:31:02::"],"candidates":[{"index":1,"sense_key":"know%2:31:01::","synset_id":"know.v.01"},{"index":2,"sense_key":"know%2:31:03::","synset_id":"know.v.02"},{"index":3,"sense_key":"know%2:31:02::","synset_id":"know.v.03"},{"index":4,"sense_key":"know%2:31:00::","synset_id":"know.v.04"},{"index":5,"sense_key":"know%2:31:04::","synset_id":"know.v.05"},{"index":6,"sense_key":"know%2:31:06::","synset_id":"acknowledge.v.06"},{"index":7,"sense_key":"know%2:31:15::","synset_id":"know.v.07"},{"index":8,"sense_key":"know%2:35:00::","synset_id":"sleep_together.v.01"},{"index":9,"sense_key":"know%2:31:11::","synset_id":"know.v.09"},{"index":10,"sense_key":"know%2:31:12::","synset_id":"know.v.10"},{"index":11,"sense_key":"know%2:31:14::","synset_id":"know.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"know%2:31:01::","call_ids":["senseval2.d002.s049.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"know%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1212,"cached_input_tokens":0,"output_tokens":221,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004634,"input_uncached_usd":0.002424,"input_cached_usd":0.0,"output_usd":0.00221,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.559232832980342}
{"item_id":"senseval2.d002.s049.t001","gold_sense_keys":["very%4:02:00::"],"candidates":[{"index":1,"sense_key":"very%4:02:00::","synset_id":"very.r.01"},{"index":2,"sense_key":"very%4:02:01::","synset_id":"very.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"very%4:02:00::","call_ids":["senseval2.d002.s049.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"very%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":614,"cached_input_tokens":0,"output_tokens":82,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002048,"input_uncached_usd":0.001228,"input_cached_usd":0.0,"output_usd":0.0008200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5623957089846954}
{"item_id":"senseval2.d002.s049.t002","gold_sense_keys":["early%3:00:00::","early%3:00:02::"],"candidates":[{"index":1,"sense_key":"early%3:00:00::","synset_id":"early.a.01"},{"index":2,"sense_key":"early%3:00:02::","synset_id":"early.a.02"},{"index":3,"sense_key":"early%5:00:00:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"early%5:00:00:young:00","synset_id":"early.s.02"},{"index":5,"sense_key":"early%3:00:01::","synset_id":"early.a.05"},{"index":6,"sense_key":"early%5:00:01:future:00","synset_id":"early.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"early%5:00:00:young:00","call_ids":["senseval2.d002.s049.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"early%5:00:00:young:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":835,"cached_input_tokens":0,"output_tokens":139,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0030600000000000002,"input_uncached_usd":0.0016699999999999998,"input_cached_usd":0.0,"output_usd":0.0013900000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8804964170558378}
{"item_id":"senseval2.d002.s049.t003","gold_sense_keys":["exposure%1:04:06::"],"candidates":[{"index":1,"sense_key":"exposure%1:07:00::","synset_id":"exposure.n.01"},{"index":2,"sense_key":"exposure%1:04:06::","synset_id":"exposure.n.02"},{"index":3,"sense_key":"exposure%1:10:00::","synset_id":"exposure.n.03"},{"index":4,"sense_key":"exposure%1:09:00::","synset_id":"exposure.n.04"},{"index":5,"sense_key":"exposure%1:26:01::","synset_id":"vulnerability.n.01"},{"index":6,"sense_key":"exposure%1:23:00::","synset_id":"exposure.n.06"},{"index":7,"sense_key":"exposure%1:06:00::","synset_id":"photograph.n.01"},{"index":8,"sense_key":"exposure%1:04:02::","synset_id":"exposure.n.08"},{"index":9,"sense_key":"exposure%1:04:05::","synset_id":"exposure.n.09"},{"index":10,"sense_key":"exposure%1:04:01::","synset_id":"exposure.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"exposure%1:04:06::","call_ids":["senseval2.d002.s049.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"exposure%1:04:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1055,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0022299999999999998,"input_uncached_usd":0.00211,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1049580830149353}
{"item_id":"senseval2.d002.s049.t004","gold_sense_keys":["schooling%1:09:00::"],"candidates":[{"index":1,"sense_key":"schooling%1:04:00::","synset_id":"schooling.n.01"},{"index":2,"sense_key":"schooling%1:09:00::","synset_id":"school.n.03"},{"index":3,"sense_key":"schooling%1:04:01::","synset_id":"schooling.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"schooling%1:09:00::","call_ids":["senseval2.d002.s049.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"schooling%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":623,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001916,"input_uncached_usd":0.001246,"input_cached_usd":0.0,"output_usd":0.00067,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4325249589746818}
{"item_id":"senseval2.d002.s049.t005","gold_sense_keys":["improve%2:30:01::"],"candidates":[{"index":1,"sense_key":"improve%2:30:01::","synset_id":"better.v.02"},{"index":2,"sense_key":"improve%2:30:00::","synset_id":"better.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"improve%2:30:01::","call_ids":["senseval2.d002.s049.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"improve%2:30:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":611,"cached_input_tokens":0,"output_tokens":81,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002032,"input_uncached_usd":0.001222,"input_cached_usd":0.0,"output_usd":0.0008100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.846888208994642}
{"item_id":"senseval2.d002.s049.t006","gold_sense_keys":["performance%1:04:00::","performance%1:04:03::"],"candidates":[{"index":1,"sense_key":"performance%1:10:00::","synset_id":"performance.n.01"},{"index":2,"sense_key":"performance%1:04:01::","synset_id":"performance.n.02"},{"index":3,"sense_key":"performance%1:04:00::","synset_id":"performance.n.03"},{"index":4,"sense_key":"performance%1:04:03::","synset_id":"performance.n.04"},{"index":5,"sense_key":"performance%1:22:00::","synset_id":"operation.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"performance%1:04:00::","call_ids":["senseval2.d002.s049.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"performance%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":877,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002524,"input_uncached_usd":0.001754,"input_cached_usd":0.0,"output_usd":0.0007700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5195600830484182}
{"item_id":"senseval2.d002.s049.t007","gold_sense_keys":["first%5:00:00:ordinal:00"],"candidates":[{"index":1,"sense_key":"first%3:00:00::","synset_id":"first.a.01"},{"index":2,"sense_key":"first%5:00:00:ordinal:00","synset_id":"first.s.01"},{"index":3,"sense_key":"first%5:00:02:opening:00","synset_id":"inaugural.s.01"},{"index":4,"sense_key":"first%5:00:01:opening:00","synset_id":"beginning.s.01"},{"index":5,"sense_key":"first%5:00:00:best:00","synset_id":"first.s.04"},{"index":6,"sense_key":"first%3:00:03::","synset_id":"first.a.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"first%5:00:00:ordinal:00","call_ids":["senseval2.d002.s049.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"first%5:00:00:ordinal:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":853,"cached_input_tokens":0,"output_tokens":74,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002446,"input_uncached_usd":0.0017059999999999998,"input_cached_usd":0.0,"output_usd":0.0007400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3400565420743078}
{"item_id":"senseval2.d002.s049.t008","gold_sense_keys":["grade%1:14:00::"],"candidates":[{"index":1,"sense_key":"grade%1:14:00::","synset_id":"class.n.02"},{"index":2,"sense_key":"grade%1:26:00::","synset_id":"grade.n.02"},{"index":3,"sense_key":"grade%1:07:01::","synset_id":"grade.n.03"},{"index":4,"sense_key":"grade%1:23:00::","synset_id":"grad.n.01"},{"index":5,"sense_key":"grade%1:10:00::","synset_id":"grade.n.05"},{"index":6,"sense_key":"grade%1:09:00::","synset_id":"mark.n.01"},{"index":7,"sense_key":"grade%1:07:02::","synset_id":"grade.n.07"},{"index":8,"sense_key":"grade%1:07:00::","synset_id":"degree.n.01"},{"index":9,"sense_key":"grade%1:05:00::","synset_id":"grade.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"grade%1:14:00::","call_ids":["senseval2.d002.s049.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"grade%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":978,"cached_input_tokens":0,"output_tokens":181,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0037660000000000003,"input_uncached_usd":0.001956,"input_cached_usd":0.0,"output_usd":0.0018100000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.3842779169790447}
{"item_id":"senseval2.d002.s049.t010","gold_sense_keys":["difference%1:07:00::"],"candidates":[{"index":1,"sense_key":"difference%1:07:00::","synset_id":"difference.n.01"},{"index":2,"sense_key":"difference%1:11:00::","synset_id":"deviation.n.01"},{"index":3,"sense_key":"difference%1:10:00::","synset_id":"dispute.n.01"},{"index":4,"sense_key":"difference%1:24:00::","synset_id":"difference.n.04"},{"index":5,"sense_key":"difference%1:23:00::","synset_id":"remainder.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"difference%1:07:00::","call_ids":["senseval2.d002.s049.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"difference%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":793,"cached_input_tokens":0,"output_tokens":134,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0029259999999999998,"input_uncached_usd":0.001586,"input_cached_usd":0.0,"output_usd":0.00134,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1931493330048397}
{"item_id":"senseval2.d002.s049.t011","gold_sense_keys":["quickly%4:02:02::"],"candidates":[{"index":1,"sense_key":"quickly%4:02:00::","synset_id":"quickly.r.01"},{"index":2,"sense_key":"quickly%4:02:02::","synset_id":"promptly.r.01"},{"index":3,"sense_key":"quickly%4:02:03::","synset_id":"cursorily.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"quickly%4:02:02::","call_ids":["senseval2.d002.s049.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"quickly%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":673,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0034260000000000002,"input_uncached_usd":0.001346,"input_cached_usd":0.0,"output_usd":0.0020800000000000003,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2734990830067545}
{"item_id":"senseval2.d002.s049.t013","gold_sense_keys":["wash_away%2:30:01::"],"candidates":[{"index":1,"sense_key":"wash_away%2:30:01::","synset_id":"wash_away.v.01"},{"index":2,"sense_key":"wash_away%2:30:00::","synset_id":"wash.v.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"wash_away%2:30:01::","call_ids":["senseval2.d002.s049.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"wash_away%2:30:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":624,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001928,"input_uncached_usd":0.001248,"input_cached_usd":0.0,"output_usd":0.00068,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.981465292046778}
{"item_id":"senseval2.d002.s050.t000","gold_sense_keys":["sum_up%2:32:00::"],"candidates":[{"index":1,"sense_key":"sum_up%2:32:00::","synset_id":"sum_up.v.01"},{"index":2,"sense_key":"sum_up%2:42:00::","synset_id":"summarize.v.02"},{"index":3,"sense_key":"sum_up%2:32:03::","synset_id":"total.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"sum_up%2:32:00::","call_ids":["senseval2.d002.s050.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"sum_up%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":689,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001498,"input_uncached_usd":0.0013779999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4211575839435682}
{"item_id":"senseval2.d002.s050.t001","gold_sense_keys":["know%2:31:01::"],"candidates":[{"index":1,"sense_key":"know%2:31:01::","synset_id":"know.v.01"},{"index":2,"sense_key":"know%2:31:03::","synset_id":"know.v.02"},{"index":3,"sense_key":"know%2:31:02::","synset_id":"know.v.03"},{"index":4,"sense_key":"know%2:31:00::","synset_id":"know.v.04"},{"index":5,"sense_key":"know%2:31:04::","synset_id":"know.v.05"},{"index":6,"sense_key":"know%2:31:06::","synset_id":"acknowledge.v.06"},{"index":7,"sense_key":"know%2:31:15::","synset_id":"know.v.07"},{"index":8,"sense_key":"know%2:35:00::","synset_id":"sleep_together.v.01"},{"index":9,"sense_key":"know%2:31:11::","synset_id":"know.v.09"},{"index":10,"sense_key":"know%2:31:12::","synset_id":"know.v.10"},{"index":11,"sense_key":"know%2:31:14::","synset_id":"know.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"know%2:31:01::","call_ids":["senseval2.d002.s050.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"know%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1201,"cached_input_tokens":0,"output_tokens":155,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003952000000000001,"input_uncached_usd":0.002402,"input_cached_usd":0.0,"output_usd":0.0015500000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1739140829304233}
{"item_id":"senseval2.d002.s050.t002","gold_sense_keys":["education%1:04:00::"],"candidates":[{"index":1,"sense_key":"education%1:04:00::","synset_id":"education.n.01"},{"index":2,"sense_key":"education%1:09:00::","synset_id":"education.n.02"},{"index":3,"sense_key":"education%1:09:01::","synset_id":"education.n.03"},{"index":4,"sense_key":"education%1:04:01::","synset_id":"education.n.04"},{"index":5,"sense_key":"education%1:07:01::","synset_id":"education.n.05"},{"index":6,"sense_key":"education%1:14:00::","synset_id":"department_of_education.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"education%1:04:00::","call_ids":["senseval2.d002.s050.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"education%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":870,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00236,"input_uncached_usd":0.00174,"input_cached_usd":0.0,"output_usd":0.00062,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8730795410228893}
{"item_id":"senseval2.d002.s050.t003","gold_sense_keys":["education%1:04:00::"],"candidates":[{"index":1,"sense_key":"education%1:04:00::","synset_id":"education.n.01"},{"index":2,"sense_key":"education%1:09:00::","synset_id":"education.n.02"},{"index":3,"sense_key":"education%1:09:01::","synset_id":"education.n.03"},{"index":4,"sense_key":"education%1:04:01::","synset_id":"education.n.04"},{"index":5,"sense_key":"education%1:07:01::","synset_id":"education.n.05"},{"index":6,"sense_key":"education%1:14:00::","synset_id":"department_of_education.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"education%1:04:00::","call_ids":["senseval2.d002.s050.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"education%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":870,"cached_input_tokens":0,"output_tokens":100,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0027400000000000002,"input_uncached_usd":0.00174,"input_cached_usd":0.0,"output_usd":0.001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.254094666102901}
{"item_id":"senseval2.d002.s050.t004","gold_sense_keys":["reform%1:04:00::"],"candidates":[{"index":1,"sense_key":"reform%1:04:00::","synset_id":"reform.n.01"},{"index":2,"sense_key":"reform%1:04:01::","synset_id":"reform.n.02"},{"index":3,"sense_key":"reform%1:04:02::","synset_id":"reform.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"reform%1:04:01::","call_ids":["senseval2.d002.s050.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"reform%1:04:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":661,"cached_input_tokens":0,"output_tokens":154,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002862,"input_uncached_usd":0.001322,"input_cached_usd":0.0,"output_usd":0.0015400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8096673330292106}
{"item_id":"senseval2.d002.s050.t005","gold_sense_keys":["work%2:41:03::"],"candidates":[{"index":1,"sense_key":"work%2:41:02::","synset_id":"work.v.01"},{"index":2,"sense_key":"work%2:41:00::","synset_id":"work.v.02"},{"index":3,"sense_key":"work%2:41:03::","synset_id":"work.v.03"},{"index":4,"sense_key":"work%2:35:02::","synset_id":"function.v.01"},{"index":5,"sense_key":"work%2:36:00::","synset_id":"work.v.05"},{"index":6,"sense_key":"work%2:29:00::","synset_id":"exercise.v.03"},{"index":7,"sense_key":"work%2:38:00::","synset_id":"make.v.36"},{"index":8,"sense_key":"work%2:41:04::","synset_id":"work.v.08"},{"index":9,"sense_key":"work%2:41:05::","synset_id":"work.v.09"},{"index":10,"sense_key":"work%2:38:02::","synset_id":"work.v.10"},{"index":11,"sense_key":"work%2:36:01::","synset_id":"bring.v.03"},{"index":12,"sense_key":"work%2:41:01::","synset_id":"work.v.12"},{"index":13,"sense_key":"work%2:36:09::","synset_id":"cultivate.v.02"},{"index":14,"sense_key":"work%2:42:10::","synset_id":"work.v.14"},{"index":15,"sense_key":"work%2:41:10::","synset_id":"influence.v.01"},{"index":16,"sense_key":"work%2:41:11::","synset_id":"work.v.16"},{"index":17,"sense_key":"work%2:41:12::","synset_id":"work.v.17"},{"index":18,"sense_key":"work%2:39:10::","synset_id":"work.v.18"},{"index":19,"sense_key":"work%2:37:10::","synset_id":"work.v.19"},{"index":20,"sense_key":"work%2:36:12::","synset_id":"shape.v.02"},{"index":21,"sense_key":"work%2:35:14::","synset_id":"work.v.21"},{"index":22,"sense_key":"work%2:35:00::","synset_id":"knead.v.01"},{"index":23,"sense_key":"work%2:34:10::","synset_id":"exploit.v.01"},{"index":24,"sense_key":"work%2:31:13::","synset_id":"solve.v.01"},{"index":25,"sense_key":"work%2:30:14::","synset_id":"ferment.v.03"},{"index":26,"sense_key":"work%2:30:13::","synset_id":"sour.v.01"},{"index":27,"sense_key":"work%2:30:10::","synset_id":"work.v.27"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"work%2:41:03::","call_ids":["senseval2.d002.s050.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"work%2:41:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2119,"cached_input_tokens":0,"output_tokens":135,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.005587999999999999,"input_uncached_usd":0.004238,"input_cached_usd":0.0,"output_usd":0.00135,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.3705455829622224}
{"item_id":"senseval2.d002.s050.t006","gold_sense_keys":["work%2:41:03::"],"candidates":[{"index":1,"sense_key":"work%2:41:02::","synset_id":"work.v.01"},{"index":2,"sense_key":"work%2:41:00::","synset_id":"work.v.02"},{"index":3,"sense_key":"work%2:41:03::","synset_id":"work.v.03"},{"index":4,"sense_key":"work%2:35:02::","synset_id":"function.v.01"},{"index":5,"sense_key":"work%2:36:00::","synset_id":"work.v.05"},{"index":6,"sense_key":"work%2:29:00::","synset_id":"exercise.v.03"},{"index":7,"sense_key":"work%2:38:00::","synset_id":"make.v.36"},{"index":8,"sense_key":"work%2:41:04::","synset_id":"work.v.08"},{"index":9,"sense_key":"work%2:41:05::","synset_id":"work.v.09"},{"index":10,"sense_key":"work%2:38:02::","synset_id":"work.v.10"},{"index":11,"sense_key":"work%2:36:01::","synset_id":"bring.v.03"},{"index":12,"sense_key":"work%2:41:01::","synset_id":"work.v.12"},{"index":13,"sense_key":"work%2:36:09::","synset_id":"cultivate.v.02"},{"index":14,"sense_key":"work%2:42:10::","synset_id":"work.v.14"},{"index":15,"sense_key":"work%2:41:10::","synset_id":"influence.v.01"},{"index":16,"sense_key":"work%2:41:11::","synset_id":"work.v.16"},{"index":17,"sense_key":"work%2:41:12::","synset_id":"work.v.17"},{"index":18,"sense_key":"work%2:39:10::","synset_id":"work.v.18"},{"index":19,"sense_key":"work%2:37:10::","synset_id":"work.v.19"},{"index":20,"sense_key":"work%2:36:12::","synset_id":"shape.v.02"},{"index":21,"sense_key":"work%2:35:14::","synset_id":"work.v.21"},{"index":22,"sense_key":"work%2:35:00::","synset_id":"knead.v.01"},{"index":23,"sense_key":"work%2:34:10::","synset_id":"exploit.v.01"},{"index":24,"sense_key":"work%2:31:13::","synset_id":"solve.v.01"},{"index":25,"sense_key":"work%2:30:14::","synset_id":"ferment.v.03"},{"index":26,"sense_key":"work%2:30:13::","synset_id":"sour.v.01"},{"index":27,"sense_key":"work%2:30:10::","synset_id":"work.v.27"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"work%2:41:03::","call_ids":["senseval2.d002.s050.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"work%2:41:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2119,"cached_input_tokens":0,"output_tokens":97,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.005208,"input_uncached_usd":0.004238,"input_cached_usd":0.0,"output_usd":0.00097,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.591078249970451}
{"item_id":"senseval2.d002.s050.t007","gold_sense_keys":["parental%3:00:00::"],"candidates":[{"index":1,"sense_key":"parental%3:01:00::","synset_id":"parental.a.01"},{"index":2,"sense_key":"parental%3:00:00::","synset_id":"parental.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"parental%3:00:00::","call_ids":["senseval2.d002.s050.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"parental%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":585,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0012900000000000001,"input_uncached_usd":0.00117,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0054080419940874}
{"item_id":"senseval2.d002.s050.t008","gold_sense_keys":["involvement%1:04:00::"],"candidates":[{"index":1,"sense_key":"involvement%1:04:00::","synset_id":"engagement.n.07"},{"index":2,"sense_key":"involvement%1:24:00::","synset_id":"involvement.n.02"},{"index":3,"sense_key":"involvement%1:09:00::","synset_id":"interest.n.01"},{"index":4,"sense_key":"involvement%1:26:00::","synset_id":"affair.n.02"},{"index":5,"sense_key":"involvement%1:26:01::","synset_id":"participation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"involvement%1:04:00::","call_ids":["senseval2.d002.s050.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"involvement%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":796,"cached_input_tokens":0,"output_tokens":126,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002852,"input_uncached_usd":0.0015919999999999999,"input_cached_usd":0.0,"output_usd":0.00126,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7071154169971123}
{"item_id":"senseval2.d002.s050.t009","gold_sense_keys":["bad%3:00:00::"],"candidates":[{"index":1,"sense_key":"bad%3:00:00::","synset_id":"bad.a.01"},{"index":2,"sense_key":"bad%5:00:00:intense:00","synset_id":"bad.s.01"},{"index":3,"sense_key":"bad%5:00:00:uncomfortable:00","synset_id":"bad.s.02"},{"index":4,"sense_key":"bad%5:00:00:stale:00","synset_id":"bad.s.03"},{"index":5,"sense_key":"bad%3:00:09::","synset_id":"regretful.a.01"},{"index":6,"sense_key":"bad%5:00:00:invalid:00","synset_id":"bad.s.04"},{"index":7,"sense_key":"bad%5:00:00:inferior:02","synset_id":"bad.s.05"},{"index":8,"sense_key":"bad%5:00:00:nonstandard:03","synset_id":"bad.s.06"},{"index":9,"sense_key":"bad%5:00:02:unsound:00","synset_id":"bad.s.07"},{"index":10,"sense_key":"bad%5:00:00:unhealthy:00","synset_id":"bad.s.08"},{"index":11,"sense_key":"bad%5:00:00:harmful:00","synset_id":"bad.s.09"},{"index":12,"sense_key":"bad%5:00:00:evil:00","synset_id":"bad.s.10"},{"index":13,"sense_key":"bad%5:00:00:counterfeit:00","synset_id":"bad.s.11"},{"index":14,"sense_key":"bad%5:00:00:malfunctioning:00","synset_id":"bad.s.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bad%3:00:00::","call_ids":["senseval2.d002.s050.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bad%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1368,"cached_input_tokens":0,"output_tokens":103,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003766,"input_uncached_usd":0.0027359999999999997,"input_cached_usd":0.0,"output_usd":0.00103,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9856829580385238}
{"item_id":"senseval2.d002.s050.t010","gold_sense_keys":["idea%1:09:00::"],"candidates":[{"index":1,"sense_key":"idea%1:09:00::","synset_id":"idea.n.01"},{"index":2,"sense_key":"idea%1:09:01::","synset_id":"mind.n.06"},{"index":3,"sense_key":"idea%1:09:03::","synset_id":"idea.n.03"},{"index":4,"sense_key":"idea%1:09:02::","synset_id":"estimate.n.01"},{"index":5,"sense_key":"idea%1:10:00::","synset_id":"theme.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"idea%1:09:00::","call_ids":["senseval2.d002.s050.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"idea%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":831,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002202,"input_uncached_usd":0.0016619999999999998,"input_cached_usd":0.0,"output_usd":0.00054,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3011878749821335}
{"item_id":"senseval2.d002.s051.t000","gold_sense_keys":["parent%1:18:00::"],"candidates":[{"index":1,"sense_key":"parent%1:18:00::","synset_id":"parent.n.01"},{"index":2,"sense_key":"parent%1:03:00::","synset_id":"parent.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"parent%1:18:00::","call_ids":["senseval2.d002.s051.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"parent%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":549,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0012180000000000001,"input_uncached_usd":0.001098,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1196930420119315}
{"item_id":"senseval2.d002.s051.t001","gold_sense_keys":["too%4:02:00::"],"candidates":[{"index":1,"sense_key":"too%4:02:00::","synset_id":"excessively.r.01"},{"index":2,"sense_key":"too%4:02:01::","synset_id":"besides.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"too%4:02:00::","call_ids":["senseval2.d002.s051.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"too%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":555,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00123,"input_uncached_usd":0.0011099999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0076927079353482}
{"item_id":"senseval2.d002.s051.t002","gold_sense_keys":["likely%3:00:00::"],"candidates":[{"index":1,"sense_key":"likely%3:00:00::","synset_id":"likely.a.01"},{"index":2,"sense_key":"likely%3:00:04::","synset_id":"probable.a.01"},{"index":3,"sense_key":"likely%5:00:00:prospective:00","synset_id":"likely.s.01"},{"index":4,"sense_key":"likely%5:00:00:credible:00","synset_id":"likely.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"likely%3:00:00::","call_ids":["senseval2.d002.s051.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"likely%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":666,"cached_input_tokens":0,"output_tokens":70,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002032,"input_uncached_usd":0.0013319999999999999,"input_cached_usd":0.0,"output_usd":0.0007000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5446067498996854}
{"item_id":"senseval2.d002.s051.t003","gold_sense_keys":["blame%2:31:00::","blame%2:32:00::"],"candidates":[{"index":1,"sense_key":"blame%2:32:00::","synset_id":"blame.v.01"},{"index":2,"sense_key":"blame%2:32:01::","synset_id":"blame.v.02"},{"index":3,"sense_key":"blame%2:31:00::","synset_id":"blame.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"blame%2:31:00::","call_ids":["senseval2.d002.s051.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"blame%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":604,"cached_input_tokens":0,"output_tokens":174,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002948,"input_uncached_usd":0.0012079999999999999,"input_cached_usd":0.0,"output_usd":0.0017400000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.004209583043121}
{"item_id":"senseval2.d002.s051.t004","gold_sense_keys":["school%1:14:00::"],"candidates":[{"index":1,"sense_key":"school%1:14:00::","synset_id":"school.n.01"},{"index":2,"sense_key":"school%1:06:00::","synset_id":"school.n.02"},{"index":3,"sense_key":"school%1:09:00::","synset_id":"school.n.03"},{"index":4,"sense_key":"school%1:14:02::","synset_id":"school.n.04"},{"index":5,"sense_key":"school%1:28:00::","synset_id":"school.n.05"},{"index":6,"sense_key":"school%1:14:03::","synset_id":"school.n.06"},{"index":7,"sense_key":"school%1:14:01::","synset_id":"school.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"school%1:14:00::","call_ids":["senseval2.d002.s051.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"school%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":841,"cached_input_tokens":0,"output_tokens":133,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003012,"input_uncached_usd":0.001682,"input_cached_usd":0.0,"output_usd":0.00133,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8196064579533413}
{"item_id":"senseval2.d002.s051.t005","gold_sense_keys":["educational%3:01:00::"],"candidates":[{"index":1,"sense_key":"educational%3:01:00::","synset_id":"educational.a.01"},{"index":2,"sense_key":"educational%5:00:00:instructive:00","synset_id":"educational.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"educational%3:01:00::","call_ids":["senseval2.d002.s051.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"educational%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":522,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0016840000000000002,"input_uncached_usd":0.001044,"input_cached_usd":0.0,"output_usd":0.00064,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3944014998851344}
{"item_id":"senseval2.d002.s051.t006","gold_sense_keys":["limitation%1:07:00::"],"candidates":[{"index":1,"sense_key":"limitation%1:09:00::","synset_id":"restriction.n.01"},{"index":2,"sense_key":"limitation%1:07:00::","synset_id":"limitation.n.02"},{"index":3,"sense_key":"limitation%1:23:00::","synset_id":"limit.n.06"},{"index":4,"sense_key":"limitation%1:28:00::","synset_id":"limitation.n.04"},{"index":5,"sense_key":"limitation%1:04:00::","synset_id":"limitation.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"limitation%1:07:00::","call_ids":["senseval2.d002.s051.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"limitation%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":739,"cached_input_tokens":0,"output_tokens":79,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002268,"input_uncached_usd":0.001478,"input_cached_usd":0.0,"output_usd":0.00079,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5849296250380576}
{"item_id":"senseval2.d002.s051.t007","gold_sense_keys":["child%1:18:01::"],"candidates":[{"index":1,"sense_key":"child%1:18:00::","synset_id":"child.n.01"},{"index":2,"sense_key":"child%1:18:01::","synset_id":"child.n.02"},{"index":3,"sense_key":"child%1:18:06::","synset_id":"child.n.03"},{"index":4,"sense_key":"child%1:18:07::","synset_id":"child.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"child%1:18:01::","call_ids":["senseval2.d002.s051.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"child%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":679,"cached_input_tokens":0,"output_tokens":79,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0021479999999999997,"input_uncached_usd":0.0013579999999999998,"input_cached_usd":0.0,"output_usd":0.00079,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5283690830692649}
{"item_id":"senseval2.d002.s052.t000","gold_sense_keys":["parent%1:18:00::"],"candidates":[{"index":1,"sense_key":"parent%1:18:00::","synset_id":"parent.n.01"},{"index":2,"sense_key":"parent%1:03:00::","synset_id":"parent.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"parent%1:18:00::","call_ids":["senseval2.d002.s052.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"parent%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":565,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00125,"input_uncached_usd":0.00113,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1795009589986876}
{"item_id":"senseval2.d002.s052.t001","gold_sense_keys":["involved%3:00:00::"],"candidates":[{"index":1,"sense_key":"involved%3:00:00::","synset_id":"involved.a.01"},{"index":2,"sense_key":"involved%5:00:00:encumbered:00","synset_id":"involved.s.01"},{"index":3,"sense_key":"involved%5:00:00:attached:01","synset_id":"involved.s.02"},{"index":4,"sense_key":"involved%5:00:00:complex:00","synset_id":"byzantine.s.01"},{"index":5,"sense_key":"involved%5:00:00:enclosed:00","synset_id":"involved.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"involved%3:00:00::","call_ids":["senseval2.d002.s052.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"involved%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":821,"cached_input_tokens":0,"output_tokens":96,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002602,"input_uncached_usd":0.001642,"input_cached_usd":0.0,"output_usd":0.0009600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5896307500079274}
{"item_id":"senseval2.d002.s052.t002","gold_sense_keys":["child%1:18:01::"],"candidates":[{"index":1,"sense_key":"child%1:18:00::","synset_id":"child.n.01"},{"index":2,"sense_key":"child%1:18:01::","synset_id":"child.n.02"},{"index":3,"sense_key":"child%1:18:06::","synset_id":"child.n.03"},{"index":4,"sense_key":"child%1:18:07::","synset_id":"child.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"child%1:18:01::","call_ids":["senseval2.d002.s052.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"child%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":696,"cached_input_tokens":0,"output_tokens":99,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002382,"input_uncached_usd":0.001392,"input_cached_usd":0.0,"output_usd":0.00099,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0070604999782518}
{"item_id":"senseval2.d002.s052.t003","gold_sense_keys":["education%1:09:01::"],"candidates":[{"index":1,"sense_key":"education%1:04:00::","synset_id":"education.n.01"},{"index":2,"sense_key":"education%1:09:00::","synset_id":"education.n.02"},{"index":3,"sense_key":"education%1:09:01::","synset_id":"education.n.03"},{"index":4,"sense_key":"education%1:04:01::","synset_id":"education.n.04"},{"index":5,"sense_key":"education%1:07:01::","synset_id":"education.n.05"},{"index":6,"sense_key":"education%1:14:00::","synset_id":"department_of_education.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"education%1:09:01::","call_ids":["senseval2.d002.s052.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"education%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":848,"cached_input_tokens":0,"output_tokens":113,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0028260000000000004,"input_uncached_usd":0.001696,"input_cached_usd":0.0,"output_usd":0.0011300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.137701083905995}
{"item_id":"senseval2.d002.s052.t004","gold_sense_keys":["home%1:15:04::"],"candidates":[{"index":1,"sense_key":"home%1:15:04::","synset_id":"home.n.01"},{"index":2,"sense_key":"home%1:06:00::","synset_id":"dwelling.n.01"},{"index":3,"sense_key":"home%1:15:02::","synset_id":"home.n.03"},{"index":4,"sense_key":"home%1:06:02::","synset_id":"home_plate.n.01"},{"index":5,"sense_key":"home%1:15:05::","synset_id":"base.n.14"},{"index":6,"sense_key":"home%1:15:03::","synset_id":"home.n.06"},{"index":7,"sense_key":"home%1:26:00::","synset_id":"home.n.07"},{"index":8,"sense_key":"home%1:14:00::","synset_id":"family.n.01"},{"index":9,"sense_key":"home%1:06:01::","synset_id":"home.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"home%1:15:04::","call_ids":["senseval2.d002.s052.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"home%1:15:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1023,"cached_input_tokens":0,"output_tokens":38,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002426,"input_uncached_usd":0.002046,"input_cached_usd":0.0,"output_usd":0.00038,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.175839208997786}
{"item_id":"senseval2.d002.s053.t000","gold_sense_keys":["see%2:31:03::"],"candidates":[{"index":1,"sense_key":"see%2:39:00::","synset_id":"see.v.01"},{"index":2,"sense_key":"see%2:31:01::","synset_id":"understand.v.02"},{"index":3,"sense_key":"see%2:39:02::","synset_id":"witness.v.02"},{"index":4,"sense_key":"see%2:36:00::","synset_id":"visualize.v.01"},{"index":5,"sense_key":"see%2:31:00::","synset_id":"see.v.05"},{"index":6,"sense_key":"see%2:31:02::","synset_id":"learn.v.02"},{"index":7,"sense_key":"see%2:39:03::","synset_id":"watch.v.03"},{"index":8,"sense_key":"see%2:38:00::","synset_id":"meet.v.01"},{"index":9,"sense_key":"see%2:32:00::","synset_id":"determine.v.08"},{"index":10,"sense_key":"see%2:31:03::","synset_id":"see.v.10"},{"index":11,"sense_key":"see%2:41:11::","synset_id":"see.v.11"},{"index":12,"sense_key":"see%2:41:12::","synset_id":"see.v.12"},{"index":13,"sense_key":"see%2:41:10::","synset_id":"visit.v.01"},{"index":14,"sense_key":"see%2:41:14::","synset_id":"attend.v.02"},{"index":15,"sense_key":"see%2:35:09::","synset_id":"see.v.15"},{"index":16,"sense_key":"see%2:41:00::","synset_id":"go_steady.v.01"},{"index":17,"sense_key":"see%2:39:05::","synset_id":"see.v.17"},{"index":18,"sense_key":"see%2:31:08::","synset_id":"see.v.18"},{"index":19,"sense_key":"see%2:39:11::","synset_id":"see.v.19"},{"index":20,"sense_key":"see%2:39:13::","synset_id":"examine.v.02"},{"index":21,"sense_key":"see%2:39:12::","synset_id":"experience.v.01"},{"index":22,"sense_key":"see%2:38:03::","synset_id":"see.v.22"},{"index":23,"sense_key":"see%2:33:00::","synset_id":"see.v.23"},{"index":24,"sense_key":"see%2:31:13::","synset_id":"interpret.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"see%2:31:03::","call_ids":["senseval2.d002.s053.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"see%2:31:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1999,"cached_input_tokens":0,"output_tokens":51,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004508,"input_uncached_usd":0.003998,"input_cached_usd":0.0,"output_usd":0.00051,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7417151249246672}
{"item_id":"senseval2.d002.s053.t001","gold_sense_keys":["kid%1:18:01::"],"candidates":[{"index":1,"sense_key":"kid%1:18:00::","synset_id":"child.n.01"},{"index":2,"sense_key":"kid%1:27:00::","synset_id":"kid.n.02"},{"index":3,"sense_key":"kid%1:18:02::","synset_id":"kyd.n.01"},{"index":4,"sense_key":"kid%1:18:01::","synset_id":"child.n.02"},{"index":5,"sense_key":"kid%1:05:00::","synset_id":"kid.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"kid%1:18:01::","call_ids":["senseval2.d002.s053.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"kid%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":723,"cached_input_tokens":0,"output_tokens":120,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002646,"input_uncached_usd":0.001446,"input_cached_usd":0.0,"output_usd":0.0012000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.286387167056091}
{"item_id":"senseval2.d002.s053.t002","gold_sense_keys":["play%2:41:02::"],"candidates":[{"index":1,"sense_key":"play%2:33:00::","synset_id":"play.v.01"},{"index":2,"sense_key":"play%2:41:12::","synset_id":"play.v.02"},{"index":3,"sense_key":"play%2:36:12::","synset_id":"play.v.03"},{"index":4,"sense_key":"play%2:36:02::","synset_id":"act.v.03"},{"index":5,"sense_key":"play%2:41:03::","synset_id":"play.v.05"},{"index":6,"sense_key":"play%2:36:00::","synset_id":"play.v.06"},{"index":7,"sense_key":"play%2:36:01::","synset_id":"play.v.07"},{"index":8,"sense_key":"play%2:29:01::","synset_id":"act.v.05"},{"index":9,"sense_key":"play%2:38:00::","synset_id":"play.v.09"},{"index":10,"sense_key":"play%2:33:01::","synset_id":"play.v.10"},{"index":11,"sense_key":"play%2:41:00::","synset_id":"play.v.11"},{"index":12,"sense_key":"play%2:36:06::","synset_id":"play.v.12"},{"index":13,"sense_key":"play%2:36:11::","synset_id":"play.v.13"},{"index":14,"sense_key":"play%2:36:03::","synset_id":"play.v.14"},{"index":15,"sense_key":"play%2:33:03::","synset_id":"play.v.15"},{"index":16,"sense_key":"play%2:29:00::","synset_id":"play.v.16"},{"index":17,"sense_key":"play%2:41:02::","synset_id":"play.v.17"},{"index":18,"sense_key":"play%2:36:05::","synset_id":"play.v.18"},{"index":19,"sense_key":"play%2:35:00::","synset_id":"toy.v.02"},{"index":20,"sense_key":"play%2:34:13::","synset_id":"play.v.20"},{"index":21,"sense_key":"play%2:31:01::","synset_id":"dally.v.04"},{"index":22,"sense_key":"play%2:42:00::","synset_id":"play.v.22"},{"index":23,"sense_key":"play%2:41:13::","synset_id":"dally.v.01"},{"index":24,"sense_key":"play%2:38:03::","synset_id":"play.v.24"},{"index":25,"sense_key":"play%2:36:10::","synset_id":"act.v.10"},{"index":26,"sense_key":"play%2:36:04::","synset_id":"play.v.26"},{"index":27,"sense_key":"play%2:36:13::","synset_id":"bring.v.03"},{"index":28,"sense_key":"play%2:35:07::","synset_id":"play.v.28"},{"index":29,"sense_key":"play%2:33:07::","synset_id":"play.v.29"},{"index":30,"sense_key":"play%2:33:08::","synset_id":"bet.v.02"},{"index":31,"sense_key":"play%2:33:09::","synset_id":"play.v.31"},{"index":32,"sense_key":"play%2:33:13::","synset_id":"play.v.32"},{"index":33,"sense_key":"play%2:33:12::","synset_id":"play.v.33"},{"index":34,"sense_key":"play%2:33:02::","synset_id":"meet.v.10"},{"index":35,"sense_key":"play%2:29:04::","synset_id":"play.v.35"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":17,"chosen_sense_key":"play%2:41:02::","call_ids":["senseval2.d002.s053.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":17,"predicted_sense_key":"play%2:41:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2572,"cached_input_tokens":0,"output_tokens":199,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.007134,"input_uncached_usd":0.005144,"input_cached_usd":0.0,"output_usd":0.00199,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.3211355410749093}
{"item_id":"senseval2.d002.s053.t006","gold_sense_keys":["child%1:18:00::"],"candidates":[{"index":1,"sense_key":"child%1:18:00::","synset_id":"child.n.01"},{"index":2,"sense_key":"child%1:18:01::","synset_id":"child.n.02"},{"index":3,"sense_key":"child%1:18:06::","synset_id":"child.n.03"},{"index":4,"sense_key":"child%1:18:07::","synset_id":"child.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"child%1:18:00::","call_ids":["senseval2.d002.s053.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"child%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":682,"cached_input_tokens":0,"output_tokens":120,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0025640000000000003,"input_uncached_usd":0.001364,"input_cached_usd":0.0,"output_usd":0.0012000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.523214332992211}
{"item_id":"senseval2.d002.s053.t007","gold_sense_keys":["spend%2:42:00::"],"candidates":[{"index":1,"sense_key":"spend%2:42:00::","synset_id":"spend.v.01"},{"index":2,"sense_key":"spend%2:40:00::","synset_id":"spend.v.02"},{"index":3,"sense_key":"spend%2:40:01::","synset_id":"spend.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"spend%2:42:00::","call_ids":["senseval2.d002.s053.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"spend%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":584,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001288,"input_uncached_usd":0.001168,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9841061669867486}
{"item_id":"senseval2.d002.s053.t009","gold_sense_keys":["time%1:28:05::"],"candidates":[{"index":1,"sense_key":"time%1:11:00::","synset_id":"time.n.01"},{"index":2,"sense_key":"time%1:28:05::","synset_id":"time.n.02"},{"index":3,"sense_key":"time%1:28:00::","synset_id":"time.n.03"},{"index":4,"sense_key":"time%1:28:06::","synset_id":"time.n.04"},{"index":5,"sense_key":"time%1:03:00::","synset_id":"time.n.05"},{"index":6,"sense_key":"time%1:11:01::","synset_id":"time.n.06"},{"index":7,"sense_key":"time%1:28:03::","synset_id":"clock_time.n.01"},{"index":8,"sense_key":"time%1:28:04::","synset_id":"fourth_dimension.n.01"},{"index":9,"sense_key":"time%1:07:01::","synset_id":"meter.n.04"},{"index":10,"sense_key":"time%1:28:01::","synset_id":"prison_term.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"time%1:28:05::","call_ids":["senseval2.d002.s053.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"time%1:28:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1015,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0027600000000000003,"input_uncached_usd":0.00203,"input_cached_usd":0.0,"output_usd":0.0007300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":4.016768000088632}
{"item_id":"senseval2.d002.s053.t011","gold_sense_keys":["scrutinize%2:39:00::"],"candidates":[{"index":1,"sense_key":"scrutinize%2:39:00::","synset_id":"size_up.v.01"},{"index":2,"sense_key":"scrutinize%2:31:00::","synset_id":"audit.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"scrutinize%2:31:00::","call_ids":["senseval2.d002.s053.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"scrutinize%2:31:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":605,"cached_input_tokens":0,"output_tokens":196,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00317,"input_uncached_usd":0.00121,"input_cached_usd":0.0,"output_usd":0.0019600000000000004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.155344667029567}
{"item_id":"senseval2.d002.s054.t000","gold_sense_keys":["parent%1:18:00::"],"candidates":[{"index":1,"sense_key":"parent%1:18:00::","synset_id":"parent.n.01"},{"index":2,"sense_key":"parent%1:03:00::","synset_id":"parent.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"parent%1:18:00::","call_ids":["senseval2.d002.s054.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"parent%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":545,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0012100000000000001,"input_uncached_usd":0.00109,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.986059834016487}
{"item_id":"senseval2.d002.s054.t002","gold_sense_keys":["option%1:09:00::"],"candidates":[{"index":1,"sense_key":"option%1:21:00::","synset_id":"option.n.01"},{"index":2,"sense_key":"option%1:09:00::","synset_id":"option.n.02"},{"index":3,"sense_key":"option%1:04:00::","synset_id":"choice.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"option%1:09:00::","call_ids":["senseval2.d002.s054.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"option%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":634,"cached_input_tokens":0,"output_tokens":132,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002588,"input_uncached_usd":0.001268,"input_cached_usd":0.0,"output_usd":0.0013200000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8120113329496235}
{"item_id":"senseval2.d002.s054.t003","gold_sense_keys":["switch%2:30:02::"],"candidates":[{"index":1,"sense_key":"switch%2:30:03::","synset_id":"switch_over.v.01"},{"index":2,"sense_key":"switch%2:40:00::","synset_id":"trade.v.04"},{"index":3,"sense_key":"switch%2:30:02::","synset_id":"switch.v.03"},{"index":4,"sense_key":"switch%2:30:01::","synset_id":"switch.v.04"},{"index":5,"sense_key":"switch%2:35:00::","synset_id":"throw.v.06"},{"index":6,"sense_key":"switch%2:35:03::","synset_id":"switch.v.06"},{"index":7,"sense_key":"switch%2:30:00::","synset_id":"interchange.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"switch%2:30:02::","call_ids":["senseval2.d002.s054.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"switch%2:30:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":829,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002428,"input_uncached_usd":0.001658,"input_cached_usd":0.0,"output_usd":0.0007700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5447205000091344}
{"item_id":"senseval2.d002.s055.t000","gold_sense_keys":["community%1:14:00::"],"candidates":[{"index":1,"sense_key":"community%1:14:00::","synset_id":"community.n.01"},{"index":2,"sense_key":"community%1:21:00::","synset_id":"community.n.02"},{"index":3,"sense_key":"community%1:14:04::","synset_id":"community.n.03"},{"index":4,"sense_key":"community%1:26:00::","synset_id":"community.n.04"},{"index":5,"sense_key":"community%1:15:00::","synset_id":"residential_district.n.01"},{"index":6,"sense_key":"community%1:14:02::","synset_id":"community.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"community%1:14:00::","call_ids":["senseval2.d002.s055.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"community%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":721,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001562,"input_uncached_usd":0.001442,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.863502583000809}
{"item_id":"senseval2.d002.s055.t002","gold_sense_keys":["even%4:02:01::"],"candidates":[{"index":1,"sense_key":"even%4:02:00::","synset_id":"even.r.01"},{"index":2,"sense_key":"even%4:02:03::","synset_id":"even.r.02"},{"index":3,"sense_key":"even%4:02:01::","synset_id":"even.r.03"},{"index":4,"sense_key":"even%4:02:04::","synset_id":"even.r.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"even%4:02:01::","call_ids":["senseval2.d002.s055.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"even%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":644,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001408,"input_uncached_usd":0.0012879999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.987456624978222}
{"item_id":"senseval2.d002.s055.t003","gold_sense_keys":["worse%3:00:00::"],"candidates":[{"index":1,"sense_key":"worse%3:00:00::","synset_id":"worse.a.01"},{"index":2,"sense_key":"worse%3:00:02::","synset_id":"worse.a.02"},{"index":3,"sense_key":"bad%3:00:00::","synset_id":"bad.a.01"},{"index":4,"sense_key":"bad%5:00:00:intense:00","synset_id":"bad.s.01"},{"index":5,"sense_key":"bad%5:00:00:uncomfortable:00","synset_id":"bad.s.02"},{"index":6,"sense_key":"bad%5:00:00:stale:00","synset_id":"bad.s.03"},{"index":7,"sense_key":"regretful%3:00:00::","synset_id":"regretful.a.01"},{"index":8,"sense_key":"bad%5:00:00:invalid:00","synset_id":"bad.s.04"},{"index":9,"sense_key":"bad%5:00:00:inferior:02","synset_id":"bad.s.05"},{"index":10,"sense_key":"bad%5:00:00:nonstandard:03","synset_id":"bad.s.06"},{"index":11,"sense_key":"bad%5:00:02:unsound:00","synset_id":"bad.s.07"},{"index":12,"sense_key":"bad%5:00:00:unhealthy:00","synset_id":"bad.s.08"},{"index":13,"sense_key":"bad%5:00:00:harmful:00","synset_id":"bad.s.09"},{"index":14,"sense_key":"bad%5:00:00:evil:00","synset_id":"bad.s.10"},{"index":15,"sense_key":"bad%5:00:00:counterfeit:00","synset_id":"bad.s.11"},{"index":16,"sense_key":"bad%5:00:00:malfunctioning:00","synset_id":"bad.s.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"worse%3:00:00::","call_ids":["senseval2.d002.s055.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"worse%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1506,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003582,"input_uncached_usd":0.003012,"input_cached_usd":0.0,"output_usd":0.0005700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4983559579122812}
{"item_id":"senseval2.d002.s055.t004","gold_sense_keys":["idea%1:09:00::"],"candidates":[{"index":1,"sense_key":"idea%1:09:00::","synset_id":"idea.n.01"},{"index":2,"sense_key":"idea%1:09:01::","synset_id":"mind.n.06"},{"index":3,"sense_key":"idea%1:09:03::","synset_id":"idea.n.03"},{"index":4,"sense_key":"idea%1:09:02::","synset_id":"estimate.n.01"},{"index":5,"sense_key":"idea%1:10:00::","synset_id":"theme.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"idea%1:09:00::","call_ids":["senseval2.d002.s055.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"idea%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":768,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0016560000000000001,"input_uncached_usd":0.001536,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4394681659759954}
{"item_id":"senseval2.d002.s056.t000","gold_sense_keys":["here%4:02:02::"],"candidates":[{"index":1,"sense_key":"here%4:02:00::","synset_id":"here.r.01"},{"index":2,"sense_key":"here%4:02:02::","synset_id":"here.r.02"},{"index":3,"sense_key":"here%4:02:01::","synset_id":"here.r.03"},{"index":4,"sense_key":"here%4:02:04::","synset_id":"here.r.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"here%4:02:02::","call_ids":["senseval2.d002.s056.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"here%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":636,"cached_input_tokens":0,"output_tokens":78,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002052,"input_uncached_usd":0.001272,"input_cached_usd":0.0,"output_usd":0.0007800000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4264979580184445}
{"item_id":"senseval2.d002.s057.t000","gold_sense_keys":["locally%4:02:00::"],"candidates":[{"index":1,"sense_key":"locally%4:02:00::","synset_id":"locally.r.01"},{"index":2,"sense_key":"locally%4:02:01::","synset_id":"locally.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"locally%4:02:00::","call_ids":["senseval2.d002.s057.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"locally%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":548,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001216,"input_uncached_usd":0.001096,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2148025420028716}
{"item_id":"senseval2.d002.s057.t003","gold_sense_keys":["especially%4:02:00::"],"candidates":[{"index":1,"sense_key":"especially%4:02:00::","synset_id":"particularly.r.01"},{"index":2,"sense_key":"especially%4:02:01::","synset_id":"specially.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"especially%4:02:00::","call_ids":["senseval2.d002.s057.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"especially%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":589,"cached_input_tokens":0,"output_tokens":83,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0020080000000000002,"input_uncached_usd":0.001178,"input_cached_usd":0.0,"output_usd":0.0008300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.172793374978937}
{"item_id":"senseval2.d002.s057.t005","gold_sense_keys":["city%1:15:00::"],"candidates":[{"index":1,"sense_key":"city%1:15:00::","synset_id":"city.n.01"},{"index":2,"sense_key":"city%1:15:01::","synset_id":"city.n.02"},{"index":3,"sense_key":"city%1:14:00::","synset_id":"city.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"city%1:15:01::","call_ids":["senseval2.d002.s057.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"city%1:15:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":636,"cached_input_tokens":0,"output_tokens":200,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0032719999999999997,"input_uncached_usd":0.001272,"input_cached_usd":0.0,"output_usd":0.002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.4499269579537213}
{"item_id":"senseval2.d002.s057.t006","gold_sense_keys":["become%2:42:01::"],"candidates":[{"index":1,"sense_key":"become%2:30:00::","synset_id":"become.v.01"},{"index":2,"sense_key":"become%2:42:01::","synset_id":"become.v.02"},{"index":3,"sense_key":"become%2:42:00::","synset_id":"become.v.03"},{"index":4,"sense_key":"become%2:42:03::","synset_id":"become.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"become%2:42:01::","call_ids":["senseval2.d002.s057.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"become%2:42:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":685,"cached_input_tokens":0,"output_tokens":134,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0027099999999999997,"input_uncached_usd":0.00137,"input_cached_usd":0.0,"output_usd":0.00134,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0111986249685287}
{"item_id":"senseval2.d002.s057.t007","gold_sense_keys":["ambitious%3:00:00::"],"candidates":[{"index":1,"sense_key":"ambitious%3:00:00::","synset_id":"ambitious.a.01"},{"index":2,"sense_key":"ambitious%5:00:00:difficult:00","synset_id":"ambitious.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ambitious%3:00:00::","call_ids":["senseval2.d002.s057.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ambitious%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":574,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001268,"input_uncached_usd":0.001148,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5126021660398692}
{"item_id":"senseval2.d002.s057.t008","gold_sense_keys":["generally%4:02:02::"],"candidates":[{"index":1,"sense_key":"generally%4:02:02::","synset_id":"by_and_large.r.01"},{"index":2,"sense_key":"generally%4:02:01::","synset_id":"generally.r.02"},{"index":3,"sense_key":"generally%4:02:00::","synset_id":"broadly.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"generally%4:02:02::","call_ids":["senseval2.d002.s057.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"generally%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":644,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0018679999999999999,"input_uncached_usd":0.0012879999999999999,"input_cached_usd":0.0,"output_usd":0.00058,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6376376670086756}
{"item_id":"senseval2.d002.s057.t009","gold_sense_keys":["corrupt%3:00:00::","corrupt%3:00:04::"],"candidates":[{"index":1,"sense_key":"corrupt%3:00:00::","synset_id":"corrupt.a.01"},{"index":2,"sense_key":"corrupt%3:00:04::","synset_id":"crooked.a.02"},{"index":3,"sense_key":"corrupt%5:00:00:imperfect:00","synset_id":"corrupt.s.01"},{"index":4,"sense_key":"corrupt%5:00:00:stale:00","synset_id":"corrupt.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"corrupt%3:00:00::","call_ids":["senseval2.d002.s057.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"corrupt%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":719,"cached_input_tokens":0,"output_tokens":198,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003418,"input_uncached_usd":0.001438,"input_cached_usd":0.0,"output_usd":0.00198,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.0334044999908656}
{"item_id":"senseval2.d002.s057.t012","gold_sense_keys":["local%3:00:01::"],"candidates":[{"index":1,"sense_key":"local%3:00:01::","synset_id":"local.a.01"},{"index":2,"sense_key":"local%3:01:01::","synset_id":"local.a.02"},{"index":3,"sense_key":"local%3:00:02::","synset_id":"local.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"local%3:00:01::","call_ids":["senseval2.d002.s057.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"local%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":620,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0018800000000000002,"input_uncached_usd":0.00124,"input_cached_usd":0.0,"output_usd":0.00064,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3283289580140263}
{"item_id":"senseval2.d002.s057.t013","gold_sense_keys":["politician%1:18:00::","politician%1:18:01::"],"candidates":[{"index":1,"sense_key":"politician%1:18:01::","synset_id":"politician.n.01"},{"index":2,"sense_key":"politician%1:18:00::","synset_id":"politician.n.02"},{"index":3,"sense_key":"politician%1:18:02::","synset_id":"politician.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"politician%1:18:00::","call_ids":["senseval2.d002.s057.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"politician%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":585,"cached_input_tokens":0,"output_tokens":173,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0029000000000000002,"input_uncached_usd":0.00117,"input_cached_usd":0.0,"output_usd":0.0017300000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.403259207960218}
{"item_id":"senseval2.d002.s057.t015","gold_sense_keys":["politician%1:18:00::","politician%1:18:01::"],"candidates":[{"index":1,"sense_key":"politician%1:18:01::","synset_id":"politician.n.01"},{"index":2,"sense_key":"politician%1:18:00::","synset_id":"politician.n.02"},{"index":3,"sense_key":"politician%1:18:02::","synset_id":"politician.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"politician%1:18:00::","call_ids":["senseval2.d002.s057.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"politician%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":585,"cached_input_tokens":0,"output_tokens":157,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0027400000000000002,"input_uncached_usd":0.00117,"input_cached_usd":0.0,"output_usd":0.0015700000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9079479159554467}
{"item_id":"senseval2.d002.s058.t000","gold_sense_keys":["process%1:04:00::"],"candidates":[{"index":1,"sense_key":"process%1:04:00::","synset_id":"procedure.n.01"},{"index":2,"sense_key":"process%1:09:00::","synset_id":"process.n.02"},{"index":3,"sense_key":"process%1:10:00::","synset_id":"summons.n.03"},{"index":4,"sense_key":"process%1:09:01::","synset_id":"process.n.04"},{"index":5,"sense_key":"process%1:08:00::","synset_id":"process.n.05"},{"index":6,"sense_key":"process%1:03:00::","synset_id":"process.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"process%1:03:00::","call_ids":["senseval2.d002.s058.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"process%1:03:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":917,"cached_input_tokens":0,"output_tokens":156,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0033940000000000003,"input_uncached_usd":0.001834,"input_cached_usd":0.0,"output_usd":0.0015600000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.5993251249892637}
{"item_id":"senseval2.d002.s058.t001","gold_sense_keys":["try%2:41:00::"],"candidates":[{"index":1,"sense_key":"try%2:41:00::","synset_id":"try.v.01"},{"index":2,"sense_key":"try%2:41:01::","synset_id":"test.v.01"},{"index":3,"sense_key":"try%2:41:06::","synset_id":"judge.v.05"},{"index":4,"sense_key":"try%2:34:00::","synset_id":"sample.v.01"},{"index":5,"sense_key":"try%2:41:05::","synset_id":"hear.v.03"},{"index":6,"sense_key":"try%2:37:01::","synset_id":"try.v.06"},{"index":7,"sense_key":"try%2:37:00::","synset_id":"try.v.07"},{"index":8,"sense_key":"try%2:30:00::","synset_id":"try.v.08"},{"index":9,"sense_key":"try%2:29:00::","synset_id":"try_on.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"try%2:41:00::","call_ids":["senseval2.d002.s058.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"try%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1073,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002266,"input_uncached_usd":0.002146,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1378372079925612}
{"item_id":"senseval2.d002.s058.t002","gold_sense_keys":["system%1:09:00::"],"candidates":[{"index":1,"sense_key":"system%1:06:00::","synset_id":"system.n.01"},{"index":2,"sense_key":"system%1:14:00::","synset_id":"system.n.02"},{"index":3,"sense_key":"system%1:27:00::","synset_id":"system.n.03"},{"index":4,"sense_key":"system%1:09:00::","synset_id":"system.n.04"},{"index":5,"sense_key":"system%1:09:02::","synset_id":"arrangement.n.03"},{"index":6,"sense_key":"system%1:08:00::","synset_id":"system.n.06"},{"index":7,"sense_key":"system%1:09:01::","synset_id":"system.n.07"},{"index":8,"sense_key":"system%1:08:01::","synset_id":"system.n.08"},{"index":9,"sense_key":"system%1:07:00::","synset_id":"organization.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"system%1:09:00::","call_ids":["senseval2.d002.s058.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"system%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1137,"cached_input_tokens":0,"output_tokens":283,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.005104,"input_uncached_usd":0.002274,"input_cached_usd":0.0,"output_usd":0.00283,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.7528194590704516}
{"item_id":"senseval2.d002.s058.t003","gold_sense_keys":["school%1:14:00::"],"candidates":[{"index":1,"sense_key":"school%1:14:00::","synset_id":"school.n.01"},{"index":2,"sense_key":"school%1:06:00::","synset_id":"school.n.02"},{"index":3,"sense_key":"school%1:09:00::","synset_id":"school.n.03"},{"index":4,"sense_key":"school%1:14:02::","synset_id":"school.n.04"},{"index":5,"sense_key":"school%1:28:00::","synset_id":"school.n.05"},{"index":6,"sense_key":"school%1:14:03::","synset_id":"school.n.06"},{"index":7,"sense_key":"school%1:14:01::","synset_id":"school.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"school%1:14:00::","call_ids":["senseval2.d002.s058.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"school%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":871,"cached_input_tokens":0,"output_tokens":74,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002482,"input_uncached_usd":0.0017419999999999998,"input_cached_usd":0.0,"output_usd":0.0007400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3644629589980468}
{"item_id":"senseval2.d002.s058.t004","gold_sense_keys":["governance%1:04:00::"],"candidates":[{"index":1,"sense_key":"governance%1:14:00::","synset_id":"administration.n.02"},{"index":2,"sense_key":"governance%1:04:00::","synset_id":"government.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"governance%1:04:00::","call_ids":["senseval2.d002.s058.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"governance%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":657,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002044,"input_uncached_usd":0.0013139999999999998,"input_cached_usd":0.0,"output_usd":0.0007300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":4.521653709001839}
{"item_id":"senseval2.d002.s058.t006","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":2,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":3,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"other%3:00:00::","call_ids":["senseval2.d002.s058.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":707,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001534,"input_uncached_usd":0.001414,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3934159160126}
{"item_id":"senseval2.d002.s058.t007","gold_sense_keys":["city%1:15:01::"],"candidates":[{"index":1,"sense_key":"city%1:15:00::","synset_id":"city.n.01"},{"index":2,"sense_key":"city%1:15:01::","synset_id":"city.n.02"},{"index":3,"sense_key":"city%1:14:00::","synset_id":"city.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"city%1:15:01::","call_ids":["senseval2.d002.s058.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"city%1:15:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":652,"cached_input_tokens":0,"output_tokens":135,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002654,"input_uncached_usd":0.001304,"input_cached_usd":0.0,"output_usd":0.00135,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6886607089545578}
{"item_id":"senseval2.d002.s058.t008","gold_sense_keys":["institute%2:36:01::"],"candidates":[{"index":1,"sense_key":"institute%2:36:01::","synset_id":"establish.v.02"},{"index":2,"sense_key":"institute%2:36:00::","synset_id":"institute.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"institute%2:36:01::","call_ids":["senseval2.d002.s058.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"institute%2:36:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":588,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001296,"input_uncached_usd":0.001176,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7947846670867875}
{"item_id":"senseval2.d002.s059.t000","gold_sense_keys":["most%3:00:02::"],"candidates":[{"index":1,"sense_key":"most%3:00:02::","synset_id":"most.a.01"},{"index":2,"sense_key":"most%3:00:01::","synset_id":"most.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"most%3:00:02::","call_ids":["senseval2.d002.s059.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"most%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":624,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001788,"input_uncached_usd":0.001248,"input_cached_usd":0.0,"output_usd":0.00054,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.466591083095409}
{"item_id":"senseval2.d002.s059.t002","gold_sense_keys":["expenditure%1:21:00::"],"candidates":[{"index":1,"sense_key":"expenditure%1:21:00::","synset_id":"outgo.n.01"},{"index":2,"sense_key":"expenditure%1:04:00::","synset_id":"expending.n.01"},{"index":3,"sense_key":"expenditure%1:04:01::","synset_id":"consumption.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"expenditure%1:21:00::","call_ids":["senseval2.d002.s059.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"expenditure%1:21:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":599,"cached_input_tokens":0,"output_tokens":92,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002118,"input_uncached_usd":0.0011979999999999998,"input_cached_usd":0.0,"output_usd":0.00092,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.891563875018619}
{"item_id":"senseval2.d002.s059.t003","gold_sense_keys":["education%1:04:00::"],"candidates":[{"index":1,"sense_key":"education%1:04:00::","synset_id":"education.n.01"},{"index":2,"sense_key":"education%1:09:00::","synset_id":"education.n.02"},{"index":3,"sense_key":"education%1:09:01::","synset_id":"education.n.03"},{"index":4,"sense_key":"education%1:04:01::","synset_id":"education.n.04"},{"index":5,"sense_key":"education%1:07:01::","synset_id":"education.n.05"},{"index":6,"sense_key":"education%1:14:00::","synset_id":"department_of_education.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"education%1:04:00::","call_ids":["senseval2.d002.s059.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"education%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":848,"cached_input_tokens":0,"output_tokens":127,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002966,"input_uncached_usd":0.001696,"input_cached_usd":0.0,"output_usd":0.00127,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.919207250000909}
{"item_id":"senseval2.d002.s059.t005","gold_sense_keys":["circumstance%1:26:01::","circumstance%1:26:02::"],"candidates":[{"index":1,"sense_key":"circumstance%1:26:01::","synset_id":"circumstance.n.01"},{"index":2,"sense_key":"circumstance%1:26:02::","synset_id":"context.n.02"},{"index":3,"sense_key":"circumstance%1:09:00::","synset_id":"circumstance.n.03"},{"index":4,"sense_key":"circumstance%1:11:00::","synset_id":"circumstance.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"circumstance%1:26:02::","call_ids":["senseval2.d002.s059.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"circumstance%1:26:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":674,"cached_input_tokens":0,"output_tokens":80,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002148,"input_uncached_usd":0.001348,"input_cached_usd":0.0,"output_usd":0.0008,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4141914999345317}
{"item_id":"senseval2.d002.s059.t006","gold_sense_keys":["probably%4:02:00::"],"candidates":[{"index":1,"sense_key":"probably%4:02:00::","synset_id":"probably.r.01"},{"index":2,"sense_key":"probably%4:02:01::","synset_id":"credibly.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"probably%4:02:00::","call_ids":["senseval2.d002.s059.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"probably%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":618,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001356,"input_uncached_usd":0.001236,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0272274169838056}
{"item_id":"senseval2.d002.s059.t007","gold_sense_keys":["make%2:30:00::"],"candidates":[{"index":1,"sense_key":"make%2:41:00::","synset_id":"make.v.01"},{"index":2,"sense_key":"make%2:30:00::","synset_id":"make.v.02"},{"index":3,"sense_key":"make%2:36:00::","synset_id":"make.v.03"},{"index":4,"sense_key":"make%2:32:00::","synset_id":"induce.v.02"},{"index":5,"sense_key":"make%2:36:08::","synset_id":"cause.v.01"},{"index":6,"sense_key":"make%2:36:01::","synset_id":"produce.v.02"},{"index":7,"sense_key":"make%2:31:00::","synset_id":"draw.v.04"},{"index":8,"sense_key":"make%2:36:15::","synset_id":"make.v.08"},{"index":9,"sense_key":"make%2:36:09::","synset_id":"create.v.05"},{"index":10,"sense_key":"make%2:40:01::","synset_id":"gain.v.08"},{"index":11,"sense_key":"make%2:36:05::","synset_id":"do.v.08"},{"index":12,"sense_key":"make%2:42:05::","synset_id":"form.v.02"},{"index":13,"sense_key":"make%2:38:02::","synset_id":"reach.v.07"},{"index":14,"sense_key":"make%2:42:00::","synset_id":"make.v.14"},{"index":15,"sense_key":"make%2:36:11::","synset_id":"make.v.15"},{"index":16,"sense_key":"make%2:36:12::","synset_id":"make.v.16"},{"index":17,"sense_key":"make%2:36:13::","synset_id":"construct.v.01"},{"index":18,"sense_key":"make%2:30:15::","synset_id":"make.v.18"},{"index":19,"sense_key":"make%2:29:08::","synset_id":"make.v.19"},{"index":20,"sense_key":"make%2:41:03::","synset_id":"name.v.03"},{"index":21,"sense_key":"make%2:40:02::","synset_id":"have.v.17"},{"index":22,"sense_key":"make%2:38:00::","synset_id":"reach.v.01"},{"index":23,"sense_key":"make%2:31:13::","synset_id":"lay_down.v.01"},{"index":24,"sense_key":"make%2:41:13::","synset_id":"make.v.24"},{"index":25,"sense_key":"make%2:41:01::","synset_id":"make.v.25"},{"index":26,"sense_key":"make%2:36:04::","synset_id":"hold.v.03"},{"index":27,"sense_key":"make%2:30:02::","synset_id":"make.v.27"},{"index":28,"sense_key":"make%2:38:05::","synset_id":"take.v.27"},{"index":29,"sense_key":"make%2:29:00::","synset_id":"stool.v.04"},{"index":30,"sense_key":"make%2:42:09::","synset_id":"make.v.30"},{"index":31,"sense_key":"make%2:42:13::","synset_id":"make.v.31"},{"index":32,"sense_key":"make%2:42:12::","synset_id":"make.v.32"},{"index":33,"sense_key":"make%2:42:11::","synset_id":"make.v.33"},{"index":34,"sense_key":"make%2:42:15::","synset_id":"make.v.34"},{"index":35,"sense_key":"make%2:39:12::","synset_id":"make.v.35"},{"index":36,"sense_key":"make%2:38:10::","synset_id":"make.v.36"},{"index":37,"sense_key":"make%2:38:15::","synset_id":"make.v.37"},{"index":38,"sense_key":"make%2:36:10::","synset_id":"make.v.38"},{"index":39,"sense_key":"make%2:36:07::","synset_id":"cook.v.02"},{"index":40,"sense_key":"make%2:35:13::","synset_id":"seduce.v.01"},{"index":41,"sense_key":"make%2:32:14::","synset_id":"make.v.41"},{"index":42,"sense_key":"make%2:32:09::","synset_id":"make.v.42"},{"index":43,"sense_key":"make%2:31:10::","synset_id":"make.v.43"},{"index":44,"sense_key":"make%2:31:12::","synset_id":"make.v.44"},{"index":45,"sense_key":"make%2:30:10::","synset_id":"make.v.45"},{"index":46,"sense_key":"make%2:30:14::","synset_id":"make.v.46"},{"index":47,"sense_key":"make%2:30:12::","synset_id":"make.v.47"},{"index":48,"sense_key":"make%2:29:06::","synset_id":"make.v.48"},{"index":49,"sense_key":"make%2:29:01::","synset_id":"make.v.49"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"make%2:30:00::","call_ids":["senseval2.d002.s059.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"make%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":3124,"cached_input_tokens":0,"output_tokens":166,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.007908,"input_uncached_usd":0.006248,"input_cached_usd":0.0,"output_usd":0.0016600000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1454586250474676}
{"item_id":"senseval2.d002.s059.t008","gold_sense_keys":["thing%1:09:02::"],"candidates":[{"index":1,"sense_key":"thing%1:26:00::","synset_id":"thing.n.01"},{"index":2,"sense_key":"thing%1:04:00::","synset_id":"thing.n.02"},{"index":3,"sense_key":"thing%1:09:00::","synset_id":"thing.n.03"},{"index":4,"sense_key":"thing%1:06:00::","synset_id":"thing.n.04"},{"index":5,"sense_key":"thing%1:11:00::","synset_id":"thing.n.05"},{"index":6,"sense_key":"thing%1:09:02::","synset_id":"matter.n.01"},{"index":7,"sense_key":"thing%1:10:00::","synset_id":"thing.n.07"},{"index":8,"sense_key":"thing%1:06:01::","synset_id":"thing.n.08"},{"index":9,"sense_key":"thing%1:07:00::","synset_id":"thing.n.09"},{"index":10,"sense_key":"thing%1:09:01::","synset_id":"thing.n.10"},{"index":11,"sense_key":"thing%1:12:00::","synset_id":"thing.n.11"},{"index":12,"sense_key":"thing%1:03:00::","synset_id":"thing.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"thing%1:26:00::","call_ids":["senseval2.d002.s059.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"thing%1:26:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1014,"cached_input_tokens":0,"output_tokens":126,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003288,"input_uncached_usd":0.002028,"input_cached_usd":0.0,"output_usd":0.00126,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8290328750154004}
{"item_id":"senseval2.d002.s059.t009","gold_sense_keys":["worse%3:00:00::"],"candidates":[{"index":1,"sense_key":"worse%3:00:00::","synset_id":"worse.a.01"},{"index":2,"sense_key":"worse%3:00:02::","synset_id":"worse.a.02"},{"index":3,"sense_key":"bad%3:00:00::","synset_id":"bad.a.01"},{"index":4,"sense_key":"bad%5:00:00:intense:00","synset_id":"bad.s.01"},{"index":5,"sense_key":"bad%5:00:00:uncomfortable:00","synset_id":"bad.s.02"},{"index":6,"sense_key":"bad%5:00:00:stale:00","synset_id":"bad.s.03"},{"index":7,"sense_key":"regretful%3:00:00::","synset_id":"regretful.a.01"},{"index":8,"sense_key":"bad%5:00:00:invalid:00","synset_id":"bad.s.04"},{"index":9,"sense_key":"bad%5:00:00:inferior:02","synset_id":"bad.s.05"},{"index":10,"sense_key":"bad%5:00:00:nonstandard:03","synset_id":"bad.s.06"},{"index":11,"sense_key":"bad%5:00:02:unsound:00","synset_id":"bad.s.07"},{"index":12,"sense_key":"bad%5:00:00:unhealthy:00","synset_id":"bad.s.08"},{"index":13,"sense_key":"bad%5:00:00:harmful:00","synset_id":"bad.s.09"},{"index":14,"sense_key":"bad%5:00:00:evil:00","synset_id":"bad.s.10"},{"index":15,"sense_key":"bad%5:00:00:counterfeit:00","synset_id":"bad.s.11"},{"index":16,"sense_key":"bad%5:00:00:malfunctioning:00","synset_id":"bad.s.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"worse%3:00:00::","call_ids":["senseval2.d002.s059.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"worse%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1547,"cached_input_tokens":0,"output_tokens":71,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003804,"input_uncached_usd":0.003094,"input_cached_usd":0.0,"output_usd":0.00071,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9150483750272542}
{"item_id":"senseval2.d002.s059.t010","gold_sense_keys":["better%3:00:00::"],"candidates":[{"index":1,"sense_key":"better%3:00:00::","synset_id":"better.a.01"},{"index":2,"sense_key":"better%3:00:02::","synset_id":"better.a.02"},{"index":3,"sense_key":"better%5:00:00:advisable:00","synset_id":"better.s.01"},{"index":4,"sense_key":"better%5:00:00:major:01","synset_id":"better.s.02"},{"index":5,"sense_key":"good%3:00:01::","synset_id":"good.a.01"},{"index":6,"sense_key":"full%5:00:00:ample:00","synset_id":"full.s.04"},{"index":7,"sense_key":"good%3:00:02::","synset_id":"good.a.03"},{"index":8,"sense_key":"estimable%5:00:00:reputable:00","synset_id":"estimable.s.01"},{"index":9,"sense_key":"beneficial%5:00:00:advantageous:00","synset_id":"beneficial.s.01"},{"index":10,"sense_key":"good%5:00:00:nice:00","synset_id":"good.s.04"},{"index":11,"sense_key":"good%5:00:00:righteous:00","synset_id":"good.s.05"},{"index":12,"sense_key":"adept%5:00:00:skilled:00","synset_id":"adept.s.01"},{"index":13,"sense_key":"good%5:00:00:complete:00","synset_id":"good.s.07"},{"index":14,"sense_key":"dear%5:00:00:close:02","synset_id":"dear.s.02"},{"index":15,"sense_key":"dependable%5:00:00:sound:00","synset_id":"dependable.s.03"},{"index":16,"sense_key":"good%5:00:00:opportune:00","synset_id":"good.s.10"},{"index":17,"sense_key":"good%5:00:00:fortunate:00","synset_id":"good.s.11"},{"index":18,"sense_key":"effective%5:00:00:operative:00","synset_id":"effective.s.03"},{"index":19,"sense_key":"good%5:00:00:pleasing:00","synset_id":"good.s.13"},{"index":20,"sense_key":"good%5:00:00:intellectual:00","synset_id":"good.s.14"},{"index":21,"sense_key":"good%5:00:00:healthy:00","synset_id":"good.s.15"},{"index":22,"sense_key":"good%5:00:00:healthful:00","synset_id":"good.s.16"},{"index":23,"sense_key":"good%5:00:00:genuine:00","synset_id":"good.s.17"},{"index":24,"sense_key":"good%5:00:00:fresh:01","synset_id":"good.s.18"},{"index":25,"sense_key":"good%5:00:00:discriminating:00","synset_id":"good.s.19"},{"index":26,"sense_key":"well%3:00:01::","synset_id":"well.a.01"},{"index":27,"sense_key":"good%5:00:00:fortunate:00","synset_id":"good.s.11"},{"index":28,"sense_key":"well%5:00:00:advisable:00","synset_id":"well.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"better%3:00:00::","call_ids":["senseval2.d002.s059.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"better%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2250,"cached_input_tokens":0,"output_tokens":88,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.005379999999999999,"input_uncached_usd":0.0045,"input_cached_usd":0.0,"output_usd":0.00088,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6188239170005545}
{"item_id":"senseval2.d002.s060.t000","gold_sense_keys":["reason%1:09:01::","reason%1:10:00::","reason%1:10:01::","reason%1:16:00::"],"candidates":[{"index":1,"sense_key":"reason%1:16:00::","synset_id":"reason.n.01"},{"index":2,"sense_key":"reason%1:10:01::","synset_id":"reason.n.02"},{"index":3,"sense_key":"reason%1:09:00::","synset_id":"reason.n.03"},{"index":4,"sense_key":"reason%1:26:00::","synset_id":"rationality.n.01"},{"index":5,"sense_key":"reason%1:10:00::","synset_id":"cause.n.02"},{"index":6,"sense_key":"reason%1:09:01::","synset_id":"reason.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"reason%1:10:01::","call_ids":["senseval2.d002.s060.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"reason%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":904,"cached_input_tokens":0,"output_tokens":188,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0036880000000000003,"input_uncached_usd":0.001808,"input_cached_usd":0.0,"output_usd":0.0018800000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8873161659575999}
{"item_id":"senseval2.d002.s060.t001","gold_sense_keys":["simple%3:00:02::","simple%5:00:00:easy:01"],"candidates":[{"index":1,"sense_key":"simple%3:00:02::","synset_id":"simple.a.01"},{"index":2,"sense_key":"simple%5:00:00:easy:01","synset_id":"elementary.s.01"},{"index":3,"sense_key":"simple%5:00:02:plain:01","synset_id":"bare.s.04"},{"index":4,"sense_key":"simple%5:00:00:naive:00","synset_id":"childlike.s.02"},{"index":5,"sense_key":"simple%5:00:00:retarded:00","synset_id":"dim-witted.s.01"},{"index":6,"sense_key":"simple%3:00:01::","synset_id":"simple.a.06"},{"index":7,"sense_key":"simple%5:00:00:plain:01","synset_id":"simple.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"simple%3:00:02::","call_ids":["senseval2.d002.s060.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"simple%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":965,"cached_input_tokens":0,"output_tokens":251,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0044399999999999995,"input_uncached_usd":0.0019299999999999999,"input_cached_usd":0.0,"output_usd":0.00251,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.5268608330516145}
{"item_id":"senseval2.d002.s060.t002","gold_sense_keys":["education%1:04:00::","education%1:09:01::"],"candidates":[{"index":1,"sense_key":"education%1:04:00::","synset_id":"education.n.01"},{"index":2,"sense_key":"education%1:09:00::","synset_id":"education.n.02"},{"index":3,"sense_key":"education%1:09:01::","synset_id":"education.n.03"},{"index":4,"sense_key":"education%1:04:01::","synset_id":"education.n.04"},{"index":5,"sense_key":"education%1:07:01::","synset_id":"education.n.05"},{"index":6,"sense_key":"education%1:14:00::","synset_id":"department_of_education.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"education%1:04:00::","call_ids":["senseval2.d002.s060.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"education%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":881,"cached_input_tokens":0,"output_tokens":94,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002702,"input_uncached_usd":0.001762,"input_cached_usd":0.0,"output_usd":0.0009400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0042532079387456}
{"item_id":"senseval2.d002.s060.t005","gold_sense_keys":["influence%1:19:00::"],"candidates":[{"index":1,"sense_key":"influence%1:07:00::","synset_id":"influence.n.01"},{"index":2,"sense_key":"influence%1:04:00::","synset_id":"influence.n.02"},{"index":3,"sense_key":"influence%1:09:00::","synset_id":"influence.n.03"},{"index":4,"sense_key":"influence%1:19:00::","synset_id":"influence.n.04"},{"index":5,"sense_key":"influence%1:18:00::","synset_id":"influence.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"influence%1:19:00::","call_ids":["senseval2.d002.s060.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"influence%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":743,"cached_input_tokens":0,"output_tokens":153,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003016,"input_uncached_usd":0.001486,"input_cached_usd":0.0,"output_usd":0.0015300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0885209169937298}
{"item_id":"senseval2.d002.s060.t006","gold_sense_keys":["money%1:21:00::"],"candidates":[{"index":1,"sense_key":"money%1:21:00::","synset_id":"money.n.01"},{"index":2,"sense_key":"money%1:21:02::","synset_id":"money.n.02"},{"index":3,"sense_key":"money%1:21:01::","synset_id":"money.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"money%1:21:00::","call_ids":["senseval2.d002.s060.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"money%1:21:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":634,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001948,"input_uncached_usd":0.001268,"input_cached_usd":0.0,"output_usd":0.00068,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5343799160327762}
{"item_id":"senseval2.d002.s061.t000","gold_sense_keys":["decade%1:28:00::"],"candidates":[{"index":1,"sense_key":"decade%1:28:00::","synset_id":"decade.n.01"},{"index":2,"sense_key":"decade%1:23:00::","synset_id":"ten.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"decade%1:28:00::","call_ids":["senseval2.d002.s061.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"decade%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":619,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001358,"input_uncached_usd":0.001238,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4450225830078125}
{"item_id":"senseval2.d002.s061.t001","gold_sense_keys":["educational%3:01:00::"],"candidates":[{"index":1,"sense_key":"educational%3:01:00::","synset_id":"educational.a.01"},{"index":2,"sense_key":"educational%5:00:00:instructive:00","synset_id":"educational.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"educational%3:01:00::","call_ids":["senseval2.d002.s061.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"educational%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":591,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0018019999999999998,"input_uncached_usd":0.0011819999999999999,"input_cached_usd":0.0,"output_usd":0.00062,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5396451669512317}
{"item_id":"senseval2.d002.s061.t003","gold_sense_keys":["tell%2:32:00::"],"candidates":[{"index":1,"sense_key":"tell%2:32:04::","synset_id":"state.v.01"},{"index":2,"sense_key":"tell%2:32:00::","synset_id":"tell.v.02"},{"index":3,"sense_key":"tell%2:32:01::","synset_id":"tell.v.03"},{"index":4,"sense_key":"tell%2:32:02::","synset_id":"order.v.01"},{"index":5,"sense_key":"tell%2:31:00::","synset_id":"tell.v.05"},{"index":6,"sense_key":"tell%2:32:08::","synset_id":"assure.v.02"},{"index":7,"sense_key":"tell%2:32:03::","synset_id":"tell.v.07"},{"index":8,"sense_key":"tell%2:31:01::","synset_id":"distinguish.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"tell%2:32:00::","call_ids":["senseval2.d002.s061.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"tell%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":976,"cached_input_tokens":0,"output_tokens":161,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003562,"input_uncached_usd":0.001952,"input_cached_usd":0.0,"output_usd":0.00161,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8904840409522876}
{"item_id":"senseval2.d002.s061.t005","gold_sense_keys":["even%4:02:01::"],"candidates":[{"index":1,"sense_key":"even%4:02:00::","synset_id":"even.r.01"},{"index":2,"sense_key":"even%4:02:03::","synset_id":"even.r.02"},{"index":3,"sense_key":"even%4:02:01::","synset_id":"even.r.03"},{"index":4,"sense_key":"even%4:02:04::","synset_id":"even.r.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"even%4:02:00::","call_ids":["senseval2.d002.s061.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"even%4:02:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":738,"cached_input_tokens":0,"output_tokens":182,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0032960000000000003,"input_uncached_usd":0.0014759999999999999,"input_cached_usd":0.0,"output_usd":0.0018200000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":5.280208584037609}
{"item_id":"senseval2.d002.s061.t007","gold_sense_keys":["class%1:14:03::"],"candidates":[{"index":1,"sense_key":"class%1:14:00::","synset_id":"class.n.01"},{"index":2,"sense_key":"class%1:14:03::","synset_id":"class.n.02"},{"index":3,"sense_key":"class%1:14:01::","synset_id":"class.n.03"},{"index":4,"sense_key":"class%1:04:00::","synset_id":"course.n.01"},{"index":5,"sense_key":"class%1:14:05::","synset_id":"class.n.05"},{"index":6,"sense_key":"class%1:14:04::","synset_id":"class.n.06"},{"index":7,"sense_key":"class%1:14:02::","synset_id":"class.n.07"},{"index":8,"sense_key":"class%1:07:00::","synset_id":"class.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"class%1:14:03::","call_ids":["senseval2.d002.s061.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"class%1:14:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":983,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002506,"input_uncached_usd":0.001966,"input_cached_usd":0.0,"output_usd":0.00054,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.395634707994759}
{"item_id":"senseval2.d002.s061.t008","gold_sense_keys":["effect%1:19:00::"],"candidates":[{"index":1,"sense_key":"effect%1:19:00::","synset_id":"consequence.n.01"},{"index":2,"sense_key":"effect%1:07:00::","synset_id":"impression.n.02"},{"index":3,"sense_key":"effect%1:09:00::","synset_id":"effect.n.03"},{"index":4,"sense_key":"effect%1:10:00::","synset_id":"effect.n.04"},{"index":5,"sense_key":"effect%1:07:01::","synset_id":"effect.n.05"},{"index":6,"sense_key":"effect%1:26:00::","synset_id":"effect.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"effect%1:19:00::","call_ids":["senseval2.d002.s061.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"effect%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":889,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001898,"input_uncached_usd":0.0017779999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.407096582930535}
{"item_id":"senseval2.d002.s061.t009","gold_sense_keys":["academic%3:01:00::"],"candidates":[{"index":1,"sense_key":"academic%3:01:00::","synset_id":"academic.a.01"},{"index":2,"sense_key":"academic%5:00:00:theoretical:01","synset_id":"academic.s.01"},{"index":3,"sense_key":"academic%5:00:00:scholarly:00","synset_id":"academic.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"academic%3:01:00::","call_ids":["senseval2.d002.s061.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"academic%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":694,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001508,"input_uncached_usd":0.001388,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8249032919993624}
{"item_id":"senseval2.d002.s061.t010","gold_sense_keys":["performance%1:04:00::","performance%1:04:03::"],"candidates":[{"index":1,"sense_key":"performance%1:10:00::","synset_id":"performance.n.01"},{"index":2,"sense_key":"performance%1:04:01::","synset_id":"performance.n.02"},{"index":3,"sense_key":"performance%1:04:00::","synset_id":"performance.n.03"},{"index":4,"sense_key":"performance%1:04:03::","synset_id":"performance.n.04"},{"index":5,"sense_key":"performance%1:22:00::","synset_id":"operation.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"performance%1:04:00::","call_ids":["senseval2.d002.s061.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"performance%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":897,"cached_input_tokens":0,"output_tokens":110,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002894,"input_uncached_usd":0.001794,"input_cached_usd":0.0,"output_usd":0.0011,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.043697291985154}
{"item_id":"senseval2.d002.s061.t011","gold_sense_keys":["pupil%1:18:00::","pupil%1:18:01::"],"candidates":[{"index":1,"sense_key":"pupil%1:18:00::","synset_id":"student.n.01"},{"index":2,"sense_key":"pupil%1:08:00::","synset_id":"pupil.n.02"},{"index":3,"sense_key":"pupil%1:18:01::","synset_id":"schoolchild.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"pupil%1:18:01::","call_ids":["senseval2.d002.s061.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"pupil%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":666,"cached_input_tokens":0,"output_tokens":91,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002242,"input_uncached_usd":0.0013319999999999999,"input_cached_usd":0.0,"output_usd":0.0009100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7059860000154004}
{"item_id":"senseval2.d002.s061.t013","gold_sense_keys":["desirable%3:00:00::"],"candidates":[{"index":1,"sense_key":"desirable%3:00:00::","synset_id":"desirable.a.01"},{"index":2,"sense_key":"desirable%5:00:00:eligible:00","synset_id":"desirable.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"desirable%3:00:00::","call_ids":["senseval2.d002.s061.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"desirable%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":639,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001398,"input_uncached_usd":0.0012779999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5679190000519156}
{"item_id":"senseval2.d002.s061.t014","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":2,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":3,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"other%3:00:00::","call_ids":["senseval2.d002.s061.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":745,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00161,"input_uncached_usd":0.00149,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0210275419522077}
{"item_id":"senseval2.d002.s061.t015","gold_sense_keys":["reason%1:09:01::","reason%1:10:00::","reason%1:10:01::","reason%1:16:00::"],"candidates":[{"index":1,"sense_key":"reason%1:16:00::","synset_id":"reason.n.01"},{"index":2,"sense_key":"reason%1:10:01::","synset_id":"reason.n.02"},{"index":3,"sense_key":"reason%1:09:00::","synset_id":"reason.n.03"},{"index":4,"sense_key":"reason%1:26:00::","synset_id":"rationality.n.01"},{"index":5,"sense_key":"reason%1:10:00::","synset_id":"cause.n.02"},{"index":6,"sense_key":"reason%1:09:01::","synset_id":"reason.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"reason%1:10:00::","call_ids":["senseval2.d002.s061.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"reason%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":926,"cached_input_tokens":0,"output_tokens":224,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004092,"input_uncached_usd":0.001852,"input_cached_usd":0.0,"output_usd":0.0022400000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.9507069159299135}
{"item_id":"senseval2.d002.s062.t000","gold_sense_keys":["new%3:00:00::","new%5:00:00:other:00"],"candidates":[{"index":1,"sense_key":"new%3:00:00::","synset_id":"new.a.01"},{"index":2,"sense_key":"new%5:00:00:original:00","synset_id":"fresh.s.03"},{"index":3,"sense_key":"new%5:00:00:inexperienced:00","synset_id":"raw.s.11"},{"index":4,"sense_key":"new%5:00:00:unprecedented:00","synset_id":"new.s.03"},{"index":5,"sense_key":"new%5:00:00:other:00","synset_id":"new.s.04"},{"index":6,"sense_key":"new%3:00:09::","synset_id":"new.a.06"},{"index":7,"sense_key":"new%5:00:02:original:00","synset_id":"newfangled.s.01"},{"index":8,"sense_key":"new%5:00:02:late:01","synset_id":"new.s.06"},{"index":9,"sense_key":"new%5:00:01:late:01","synset_id":"modern.s.04"},{"index":10,"sense_key":"new%5:00:00:early:02","synset_id":"new.s.08"},{"index":11,"sense_key":"new%5:00:00:unaccustomed:00","synset_id":"new.s.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"new%3:00:00::","call_ids":["senseval2.d002.s062.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"new%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1262,"cached_input_tokens":0,"output_tokens":100,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003524,"input_uncached_usd":0.0025239999999999998,"input_cached_usd":0.0,"output_usd":0.001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7369683750439435}
{"item_id":"senseval2.d002.s062.t001","gold_sense_keys":["money%1:21:00::","money%1:21:02::"],"candidates":[{"index":1,"sense_key":"money%1:21:00::","synset_id":"money.n.01"},{"index":2,"sense_key":"money%1:21:02::","synset_id":"money.n.02"},{"index":3,"sense_key":"money%1:21:01::","synset_id":"money.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"money%1:21:00::","call_ids":["senseval2.d002.s062.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"money%1:21:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":716,"cached_input_tokens":0,"output_tokens":141,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002842,"input_uncached_usd":0.001432,"input_cached_usd":0.0,"output_usd":0.00141,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.4888696670532227}
{"item_id":"senseval2.d002.s062.t002","gold_sense_keys":["flow%2:38:01::"],"candidates":[{"index":1,"sense_key":"flow%2:38:01::","synset_id":"flow.v.01"},{"index":2,"sense_key":"flow%2:38:00::","synset_id":"run.v.06"},{"index":3,"sense_key":"flow%2:38:02::","synset_id":"flow.v.03"},{"index":4,"sense_key":"flow%2:42:00::","synset_id":"flow.v.04"},{"index":5,"sense_key":"flow%2:35:04::","synset_id":"hang.v.05"},{"index":6,"sense_key":"flow%2:30:00::","synset_id":"flow.v.06"},{"index":7,"sense_key":"flow%2:29:00::","synset_id":"menstruate.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"flow%2:38:01::","call_ids":["senseval2.d002.s062.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"flow%2:38:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":942,"cached_input_tokens":0,"output_tokens":107,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002954,"input_uncached_usd":0.0018839999999999998,"input_cached_usd":0.0,"output_usd":0.00107,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8879186250269413}
{"item_id":"senseval2.d002.s062.t007","gold_sense_keys":["pile%2:35:03::"],"candidates":[{"index":1,"sense_key":"pile%2:35:00::","synset_id":"stack.v.02"},{"index":2,"sense_key":"pile%2:38:00::","synset_id":"throng.v.01"},{"index":3,"sense_key":"pile%2:35:03::","synset_id":"pile.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"pile%2:35:03::","call_ids":["senseval2.d002.s062.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"pile%2:35:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":753,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001626,"input_uncached_usd":0.001506,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.7747380420332775}
{"item_id":"senseval2.d002.s062.t008","gold_sense_keys":["more%3:00:01::"],"candidates":[{"index":1,"sense_key":"more%3:00:01::","synset_id":"more.a.01"},{"index":2,"sense_key":"more%3:00:02::","synset_id":"more.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"more%3:00:01::","call_ids":["senseval2.d002.s062.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"more%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":712,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001954,"input_uncached_usd":0.001424,"input_cached_usd":0.0,"output_usd":0.0005300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.412137708044611}
{"item_id":"senseval2.d002.s062.t009","gold_sense_keys":["more%3:00:01::"],"candidates":[{"index":1,"sense_key":"more%3:00:01::","synset_id":"more.a.01"},{"index":2,"sense_key":"more%3:00:02::","synset_id":"more.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"more%3:00:01::","call_ids":["senseval2.d002.s062.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"more%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":712,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002014,"input_uncached_usd":0.001424,"input_cached_usd":0.0,"output_usd":0.00059,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3583014170872048}
{"item_id":"senseval2.d002.s062.t011","gold_sense_keys":["teacher%1:18:00::"],"candidates":[{"index":1,"sense_key":"teacher%1:18:00::","synset_id":"teacher.n.01"},{"index":2,"sense_key":"teacher%1:09:00::","synset_id":"teacher.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"teacher%1:18:00::","call_ids":["senseval2.d002.s062.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"teacher%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":673,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001466,"input_uncached_usd":0.001346,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.204988250043243}
{"item_id":"senseval2.d002.s063.t000","gold_sense_keys":["mystery%1:09:00::"],"candidates":[{"index":1,"sense_key":"mystery%1:09:00::","synset_id":"mystery.n.01"},{"index":2,"sense_key":"mystery%1:10:00::","synset_id":"mystery.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"mystery%1:09:00::","call_ids":["senseval2.d002.s063.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"mystery%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":668,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001456,"input_uncached_usd":0.001336,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.7563859580550343}
{"item_id":"senseval2.d002.s063.t002","gold_sense_keys":["fact%1:09:01::"],"candidates":[{"index":1,"sense_key":"fact%1:09:01::","synset_id":"fact.n.01"},{"index":2,"sense_key":"fact%1:10:01::","synset_id":"fact.n.02"},{"index":3,"sense_key":"fact%1:26:01::","synset_id":"fact.n.03"},{"index":4,"sense_key":"fact%1:09:02::","synset_id":"fact.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"fact%1:09:01::","call_ids":["senseval2.d002.s063.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"fact%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":766,"cached_input_tokens":0,"output_tokens":258,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004112,"input_uncached_usd":0.001532,"input_cached_usd":0.0,"output_usd":0.0025800000000000003,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.636548541020602}
{"item_id":"senseval2.d002.s063.t003","gold_sense_keys":["educational%3:01:00::"],"candidates":[{"index":1,"sense_key":"educational%3:01:00::","synset_id":"educational.a.01"},{"index":2,"sense_key":"educational%5:00:00:instructive:00","synset_id":"educational.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"educational%3:01:00::","call_ids":["senseval2.d002.s063.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"educational%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":605,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0019399999999999999,"input_uncached_usd":0.00121,"input_cached_usd":0.0,"output_usd":0.0007300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8497572500491515}
{"item_id":"senseval2.d002.s063.t004","gold_sense_keys":["expenditure%1:21:00::"],"candidates":[{"index":1,"sense_key":"expenditure%1:21:00::","synset_id":"outgo.n.01"},{"index":2,"sense_key":"expenditure%1:04:00::","synset_id":"expending.n.01"},{"index":3,"sense_key":"expenditure%1:04:01::","synset_id":"consumption.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"expenditure%1:21:00::","call_ids":["senseval2.d002.s063.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"expenditure%1:21:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":666,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002012,"input_uncached_usd":0.0013319999999999999,"input_cached_usd":0.0,"output_usd":0.00068,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3195228750118986}
{"item_id":"senseval2.d002.s063.t005","gold_sense_keys":["real%3:00:01::"],"candidates":[{"index":1,"sense_key":"real%3:00:00::","synset_id":"real.a.01"},{"index":2,"sense_key":"real%3:00:02::","synset_id":"real.a.02"},{"index":3,"sense_key":"real%5:00:00:serious:00","synset_id":"real.s.01"},{"index":4,"sense_key":"real%5:00:00:concrete:00","synset_id":"real.s.02"},{"index":5,"sense_key":"real%5:00:00:true:00","synset_id":"actual.s.02"},{"index":6,"sense_key":"real%3:00:01::","synset_id":"real.a.06"},{"index":7,"sense_key":"real%3:00:03::","synset_id":"substantial.a.03"},{"index":8,"sense_key":"real%5:00:00:tangible:02","synset_id":"real.s.04"},{"index":9,"sense_key":"real%5:00:02:realistic:00","synset_id":"veridical.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"real%3:00:01::","call_ids":["senseval2.d002.s063.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"real%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1168,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002456,"input_uncached_usd":0.002336,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1625146250007674}
{"item_id":"senseval2.d002.s063.t006","gold_sense_keys":["increase%2:30:00::"],"candidates":[{"index":1,"sense_key":"increase%2:30:00::","synset_id":"increase.v.01"},{"index":2,"sense_key":"increase%2:30:02::","synset_id":"increase.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"increase%2:30:00::","call_ids":["senseval2.d002.s063.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"increase%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":624,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001908,"input_uncached_usd":0.001248,"input_cached_usd":0.0,"output_usd":0.0006600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.383607167052105}
{"item_id":"senseval2.d002.s063.t007","gold_sense_keys":["sharply%4:02:03::"],"candidates":[{"index":1,"sense_key":"sharply%4:02:00::","synset_id":"aggressively.r.01"},{"index":2,"sense_key":"sharply%4:02:01::","synset_id":"sharply.r.02"},{"index":3,"sense_key":"sharply%4:02:02::","synset_id":"sharply.r.03"},{"index":4,"sense_key":"sharply%4:02:03::","synset_id":"precipitously.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"sharply%4:02:03::","call_ids":["senseval2.d002.s063.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"sharply%4:02:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":784,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001688,"input_uncached_usd":0.001568,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1763601249549538}
{"item_id":"senseval2.d002.s063.t008","gold_sense_keys":["past%3:00:00::"],"candidates":[{"index":1,"sense_key":"past%3:00:00::","synset_id":"past.a.01"},{"index":2,"sense_key":"past%5:00:00:outgoing:00","synset_id":"past.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"past%3:00:00::","call_ids":["senseval2.d002.s063.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"past%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":636,"cached_input_tokens":0,"output_tokens":88,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002152,"input_uncached_usd":0.001272,"input_cached_usd":0.0,"output_usd":0.00088,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.111783375032246}
{"item_id":"senseval2.d002.s063.t009","gold_sense_keys":["now%4:02:00::","now%4:02:04::"],"candidates":[{"index":1,"sense_key":"now%4:02:05::","synset_id":"now.r.01"},{"index":2,"sense_key":"now%4:02:00::","synset_id":"nowadays.r.01"},{"index":3,"sense_key":"now%4:02:02::","synset_id":"now.r.03"},{"index":4,"sense_key":"now%4:02:04::","synset_id":"now.r.04"},{"index":5,"sense_key":"now%4:02:01::","synset_id":"immediately.r.01"},{"index":6,"sense_key":"now%4:02:07::","synset_id":"now.r.06"},{"index":7,"sense_key":"now%4:02:06::","synset_id":"now.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"now%4:02:00::","call_ids":["senseval2.d002.s063.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"now%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":964,"cached_input_tokens":0,"output_tokens":138,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003308,"input_uncached_usd":0.0019279999999999998,"input_cached_usd":0.0,"output_usd":0.0013800000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7993772909976542}
{"item_id":"senseval2.d002.s063.t010","gold_sense_keys":["spend%2:40:00::"],"candidates":[{"index":1,"sense_key":"spend%2:42:00::","synset_id":"spend.v.01"},{"index":2,"sense_key":"spend%2:40:00::","synset_id":"spend.v.02"},{"index":3,"sense_key":"spend%2:40:01::","synset_id":"spend.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"spend%2:40:00::","call_ids":["senseval2.d002.s063.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"spend%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":664,"cached_input_tokens":0,"output_tokens":76,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002088,"input_uncached_usd":0.001328,"input_cached_usd":0.0,"output_usd":0.00076,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5372240409487858}
{"item_id":"senseval2.d002.s063.t011","gold_sense_keys":["more%4:02:01::"],"candidates":[{"index":1,"sense_key":"more%4:02:00::","synset_id":"more.r.01"},{"index":2,"sense_key":"more%4:02:01::","synset_id":"more.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"more%4:02:01::","call_ids":["senseval2.d002.s063.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"more%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":641,"cached_input_tokens":0,"output_tokens":106,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0023420000000000003,"input_uncached_usd":0.001282,"input_cached_usd":0.0,"output_usd":0.0010600000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.165154042071663}
{"item_id":"senseval2.d002.s063.t012","gold_sense_keys":["pupil%1:18:00::","pupil%1:18:01::"],"candidates":[{"index":1,"sense_key":"pupil%1:18:00::","synset_id":"student.n.01"},{"index":2,"sense_key":"pupil%1:08:00::","synset_id":"pupil.n.02"},{"index":3,"sense_key":"pupil%1:18:01::","synset_id":"schoolchild.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"pupil%1:18:01::","call_ids":["senseval2.d002.s063.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"pupil%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":680,"cached_input_tokens":0,"output_tokens":84,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0021999999999999997,"input_uncached_usd":0.0013599999999999999,"input_cached_usd":0.0,"output_usd":0.00084,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4464430829975754}
{"item_id":"senseval2.d002.s063.t013","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":2,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":3,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"other%3:00:00::","call_ids":["senseval2.d002.s063.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":759,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0016380000000000001,"input_uncached_usd":0.001518,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9924614580813795}
{"item_id":"senseval2.d002.s063.t014","gold_sense_keys":["country%1:14:00::"],"candidates":[{"index":1,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":2,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":3,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":4,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":5,"sense_key":"country%1:15:02::","synset_id":"area.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"country%1:14:00::","call_ids":["senseval2.d002.s063.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"country%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":876,"cached_input_tokens":0,"output_tokens":102,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002772,"input_uncached_usd":0.0017519999999999999,"input_cached_usd":0.0,"output_usd":0.00102,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.682246790966019}
{"item_id":"senseval2.d002.s063.t015","gold_sense_keys":["educational%3:01:00::"],"candidates":[{"index":1,"sense_key":"educational%3:01:00::","synset_id":"educational.a.01"},{"index":2,"sense_key":"educational%5:00:00:instructive:00","synset_id":"educational.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"educational%3:01:00::","call_ids":["senseval2.d002.s063.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"educational%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":605,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0019299999999999999,"input_uncached_usd":0.00121,"input_cached_usd":0.0,"output_usd":0.00072,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5202694999752566}
{"item_id":"senseval2.d002.s063.t016","gold_sense_keys":["performance%1:04:00::","performance%1:04:03::"],"candidates":[{"index":1,"sense_key":"performance%1:10:00::","synset_id":"performance.n.01"},{"index":2,"sense_key":"performance%1:04:01::","synset_id":"performance.n.02"},{"index":3,"sense_key":"performance%1:04:00::","synset_id":"performance.n.03"},{"index":4,"sense_key":"performance%1:04:03::","synset_id":"performance.n.04"},{"index":5,"sense_key":"performance%1:22:00::","synset_id":"operation.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"performance%1:04:00::","call_ids":["senseval2.d002.s063.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"performance%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":911,"cached_input_tokens":0,"output_tokens":129,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003112,"input_uncached_usd":0.0018219999999999998,"input_cached_usd":0.0,"output_usd":0.0012900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8093857080675662}
{"item_id":"senseval2.d002.s063.t017","gold_sense_keys":["decline%2:30:01::"],"candidates":[{"index":1,"sense_key":"decline%2:30:01::","synset_id":"worsen.v.01"},{"index":2,"sense_key":"decline%2:40:00::","synset_id":"refuse.v.02"},{"index":3,"sense_key":"decline%2:32:00::","synset_id":"refuse.v.01"},{"index":4,"sense_key":"decline%2:30:00::","synset_id":"decline.v.04"},{"index":5,"sense_key":"decline%2:38:00::","synset_id":"decline.v.05"},{"index":6,"sense_key":"decline%2:38:03::","synset_id":"decline.v.06"},{"index":7,"sense_key":"decline%2:32:01::","synset_id":"decline.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"decline%2:30:01::","call_ids":["senseval2.d002.s063.t017__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"decline%2:30:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":895,"cached_input_tokens":0,"output_tokens":100,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00279,"input_uncached_usd":0.00179,"input_cached_usd":0.0,"output_usd":0.001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0420350829372182}
{"item_id":"senseval2.d002.s064.t000","gold_sense_keys":["way%1:04:01::","way%1:07:01::"],"candidates":[{"index":1,"sense_key":"way%1:07:01::","synset_id":"manner.n.01"},{"index":2,"sense_key":"way%1:04:01::","synset_id":"means.n.01"},{"index":3,"sense_key":"way%1:15:00::","synset_id":"direction.n.01"},{"index":4,"sense_key":"way%1:26:00::","synset_id":"way.n.04"},{"index":5,"sense_key":"way%1:04:00::","synset_id":"way.n.05"},{"index":6,"sense_key":"way%1:06:00::","synset_id":"way.n.06"},{"index":7,"sense_key":"way%1:04:02::","synset_id":"way.n.07"},{"index":8,"sense_key":"way%1:23:00::","synset_id":"room.n.02"},{"index":9,"sense_key":"way%1:07:02::","synset_id":"way.n.09"},{"index":10,"sense_key":"way%1:09:00::","synset_id":"way.n.10"},{"index":11,"sense_key":"way%1:09:01::","synset_id":"way.n.11"},{"index":12,"sense_key":"way%1:21:00::","synset_id":"way.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"way%1:26:00::","call_ids":["senseval2.d002.s064.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"way%1:26:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1113,"cached_input_tokens":0,"output_tokens":115,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0033760000000000005,"input_uncached_usd":0.002226,"input_cached_usd":0.0,"output_usd":0.0011500000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7197574590099975}
{"item_id":"senseval2.d002.s065.t000","gold_sense_keys":["student%1:18:00::"],"candidates":[{"index":1,"sense_key":"student%1:18:00::","synset_id":"student.n.01"},{"index":2,"sense_key":"student%1:18:01::","synset_id":"scholar.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"student%1:18:00::","call_ids":["senseval2.d002.s065.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"student%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":625,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0013700000000000001,"input_uncached_usd":0.00125,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.244525708956644}
{"item_id":"senseval2.d002.s065.t001","gold_sense_keys":["educational%3:01:00::"],"candidates":[{"index":1,"sense_key":"educational%3:01:00::","synset_id":"educational.a.01"},{"index":2,"sense_key":"educational%5:00:00:instructive:00","synset_id":"educational.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"educational%3:01:00::","call_ids":["senseval2.d002.s065.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"educational%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":583,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001846,"input_uncached_usd":0.001166,"input_cached_usd":0.0,"output_usd":0.00068,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3932984579587355}
{"item_id":"senseval2.d002.s065.t002","gold_sense_keys":["academic%3:01:00::"],"candidates":[{"index":1,"sense_key":"academic%3:01:00::","synset_id":"academic.a.01"},{"index":2,"sense_key":"academic%5:00:00:theoretical:01","synset_id":"academic.s.01"},{"index":3,"sense_key":"academic%5:00:00:scholarly:00","synset_id":"academic.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"academic%3:01:00::","call_ids":["senseval2.d002.s065.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"academic%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":686,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001492,"input_uncached_usd":0.001372,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.09860537503846}
{"item_id":"senseval2.d002.s065.t003","gold_sense_keys":["potential%1:26:00::"],"candidates":[{"index":1,"sense_key":"potential%1:26:00::","synset_id":"potential.n.01"},{"index":2,"sense_key":"potential%1:19:00::","synset_id":"electric_potential.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"potential%1:26:00::","call_ids":["senseval2.d002.s065.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"potential%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":620,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00136,"input_uncached_usd":0.00124,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5622799170669168}
{"item_id":"senseval2.d002.s065.t004","gold_sense_keys":["allow%2:42:00::"],"candidates":[{"index":1,"sense_key":"allow%2:41:00::","synset_id":"let.v.01"},{"index":2,"sense_key":"allow%2:32:00::","synset_id":"permit.v.01"},{"index":3,"sense_key":"allow%2:40:00::","synset_id":"allow.v.03"},{"index":4,"sense_key":"allow%2:31:01::","synset_id":"allow.v.04"},{"index":5,"sense_key":"allow%2:42:00::","synset_id":"leave.v.06"},{"index":6,"sense_key":"allow%2:31:00::","synset_id":"allow.v.06"},{"index":7,"sense_key":"allow%2:42:07::","synset_id":"admit.v.05"},{"index":8,"sense_key":"allow%2:41:01::","synset_id":"give_up.v.11"},{"index":9,"sense_key":"allow%2:40:01::","synset_id":"allow.v.09"},{"index":10,"sense_key":"allow%2:32:06::","synset_id":"allow.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"allow%2:41:00::","call_ids":["senseval2.d002.s065.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"allow%2:41:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1230,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004540000000000001,"input_uncached_usd":0.00246,"input_cached_usd":0.0,"output_usd":0.0020800000000000003,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.174095542053692}
{"item_id":"senseval2.d002.s066.t000","gold_sense_keys":["student%1:18:00::"],"candidates":[{"index":1,"sense_key":"student%1:18:00::","synset_id":"student.n.01"},{"index":2,"sense_key":"student%1:18:01::","synset_id":"scholar.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"student%1:18:00::","call_ids":["senseval2.d002.s066.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"student%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":670,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0014600000000000001,"input_uncached_usd":0.00134,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.010557708912529}
{"item_id":"senseval2.d002.s066.t001","gold_sense_keys":["permit%2:32:00::"],"candidates":[{"index":1,"sense_key":"permit%2:32:00::","synset_id":"permit.v.01"},{"index":2,"sense_key":"permit%2:41:00::","synset_id":"let.v.01"},{"index":3,"sense_key":"permit%2:32:06::","synset_id":"allow.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"permit%2:32:00::","call_ids":["senseval2.d002.s066.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"permit%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":790,"cached_input_tokens":0,"output_tokens":161,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00319,"input_uncached_usd":0.00158,"input_cached_usd":0.0,"output_usd":0.00161,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.4026233749464154}
{"item_id":"senseval2.d002.s066.t002","gold_sense_keys":["graduate%2:40:01::"],"candidates":[{"index":1,"sense_key":"graduate%2:40:00::","synset_id":"graduate.v.01"},{"index":2,"sense_key":"graduate%2:40:01::","synset_id":"graduate.v.02"},{"index":3,"sense_key":"graduate%2:30:00::","synset_id":"calibrate.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"graduate%2:40:01::","call_ids":["senseval2.d002.s066.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"graduate%2:40:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":735,"cached_input_tokens":0,"output_tokens":173,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0032,"input_uncached_usd":0.00147,"input_cached_usd":0.0,"output_usd":0.0017300000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.98570425005164}
{"item_id":"senseval2.d002.s066.t004","gold_sense_keys":["master%2:31:01::"],"candidates":[{"index":1,"sense_key":"master%2:31:01::","synset_id":"master.v.01"},{"index":2,"sense_key":"master%2:33:00::","synset_id":"overcome.v.02"},{"index":3,"sense_key":"master%2:41:00::","synset_id":"dominate.v.03"},{"index":4,"sense_key":"master%2:31:00::","synset_id":"master.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"master%2:31:01::","call_ids":["senseval2.d002.s066.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"master%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":797,"cached_input_tokens":0,"output_tokens":126,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0028539999999999998,"input_uncached_usd":0.001594,"input_cached_usd":0.0,"output_usd":0.00126,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8548693750053644}
{"item_id":"senseval2.d002.s066.t005","gold_sense_keys":["level%1:07:00::"],"candidates":[{"index":1,"sense_key":"level%1:07:00::","synset_id":"degree.n.01"},{"index":2,"sense_key":"level%1:26:01::","synset_id":"grade.n.02"},{"index":3,"sense_key":"level%1:26:00::","synset_id":"degree.n.02"},{"index":4,"sense_key":"level%1:07:01::","synset_id":"level.n.04"},{"index":5,"sense_key":"level%1:06:00::","synset_id":"level.n.05"},{"index":6,"sense_key":"level%1:06:03::","synset_id":"horizontal_surface.n.01"},{"index":7,"sense_key":"level%1:09:00::","synset_id":"level.n.07"},{"index":8,"sense_key":"level%1:06:01::","synset_id":"floor.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"level%1:07:00::","call_ids":["senseval2.d002.s066.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"level%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1071,"cached_input_tokens":0,"output_tokens":121,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003352,"input_uncached_usd":0.002142,"input_cached_usd":0.0,"output_usd":0.0012100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7962772089522332}
{"item_id":"senseval2.d002.s066.t006","gold_sense_keys":["prevail%2:42:01::"],"candidates":[{"index":1,"sense_key":"prevail%2:42:02::","synset_id":"predominate.v.01"},{"index":2,"sense_key":"prevail%2:42:01::","synset_id":"prevail.v.02"},{"index":3,"sense_key":"prevail%2:42:00::","synset_id":"prevail.v.03"},{"index":4,"sense_key":"prevail%2:33:00::","synset_id":"prevail.v.04"},{"index":5,"sense_key":"prevail%2:32:02::","synset_id":"prevail.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"prevail%2:42:01::","call_ids":["senseval2.d002.s066.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"prevail%2:42:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":866,"cached_input_tokens":0,"output_tokens":108,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002812,"input_uncached_usd":0.0017319999999999998,"input_cached_usd":0.0,"output_usd":0.00108,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.148126458050683}
{"item_id":"senseval2.d002.s066.t007","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["senseval2.d002.s066.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":808,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0017360000000000001,"input_uncached_usd":0.001616,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1100777910323814}
{"item_id":"senseval2.d002.s067.t000","gold_sense_keys":["mean%2:42:00::"],"candidates":[{"index":1,"sense_key":"mean%2:32:01::","synset_id":"mean.v.01"},{"index":2,"sense_key":"mean%2:42:00::","synset_id":"entail.v.01"},{"index":3,"sense_key":"mean%2:32:03::","synset_id":"mean.v.03"},{"index":4,"sense_key":"mean%2:31:00::","synset_id":"intend.v.01"},{"index":5,"sense_key":"mean%2:42:03::","synset_id":"mean.v.05"},{"index":6,"sense_key":"mean%2:31:01::","synset_id":"think_of.v.04"},{"index":7,"sense_key":"mean%2:31:05::","synset_id":"mean.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"mean%2:42:00::","call_ids":["senseval2.d002.s067.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"mean%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":947,"cached_input_tokens":0,"output_tokens":84,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002734,"input_uncached_usd":0.0018939999999999999,"input_cached_usd":0.0,"output_usd":0.00084,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4285197081044316}
{"item_id":"senseval2.d002.s067.t001","gold_sense_keys":["main%5:00:00:important:00"],"candidates":[{"index":1,"sense_key":"main%5:00:00:important:00","synset_id":"chief.s.01"},{"index":2,"sense_key":"main%3:00:02::","synset_id":"independent.a.03"},{"index":3,"sense_key":"main%5:00:00:intense:00","synset_id":"main.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"main%5:00:00:important:00","call_ids":["senseval2.d002.s067.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"main%5:00:00:important:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":708,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001536,"input_uncached_usd":0.001416,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.397569458000362}
{"item_id":"senseval2.d002.s067.t002","gold_sense_keys":["purpose%1:07:00::","purpose%1:09:00::"],"candidates":[{"index":1,"sense_key":"purpose%1:09:00::","synset_id":"purpose.n.01"},{"index":2,"sense_key":"purpose%1:07:00::","synset_id":"function.n.02"},{"index":3,"sense_key":"purpose%1:07:01::","synset_id":"determination.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"purpose%1:09:00::","call_ids":["senseval2.d002.s067.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"purpose%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":748,"cached_input_tokens":0,"output_tokens":125,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002746,"input_uncached_usd":0.001496,"input_cached_usd":0.0,"output_usd":0.00125,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5093655000673607}
{"item_id":"senseval2.d002.s067.t004","gold_sense_keys":["permit%2:41:00::"],"candidates":[{"index":1,"sense_key":"permit%2:32:00::","synset_id":"permit.v.01"},{"index":2,"sense_key":"permit%2:41:00::","synset_id":"let.v.01"},{"index":3,"sense_key":"permit%2:32:06::","synset_id":"allow.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"permit%2:41:00::","call_ids":["senseval2.d002.s067.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"permit%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":755,"cached_input_tokens":0,"output_tokens":110,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00261,"input_uncached_usd":0.0015099999999999998,"input_cached_usd":0.0,"output_usd":0.0011,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.651109875063412}
{"item_id":"senseval2.d002.s067.t007","gold_sense_keys":["flourish%2:40:00::"],"candidates":[{"index":1,"sense_key":"flourish%2:30:01::","synset_id":"boom.v.05"},{"index":2,"sense_key":"flourish%2:40:00::","synset_id":"thrive.v.02"},{"index":3,"sense_key":"flourish%2:35:00::","synset_id":"brandish.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"flourish%2:40:00::","call_ids":["senseval2.d002.s067.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"flourish%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":715,"cached_input_tokens":0,"output_tokens":119,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00262,"input_uncached_usd":0.0014299999999999998,"input_cached_usd":0.0,"output_usd":0.00119,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7069600829854608}
{"item_id":"senseval2.d002.s067.t008","gold_sense_keys":["clearly%4:02:00::"],"candidates":[{"index":1,"sense_key":"clearly%4:02:00::","synset_id":"clearly.r.01"},{"index":2,"sense_key":"clearly%4:02:01::","synset_id":"intelligibly.r.01"},{"index":3,"sense_key":"clearly%4:02:02::","synset_id":"distinctly.r.01"},{"index":4,"sense_key":"clearly%4:02:03::","synset_id":"clearly.r.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"clearly%4:02:00::","call_ids":["senseval2.d002.s067.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"clearly%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":780,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00233,"input_uncached_usd":0.00156,"input_cached_usd":0.0,"output_usd":0.0007700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5104969170643017}
{"item_id":"senseval2.d002.s067.t009","gold_sense_keys":["desirable%3:00:00::"],"candidates":[{"index":1,"sense_key":"desirable%3:00:00::","synset_id":"desirable.a.01"},{"index":2,"sense_key":"desirable%5:00:00:eligible:00","synset_id":"desirable.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"desirable%3:00:00::","call_ids":["senseval2.d002.s067.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"desirable%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":643,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0014060000000000001,"input_uncached_usd":0.001286,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8421082079876214}
{"item_id":"senseval2.d002.s067.t010","gold_sense_keys":["less%4:02:00::"],"candidates":[{"index":1,"sense_key":"less%4:02:00::","synset_id":"less.r.01"},{"index":2,"sense_key":"less%4:02:01::","synset_id":"less.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"less%4:02:00::","call_ids":["senseval2.d002.s067.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"less%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":622,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001804,"input_uncached_usd":0.0012439999999999999,"input_cached_usd":0.0,"output_usd":0.0005600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3493299590190873}
{"item_id":"senseval2.d002.s067.t012","gold_sense_keys":["get%2:40:00::"],"candidates":[{"index":1,"sense_key":"get%2:40:00::","synset_id":"get.v.01"},{"index":2,"sense_key":"get%2:30:00::","synset_id":"become.v.01"},{"index":3,"sense_key":"get%2:30:02::","synset_id":"get.v.03"},{"index":4,"sense_key":"get%2:30:03::","synset_id":"receive.v.02"},{"index":5,"sense_key":"get%2:38:00::","synset_id":"arrive.v.01"},{"index":6,"sense_key":"get%2:35:03::","synset_id":"bring.v.04"},{"index":7,"sense_key":"get%2:39:00::","synset_id":"experience.v.03"},{"index":8,"sense_key":"get%2:33:00::","synset_id":"pay_back.v.02"},{"index":9,"sense_key":"get%2:40:01::","synset_id":"have.v.17"},{"index":10,"sense_key":"get%2:32:00::","synset_id":"induce.v.02"},{"index":11,"sense_key":"get%2:35:09::","synset_id":"get.v.11"},{"index":12,"sense_key":"get%2:29:01::","synset_id":"grow.v.08"},{"index":13,"sense_key":"get%2:29:00::","synset_id":"contract.v.04"},{"index":14,"sense_key":"get%2:32:03::","synset_id":"get.v.14"},{"index":15,"sense_key":"get%2:30:01::","synset_id":"make.v.02"},{"index":16,"sense_key":"get%2:32:08::","synset_id":"drive.v.11"},{"index":17,"sense_key":"get%2:31:09::","synset_id":"catch.v.18"},{"index":18,"sense_key":"get%2:35:00::","synset_id":"catch.v.07"},{"index":19,"sense_key":"get%2:35:10::","synset_id":"get.v.19"},{"index":20,"sense_key":"get%2:31:03::","synset_id":"get.v.20"},{"index":21,"sense_key":"get%2:40:14::","synset_id":"get.v.21"},{"index":22,"sense_key":"get%2:40:13::","synset_id":"get.v.22"},{"index":23,"sense_key":"get%2:39:13::","synset_id":"catch.v.21"},{"index":24,"sense_key":"get%2:39:09::","synset_id":"catch.v.22"},{"index":25,"sense_key":"get%2:39:14::","synset_id":"get.v.25"},{"index":26,"sense_key":"get%2:38:15::","synset_id":"scram.v.01"},{"index":27,"sense_key":"get%2:38:12::","synset_id":"get.v.27"},{"index":28,"sense_key":"get%2:37:12::","synset_id":"get.v.28"},{"index":29,"sense_key":"get%2:37:11::","synset_id":"get.v.29"},{"index":30,"sense_key":"get%2:36:14::","synset_id":"catch.v.24"},{"index":31,"sense_key":"get%2:36:00::","synset_id":"draw.v.15"},{"index":32,"sense_key":"get%2:35:12::","synset_id":"get.v.32"},{"index":33,"sense_key":"get%2:31:10::","synset_id":"perplex.v.01"},{"index":34,"sense_key":"get%2:30:12::","synset_id":"get_down.v.07"},{"index":35,"sense_key":"get%2:29:11::","synset_id":"suffer.v.02"},{"index":36,"sense_key":"get%2:29:08::","synset_id":"beget.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"get%2:40:00::","call_ids":["senseval2.d002.s067.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"get%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2857,"cached_input_tokens":0,"output_tokens":152,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.007233999999999999,"input_uncached_usd":0.0057139999999999995,"input_cached_usd":0.0,"output_usd":0.00152,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.4002346670022234}
{"item_id":"senseval2.d002.s067.t013","gold_sense_keys":["necessary%3:00:00::"],"candidates":[{"index":1,"sense_key":"necessary%3:00:00::","synset_id":"necessary.a.01"},{"index":2,"sense_key":"necessary%5:00:00:inevitable:00","synset_id":"necessary.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"necessary%3:00:00::","call_ids":["senseval2.d002.s067.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"necessary%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":606,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001332,"input_uncached_usd":0.001212,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0365431669633836}
{"item_id":"senseval2.d002.s067.t014","gold_sense_keys":["study%1:09:03::"],"candidates":[{"index":1,"sense_key":"study%1:04:00::","synset_id":"survey.n.01"},{"index":2,"sense_key":"study%1:09:03::","synset_id":"study.n.02"},{"index":3,"sense_key":"study%1:10:00::","synset_id":"report.n.01"},{"index":4,"sense_key":"study%1:09:04::","synset_id":"study.n.04"},{"index":5,"sense_key":"study%1:06:00::","synset_id":"study.n.05"},{"index":6,"sense_key":"study%1:09:02::","synset_id":"discipline.n.01"},{"index":7,"sense_key":"study%1:06:01::","synset_id":"sketch.n.01"},{"index":8,"sense_key":"study%1:09:01::","synset_id":"cogitation.n.02"},{"index":9,"sense_key":"study%1:18:00::","synset_id":"study.n.09"},{"index":10,"sense_key":"study%1:10:01::","synset_id":"study.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"study%1:09:03::","call_ids":["senseval2.d002.s067.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"study%1:09:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1102,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0023239999999999997,"input_uncached_usd":0.002204,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9932383330306038}
{"item_id":"senseval2.d002.s067.t015","gold_sense_keys":["enter%2:33:00::","enter%2:38:00::","enter%2:42:00::"],"candidates":[{"index":1,"sense_key":"enter%2:38:00::","synset_id":"enter.v.01"},{"index":2,"sense_key":"enter%2:33:00::","synset_id":"enter.v.02"},{"index":3,"sense_key":"enter%2:41:06::","synset_id":"enroll.v.01"},{"index":4,"sense_key":"enter%2:42:00::","synset_id":"figure.v.02"},{"index":5,"sense_key":"enter%2:32:04::","synset_id":"record.v.01"},{"index":6,"sense_key":"enter%2:36:00::","synset_id":"enter.v.06"},{"index":7,"sense_key":"enter%2:41:00::","synset_id":"accede.v.02"},{"index":8,"sense_key":"enter%2:35:03::","synset_id":"insert.v.01"},{"index":9,"sense_key":"enter%2:30:10::","synset_id":"embark.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"enter%2:30:10::","call_ids":["senseval2.d002.s067.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"enter%2:30:10::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1007,"cached_input_tokens":0,"output_tokens":143,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003444,"input_uncached_usd":0.0020139999999999997,"input_cached_usd":0.0,"output_usd":0.00143,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.415066000074148}
{"item_id":"senseval2.d002.s067.t017","gold_sense_keys":["world%1:14:01::"],"candidates":[{"index":1,"sense_key":"world%1:17:01::","synset_id":"universe.n.01"},{"index":2,"sense_key":"world%1:14:01::","synset_id":"world.n.02"},{"index":3,"sense_key":"world%1:09:00::","synset_id":"world.n.03"},{"index":4,"sense_key":"world%1:17:00::","synset_id":"earth.n.01"},{"index":5,"sense_key":"world%1:14:02::","synset_id":"populace.n.01"},{"index":6,"sense_key":"world%1:17:02::","synset_id":"world.n.06"},{"index":7,"sense_key":"world%1:09:01::","synset_id":"worldly_concern.n.01"},{"index":8,"sense_key":"world%1:05:00::","synset_id":"world.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"world%1:14:01::","call_ids":["senseval2.d002.s067.t017__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"world%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1068,"cached_input_tokens":0,"output_tokens":117,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003306,"input_uncached_usd":0.002136,"input_cached_usd":0.0,"output_usd":0.00117,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8256901669083163}
{"item_id":"senseval2.d002.s067.t018","gold_sense_keys":["work%1:04:01::"],"candidates":[{"index":1,"sense_key":"work%1:04:00::","synset_id":"work.n.01"},{"index":2,"sense_key":"work%1:06:00::","synset_id":"work.n.02"},{"index":3,"sense_key":"work%1:04:01::","synset_id":"employment.n.02"},{"index":4,"sense_key":"work%1:09:00::","synset_id":"study.n.02"},{"index":5,"sense_key":"work%1:19:00::","synset_id":"work.n.05"},{"index":6,"sense_key":"work%1:06:01::","synset_id":"workplace.n.01"},{"index":7,"sense_key":"work%1:06:02::","synset_id":"oeuvre.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"work%1:04:01::","call_ids":["senseval2.d002.s067.t018__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"work%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1016,"cached_input_tokens":0,"output_tokens":39,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002422,"input_uncached_usd":0.002032,"input_cached_usd":0.0,"output_usd":0.00039000000000000005,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.295111957937479}
{"item_id":"senseval2.d002.s068.t000","gold_sense_keys":["notion%1:09:02::"],"candidates":[{"index":1,"sense_key":"notion%1:09:02::","synset_id":"impression.n.01"},{"index":2,"sense_key":"notion%1:09:00::","synset_id":"notion.n.02"},{"index":3,"sense_key":"notion%1:09:01::","synset_id":"notion.n.03"},{"index":4,"sense_key":"notion%1:06:00::","synset_id":"notion.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"notion%1:09:02::","call_ids":["senseval2.d002.s068.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"notion%1:09:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":764,"cached_input_tokens":0,"output_tokens":134,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002868,"input_uncached_usd":0.0015279999999999998,"input_cached_usd":0.0,"output_usd":0.00134,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.043078540940769}
{"item_id":"senseval2.d002.s069.t000","gold_sense_keys":["purpose%1:07:00::","purpose%1:09:00::"],"candidates":[{"index":1,"sense_key":"purpose%1:09:00::","synset_id":"purpose.n.01"},{"index":2,"sense_key":"purpose%1:07:00::","synset_id":"function.n.02"},{"index":3,"sense_key":"purpose%1:07:01::","synset_id":"determination.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"purpose%1:09:00::","call_ids":["senseval2.d002.s069.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"purpose%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":682,"cached_input_tokens":0,"output_tokens":220,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0035640000000000003,"input_uncached_usd":0.001364,"input_cached_usd":0.0,"output_usd":0.0022,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.5390657080570236}
{"item_id":"senseval2.d002.s069.t001","gold_sense_keys":["education%1:04:00::"],"candidates":[{"index":1,"sense_key":"education%1:04:00::","synset_id":"education.n.01"},{"index":2,"sense_key":"education%1:09:00::","synset_id":"education.n.02"},{"index":3,"sense_key":"education%1:09:01::","synset_id":"education.n.03"},{"index":4,"sense_key":"education%1:04:01::","synset_id":"education.n.04"},{"index":5,"sense_key":"education%1:07:01::","synset_id":"education.n.05"},{"index":6,"sense_key":"education%1:14:00::","synset_id":"department_of_education.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"education%1:09:01::","call_ids":["senseval2.d002.s069.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"education%1:09:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":839,"cached_input_tokens":0,"output_tokens":142,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003098,"input_uncached_usd":0.001678,"input_cached_usd":0.0,"output_usd":0.00142,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8329032500041649}
{"item_id":"senseval2.d002.s069.t002","gold_sense_keys":["encourage%2:32:00::","encourage%2:37:00::"],"candidates":[{"index":1,"sense_key":"encourage%2:41:00::","synset_id":"promote.v.01"},{"index":2,"sense_key":"encourage%2:37:00::","synset_id":"encourage.v.02"},{"index":3,"sense_key":"encourage%2:32:00::","synset_id":"encourage.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"encourage%2:32:00::","call_ids":["senseval2.d002.s069.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"encourage%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":604,"cached_input_tokens":0,"output_tokens":168,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002888,"input_uncached_usd":0.0012079999999999999,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.3866633340949193}
{"item_id":"senseval2.d002.s069.t003","gold_sense_keys":["young%3:00:00::"],"candidates":[{"index":1,"sense_key":"young%3:00:00::","synset_id":"young.a.01"},{"index":2,"sense_key":"young%5:00:00:early:02","synset_id":"new.s.08"},{"index":3,"sense_key":"young%5:00:01:young:00","synset_id":"youthful.s.01"},{"index":4,"sense_key":"young%5:00:00:new:00","synset_id":"young.s.03"},{"index":5,"sense_key":"young%5:00:00:inexperienced:00","synset_id":"unseasoned.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"young%3:00:00::","call_ids":["senseval2.d002.s069.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"young%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":754,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001628,"input_uncached_usd":0.001508,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0292942089727148}
{"item_id":"senseval2.d002.s069.t004","gold_sense_keys":["man%1:18:00::"],"candidates":[{"index":1,"sense_key":"man%1:18:00::","synset_id":"man.n.01"},{"index":2,"sense_key":"man%1:18:03::","synset_id":"serviceman.n.01"},{"index":3,"sense_key":"man%1:18:04::","synset_id":"man.n.03"},{"index":4,"sense_key":"man%1:05:01::","synset_id":"homo.n.02"},{"index":5,"sense_key":"man%1:18:08::","synset_id":"man.n.05"},{"index":6,"sense_key":"man%1:18:06::","synset_id":"man.n.06"},{"index":7,"sense_key":"man%1:18:05::","synset_id":"valet.n.01"},{"index":8,"sense_key":"man%1:18:07::","synset_id":"man.n.08"},{"index":9,"sense_key":"man%1:15:00::","synset_id":"man.n.09"},{"index":10,"sense_key":"man%1:06:00::","synset_id":"man.n.10"},{"index":11,"sense_key":"man%1:05:00::","synset_id":"world.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"man%1:18:00::","call_ids":["senseval2.d002.s069.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"man%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1189,"cached_input_tokens":0,"output_tokens":44,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0028179999999999998,"input_uncached_usd":0.002378,"input_cached_usd":0.0,"output_usd":0.00044,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2490623749326915}
{"item_id":"senseval2.d002.s069.t005","gold_sense_keys":["woman%1:18:00::"],"candidates":[{"index":1,"sense_key":"woman%1:18:00::","synset_id":"woman.n.01"},{"index":2,"sense_key":"woman%1:18:01::","synset_id":"woman.n.02"},{"index":3,"sense_key":"woman%1:18:02::","synset_id":"charwoman.n.01"},{"index":4,"sense_key":"woman%1:14:00::","synset_id":"womanhood.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"woman%1:18:00::","call_ids":["senseval2.d002.s069.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"woman%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":731,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001582,"input_uncached_usd":0.001462,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.379191791987978}
{"item_id":"senseval2.d002.s069.t006","gold_sense_keys":["realize%2:36:00::"],"candidates":[{"index":1,"sense_key":"realize%2:31:01::","synset_id":"recognize.v.02"},{"index":2,"sense_key":"realize%2:31:00::","synset_id":"understand.v.02"},{"index":3,"sense_key":"realize%2:36:00::","synset_id":"realize.v.03"},{"index":4,"sense_key":"realize%2:40:01::","synset_id":"gain.v.08"},{"index":5,"sense_key":"realize%2:40:00::","synset_id":"realize.v.05"},{"index":6,"sense_key":"realize%2:36:07::","synset_id":"realize.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"realize%2:36:00::","call_ids":["senseval2.d002.s069.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"realize%2:36:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":846,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001812,"input_uncached_usd":0.001692,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0872092089848593}
{"item_id":"senseval2.d002.s069.t007","gold_sense_keys":["full%5:00:00:complete:00","full%5:00:01:whole:00"],"candidates":[{"index":1,"sense_key":"full%3:00:00::","synset_id":"full.a.01"},{"index":2,"sense_key":"full%5:00:01:whole:00","synset_id":"entire.s.01"},{"index":3,"sense_key":"full%5:00:00:complete:00","synset_id":"full.s.02"},{"index":4,"sense_key":"full%5:00:00:nourished:00","synset_id":"full.s.03"},{"index":5,"sense_key":"full%3:00:01::","synset_id":"full.a.05"},{"index":6,"sense_key":"full%5:00:00:ample:00","synset_id":"full.s.04"},{"index":7,"sense_key":"full%5:00:00:high:02","synset_id":"broad.s.04"},{"index":8,"sense_key":"full%5:00:02:ample:00","synset_id":"wide.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"full%5:00:01:whole:00","call_ids":["senseval2.d002.s069.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"full%5:00:01:whole:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":907,"cached_input_tokens":0,"output_tokens":125,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003064,"input_uncached_usd":0.0018139999999999999,"input_cached_usd":0.0,"output_usd":0.00125,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8795441670808941}
{"item_id":"senseval2.d002.s069.t008","gold_sense_keys":["academic%3:01:00::"],"candidates":[{"index":1,"sense_key":"academic%3:01:00::","synset_id":"academic.a.01"},{"index":2,"sense_key":"academic%5:00:00:theoretical:01","synset_id":"academic.s.01"},{"index":3,"sense_key":"academic%5:00:00:scholarly:00","synset_id":"academic.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"academic%3:01:00::","call_ids":["senseval2.d002.s069.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"academic%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":632,"cached_input_tokens":0,"output_tokens":94,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0022040000000000002,"input_uncached_usd":0.001264,"input_cached_usd":0.0,"output_usd":0.0009400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0208092079265043}
{"item_id":"senseval2.d002.s069.t009","gold_sense_keys":["potential%1:26:00::"],"candidates":[{"index":1,"sense_key":"potential%1:26:00::","synset_id":"potential.n.01"},{"index":2,"sense_key":"potential%1:19:00::","synset_id":"electric_potential.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"potential%1:26:00::","call_ids":["senseval2.d002.s069.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"potential%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":566,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001252,"input_uncached_usd":0.001132,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.989044834044762}
{"item_id":"senseval2.d002.s070.t000","gold_sense_keys":["actually%4:02:01::"],"candidates":[{"index":1,"sense_key":"actually%4:02:01::","synset_id":"actually.r.01"},{"index":2,"sense_key":"actually%4:02:00::","synset_id":"actually.r.02"},{"index":3,"sense_key":"actually%4:02:03::","synset_id":"actually.r.03"},{"index":4,"sense_key":"actually%4:02:02::","synset_id":"actually.r.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"actually%4:02:01::","call_ids":["senseval2.d002.s070.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"actually%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":743,"cached_input_tokens":0,"output_tokens":121,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002696,"input_uncached_usd":0.001486,"input_cached_usd":0.0,"output_usd":0.0012100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1410514999879524}
{"item_id":"senseval2.d002.s070.t001","gold_sense_keys":["believe%2:31:00::"],"candidates":[{"index":1,"sense_key":"believe%2:31:00::","synset_id":"believe.v.01"},{"index":2,"sense_key":"believe%2:31:04::","synset_id":"think.v.01"},{"index":3,"sense_key":"believe%2:31:03::","synset_id":"believe.v.03"},{"index":4,"sense_key":"believe%2:31:01::","synset_id":"believe.v.04"},{"index":5,"sense_key":"believe%2:31:02::","synset_id":"believe.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"believe%2:31:00::","call_ids":["senseval2.d002.s070.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"believe%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":769,"cached_input_tokens":0,"output_tokens":100,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002538,"input_uncached_usd":0.0015379999999999999,"input_cached_usd":0.0,"output_usd":0.001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5204042909899727}
{"item_id":"senseval2.d002.s070.t002","gold_sense_keys":["equal%3:00:00::"],"candidates":[{"index":1,"sense_key":"equal%3:00:00::","synset_id":"equal.a.01"},{"index":2,"sense_key":"equal%3:00:03::","synset_id":"adequate.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"equal%3:00:00::","call_ids":["senseval2.d002.s070.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"equal%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":598,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0013160000000000001,"input_uncached_usd":0.001196,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9624063749797642}
{"item_id":"senseval2.d002.s070.t003","gold_sense_keys":["academic%3:01:00::"],"candidates":[{"index":1,"sense_key":"academic%3:01:00::","synset_id":"academic.a.01"},{"index":2,"sense_key":"academic%5:00:00:theoretical:01","synset_id":"academic.s.01"},{"index":3,"sense_key":"academic%5:00:00:scholarly:00","synset_id":"academic.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"academic%3:01:00::","call_ids":["senseval2.d002.s070.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"academic%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":662,"cached_input_tokens":0,"output_tokens":90,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002224,"input_uncached_usd":0.0013239999999999999,"input_cached_usd":0.0,"output_usd":0.0009000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.680815125000663}
{"item_id":"senseval2.d002.s070.t004","gold_sense_keys":["potential%1:26:00::"],"candidates":[{"index":1,"sense_key":"potential%1:26:00::","synset_id":"potential.n.01"},{"index":2,"sense_key":"potential%1:19:00::","synset_id":"electric_potential.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"potential%1:26:00::","call_ids":["senseval2.d002.s070.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"potential%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":596,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001312,"input_uncached_usd":0.001192,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2664988749893382}
{"item_id":"senseval2.d002.s071.t000","gold_sense_keys":["generally%4:02:02::"],"candidates":[{"index":1,"sense_key":"generally%4:02:02::","synset_id":"by_and_large.r.01"},{"index":2,"sense_key":"generally%4:02:01::","synset_id":"generally.r.02"},{"index":3,"sense_key":"generally%4:02:00::","synset_id":"broadly.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"generally%4:02:02::","call_ids":["senseval2.d002.s071.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"generally%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":670,"cached_input_tokens":0,"output_tokens":108,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0024200000000000003,"input_uncached_usd":0.00134,"input_cached_usd":0.0,"output_usd":0.00108,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7187502500601113}
{"item_id":"senseval2.d002.s071.t001","gold_sense_keys":["desirable%3:00:00::"],"candidates":[{"index":1,"sense_key":"desirable%3:00:00::","synset_id":"desirable.a.01"},{"index":2,"sense_key":"desirable%5:00:00:eligible:00","synset_id":"desirable.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"desirable%3:00:00::","call_ids":["senseval2.d002.s071.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"desirable%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":611,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001342,"input_uncached_usd":0.001222,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0174643750069663}
{"item_id":"senseval2.d002.s071.t002","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":2,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":3,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":4,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":5,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":6,"sense_key":"use%2:41:03::","synset_id":"use.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%2:34:01::","call_ids":["senseval2.d002.s071.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":869,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001858,"input_uncached_usd":0.001738,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1341324169188738}
{"item_id":"senseval2.d002.s071.t005","gold_sense_keys":["rather%4:02:02::"],"candidates":[{"index":1,"sense_key":"rather%4:02:02::","synset_id":"rather.r.01"},{"index":2,"sense_key":"rather%4:02:04::","synset_id":"rather.r.02"},{"index":3,"sense_key":"rather%4:02:01::","synset_id":"preferably.r.01"},{"index":4,"sense_key":"rather%4:02:00::","synset_id":"quite.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"rather%4:02:01::","call_ids":["senseval2.d002.s071.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"rather%4:02:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":760,"cached_input_tokens":0,"output_tokens":282,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00434,"input_uncached_usd":0.0015199999999999999,"input_cached_usd":0.0,"output_usd":0.00282,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.8195656669558957}
{"item_id":"senseval2.d002.s071.t006","gold_sense_keys":["new%3:00:00::"],"candidates":[{"index":1,"sense_key":"new%3:00:00::","synset_id":"new.a.01"},{"index":2,"sense_key":"new%5:00:00:original:00","synset_id":"fresh.s.03"},{"index":3,"sense_key":"new%5:00:00:inexperienced:00","synset_id":"raw.s.11"},{"index":4,"sense_key":"new%5:00:00:unprecedented:00","synset_id":"new.s.03"},{"index":5,"sense_key":"new%5:00:00:other:00","synset_id":"new.s.04"},{"index":6,"sense_key":"new%3:00:09::","synset_id":"new.a.06"},{"index":7,"sense_key":"new%5:00:02:original:00","synset_id":"newfangled.s.01"},{"index":8,"sense_key":"new%5:00:02:late:01","synset_id":"new.s.06"},{"index":9,"sense_key":"new%5:00:01:late:01","synset_id":"modern.s.04"},{"index":10,"sense_key":"new%5:00:00:early:02","synset_id":"new.s.08"},{"index":11,"sense_key":"new%5:00:00:unaccustomed:00","synset_id":"new.s.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"new%3:00:00::","call_ids":["senseval2.d002.s071.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"new%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1174,"cached_input_tokens":0,"output_tokens":112,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0034679999999999997,"input_uncached_usd":0.002348,"input_cached_usd":0.0,"output_usd":0.0011200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9177081669913605}
{"item_id":"senseval2.d002.s071.t007","gold_sense_keys":["one%1:09:00::"],"candidates":[{"index":1,"sense_key":"one%1:23:00::","synset_id":"one.n.01"},{"index":2,"sense_key":"one%1:09:00::","synset_id":"one.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"one%1:09:00::","call_ids":["senseval2.d002.s071.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"one%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":609,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001338,"input_uncached_usd":0.001218,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1579396669985726}
{"item_id":"senseval2.d002.s072.t003","gold_sense_keys":["often%4:02:02::"],"candidates":[{"index":1,"sense_key":"often%4:02:00::","synset_id":"frequently.r.01"},{"index":2,"sense_key":"often%4:02:01::","synset_id":"much.r.05"},{"index":3,"sense_key":"often%4:02:02::","synset_id":"often.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"often%4:02:02::","call_ids":["senseval2.d002.s072.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"often%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":618,"cached_input_tokens":0,"output_tokens":83,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002066,"input_uncached_usd":0.001236,"input_cached_usd":0.0,"output_usd":0.0008300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5752981659024954}
{"item_id":"senseval2.d002.s073.t000","gold_sense_keys":["dubious%5:00:00:questionable:00"],"candidates":[{"index":1,"sense_key":"dubious%5:00:00:uncertain:02","synset_id":"doubtful.s.02"},{"index":2,"sense_key":"dubious%5:00:00:questionable:00","synset_id":"doubtful.s.01"},{"index":3,"sense_key":"dubious%5:00:00:unconvinced:00","synset_id":"dubious.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"dubious%5:00:00:questionable:00","call_ids":["senseval2.d002.s073.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"dubious%5:00:00:questionable:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":684,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001488,"input_uncached_usd":0.0013679999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0279104999499395}
{"item_id":"senseval2.d002.s073.t001","gold_sense_keys":["psychological%3:01:00::"],"candidates":[{"index":1,"sense_key":"psychological%5:00:00:mental:00","synset_id":"psychological.s.01"},{"index":2,"sense_key":"psychological%3:01:00::","synset_id":"psychological.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"psychological%3:01:00::","call_ids":["senseval2.d002.s073.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"psychological%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":554,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0012280000000000001,"input_uncached_usd":0.001108,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1940421250183135}
{"item_id":"senseval2.d002.s073.t004","gold_sense_keys":["rather%4:02:02::"],"candidates":[{"index":1,"sense_key":"rather%4:02:02::","synset_id":"rather.r.01"},{"index":2,"sense_key":"rather%4:02:04::","synset_id":"rather.r.02"},{"index":3,"sense_key":"rather%4:02:01::","synset_id":"preferably.r.01"},{"index":4,"sense_key":"rather%4:02:00::","synset_id":"quite.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"rather%4:02:02::","call_ids":["senseval2.d002.s073.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"rather%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":725,"cached_input_tokens":0,"output_tokens":146,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0029100000000000003,"input_uncached_usd":0.00145,"input_cached_usd":0.0,"output_usd":0.0014600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0487390840426087}
{"item_id":"senseval2.d002.s073.t005","gold_sense_keys":["educational%3:01:00::"],"candidates":[{"index":1,"sense_key":"educational%3:01:00::","synset_id":"educational.a.01"},{"index":2,"sense_key":"educational%5:00:00:instructive:00","synset_id":"educational.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"educational%3:01:00::","call_ids":["senseval2.d002.s073.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"educational%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":528,"cached_input_tokens":0,"output_tokens":69,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001746,"input_uncached_usd":0.0010559999999999999,"input_cached_usd":0.0,"output_usd":0.0006900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8882953330175951}
{"item_id":"senseval2.d002.s073.t006","gold_sense_keys":["experience%1:09:01::"],"candidates":[{"index":1,"sense_key":"experience%1:09:01::","synset_id":"experience.n.01"},{"index":2,"sense_key":"experience%1:09:00::","synset_id":"experience.n.02"},{"index":3,"sense_key":"experience%1:11:00::","synset_id":"experience.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"experience%1:09:01::","call_ids":["senseval2.d002.s073.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"experience%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":623,"cached_input_tokens":0,"output_tokens":75,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001996,"input_uncached_usd":0.001246,"input_cached_usd":0.0,"output_usd":0.00075,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.40935391606763}
{"item_id":"senseval2.d002.s074.t000","gold_sense_keys":["reason%1:09:01::","reason%1:10:00::","reason%1:10:01::"],"candidates":[{"index":1,"sense_key":"reason%1:16:00::","synset_id":"reason.n.01"},{"index":2,"sense_key":"reason%1:10:01::","synset_id":"reason.n.02"},{"index":3,"sense_key":"reason%1:09:00::","synset_id":"reason.n.03"},{"index":4,"sense_key":"reason%1:26:00::","synset_id":"rationality.n.01"},{"index":5,"sense_key":"reason%1:10:00::","synset_id":"cause.n.02"},{"index":6,"sense_key":"reason%1:09:01::","synset_id":"reason.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"reason%1:10:01::","call_ids":["senseval2.d002.s074.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"reason%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":931,"cached_input_tokens":0,"output_tokens":138,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003242,"input_uncached_usd":0.001862,"input_cached_usd":0.0,"output_usd":0.0013800000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2319911250378937}
{"item_id":"senseval2.d002.s074.t001","gold_sense_keys":["american%3:01:01::"],"candidates":[{"index":1,"sense_key":"american%3:01:01::","synset_id":"american.a.01"},{"index":2,"sense_key":"american%3:01:00::","synset_id":"american.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"american%3:01:01::","call_ids":["senseval2.d002.s074.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"american%3:01:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":627,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001374,"input_uncached_usd":0.001254,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5330589580116794}
{"item_id":"senseval2.d002.s074.t002","gold_sense_keys":["student%1:18:00::"],"candidates":[{"index":1,"sense_key":"student%1:18:00::","synset_id":"student.n.01"},{"index":2,"sense_key":"student%1:18:01::","synset_id":"scholar.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"student%1:18:00::","call_ids":["senseval2.d002.s074.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"student%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":637,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001394,"input_uncached_usd":0.001274,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.114868541015312}
{"item_id":"senseval2.d002.s074.t003","gold_sense_keys":["so%4:02:02::"],"candidates":[{"index":1,"sense_key":"so%4:02:02::","synset_id":"so.r.01"},{"index":2,"sense_key":"so%4:02:06::","synset_id":"so.r.02"},{"index":3,"sense_key":"so%4:02:05::","synset_id":"so.r.03"},{"index":4,"sense_key":"so%4:02:07::","synset_id":"so.r.04"},{"index":5,"sense_key":"so%4:02:08::","synset_id":"so.r.05"},{"index":6,"sense_key":"so%4:02:04::","synset_id":"thus.r.02"},{"index":7,"sense_key":"so%4:02:03::","synset_id":"so.r.07"},{"index":8,"sense_key":"so%4:02:09::","synset_id":"then.r.01"},{"index":9,"sense_key":"so%4:02:10::","synset_id":"therefore.r.01"},{"index":10,"sense_key":"so%4:02:00::","synset_id":"indeed.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"so%4:02:02::","call_ids":["senseval2.d002.s074.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"so%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1177,"cached_input_tokens":0,"output_tokens":87,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003224,"input_uncached_usd":0.0023539999999999998,"input_cached_usd":0.0,"output_usd":0.0008700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1068852500757203}
{"item_id":"senseval2.d002.s074.t006","gold_sense_keys":["compare%2:31:00::"],"candidates":[{"index":1,"sense_key":"compare%2:31:00::","synset_id":"compare.v.01"},{"index":2,"sense_key":"compare%2:42:00::","synset_id":"compare.v.02"},{"index":3,"sense_key":"compare%2:31:01::","synset_id":"compare.v.03"},{"index":4,"sense_key":"compare%2:32:00::","synset_id":"compare.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"compare%2:31:00::","call_ids":["senseval2.d002.s074.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"compare%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":754,"cached_input_tokens":0,"output_tokens":45,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001958,"input_uncached_usd":0.001508,"input_cached_usd":0.0,"output_usd":0.00045000000000000004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2724468329688534}
{"item_id":"senseval2.d002.s074.t011","gold_sense_keys":["student%1:18:00::"],"candidates":[{"index":1,"sense_key":"student%1:18:00::","synset_id":"student.n.01"},{"index":2,"sense_key":"student%1:18:01::","synset_id":"scholar.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"student%1:18:00::","call_ids":["senseval2.d002.s074.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"student%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":637,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001394,"input_uncached_usd":0.001274,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1702045829733834}
{"item_id":"senseval2.d002.s074.t012","gold_sense_keys":["influence%1:19:00::"],"candidates":[{"index":1,"sense_key":"influence%1:07:00::","synset_id":"influence.n.01"},{"index":2,"sense_key":"influence%1:04:00::","synset_id":"influence.n.02"},{"index":3,"sense_key":"influence%1:09:00::","synset_id":"influence.n.03"},{"index":4,"sense_key":"influence%1:19:00::","synset_id":"influence.n.04"},{"index":5,"sense_key":"influence%1:18:00::","synset_id":"influence.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"influence%1:19:00::","call_ids":["senseval2.d002.s074.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"influence%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":769,"cached_input_tokens":0,"output_tokens":132,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0028580000000000003,"input_uncached_usd":0.0015379999999999999,"input_cached_usd":0.0,"output_usd":0.0013200000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9261049999622628}
{"item_id":"senseval2.d002.s074.t013","gold_sense_keys":["american%3:01:01::"],"candidates":[{"index":1,"sense_key":"american%3:01:01::","synset_id":"american.a.01"},{"index":2,"sense_key":"american%3:01:00::","synset_id":"american.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"american%3:01:01::","call_ids":["senseval2.d002.s074.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"american%3:01:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":627,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001374,"input_uncached_usd":0.001254,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.7656719590304419}
{"item_id":"senseval2.d002.s075.t000","gold_sense_keys":["want%2:37:00::"],"candidates":[{"index":1,"sense_key":"want%2:37:00::","synset_id":"desire.v.01"},{"index":2,"sense_key":"want%2:34:00::","synset_id":"want.v.02"},{"index":3,"sense_key":"want%2:35:00::","synset_id":"want.v.03"},{"index":4,"sense_key":"want%2:31:01::","synset_id":"want.v.04"},{"index":5,"sense_key":"want%2:42:01::","synset_id":"want.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"want%2:37:00::","call_ids":["senseval2.d002.s075.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"want%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":803,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001726,"input_uncached_usd":0.001606,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5249481670325622}
{"item_id":"senseval2.d002.s075.t001","gold_sense_keys":["appreciate%2:31:00::"],"candidates":[{"index":1,"sense_key":"appreciate%2:37:00::","synset_id":"appreciate.v.01"},{"index":2,"sense_key":"appreciate%2:31:00::","synset_id":"appreciate.v.02"},{"index":3,"sense_key":"appreciate%2:40:00::","synset_id":"prize.v.01"},{"index":4,"sense_key":"appreciate%2:30:00::","synset_id":"appreciate.v.04"},{"index":5,"sense_key":"appreciate%2:30:01::","synset_id":"appreciate.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"appreciate%2:31:00::","call_ids":["senseval2.d002.s075.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"appreciate%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":819,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001758,"input_uncached_usd":0.001638,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9560555419884622}
{"item_id":"senseval2.d002.s075.t002","gold_sense_keys":["just%4:02:05::"],"candidates":[{"index":1,"sense_key":"just%4:02:00::","synset_id":"merely.r.01"},{"index":2,"sense_key":"just%4:02:05::","synset_id":"precisely.r.01"},{"index":3,"sense_key":"just%4:02:01::","synset_id":"just.r.03"},{"index":4,"sense_key":"just%4:02:03::","synset_id":"just.r.04"},{"index":5,"sense_key":"just%4:02:04::","synset_id":"barely.r.01"},{"index":6,"sense_key":"just%4:02:06::","synset_id":"just.r.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"just%4:02:05::","call_ids":["senseval2.d002.s075.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"just%4:02:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":931,"cached_input_tokens":0,"output_tokens":118,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003042,"input_uncached_usd":0.001862,"input_cached_usd":0.0,"output_usd":0.00118,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.062532417010516}
{"item_id":"senseval2.d002.s075.t004","gold_sense_keys":["situation%1:26:00::"],"candidates":[{"index":1,"sense_key":"situation%1:26:00::","synset_id":"situation.n.01"},{"index":2,"sense_key":"situation%1:26:01::","synset_id":"situation.n.02"},{"index":3,"sense_key":"situation%1:26:02::","synset_id":"situation.n.03"},{"index":4,"sense_key":"situation%1:15:00::","synset_id":"site.n.02"},{"index":5,"sense_key":"situation%1:04:00::","synset_id":"position.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"situation%1:26:00::","call_ids":["senseval2.d002.s075.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"situation%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":902,"cached_input_tokens":0,"output_tokens":115,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002954,"input_uncached_usd":0.0018039999999999998,"input_cached_usd":0.0,"output_usd":0.0011500000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8659926250111312}
{"item_id":"senseval2.d002.s075.t005","gold_sense_keys":["student%1:18:00::"],"candidates":[{"index":1,"sense_key":"student%1:18:00::","synset_id":"student.n.01"},{"index":2,"sense_key":"student%1:18:01::","synset_id":"scholar.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"student%1:18:00::","call_ids":["senseval2.d002.s075.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"student%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":656,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0014320000000000001,"input_uncached_usd":0.001312,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9841056249570101}
{"item_id":"senseval2.d002.s075.t006","gold_sense_keys":["add%2:31:00::"],"candidates":[{"index":1,"sense_key":"add%2:30:00::","synset_id":"add.v.01"},{"index":2,"sense_key":"add%2:32:01::","synset_id":"add.v.02"},{"index":3,"sense_key":"add%2:40:00::","synset_id":"lend.v.01"},{"index":4,"sense_key":"add%2:31:00::","synset_id":"add.v.04"},{"index":5,"sense_key":"add%2:32:00::","synset_id":"total.v.02"},{"index":6,"sense_key":"add%2:42:00::","synset_id":"add.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"add%2:31:00::","call_ids":["senseval2.d002.s075.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"add%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":911,"cached_input_tokens":0,"output_tokens":114,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0029620000000000002,"input_uncached_usd":0.0018219999999999998,"input_cached_usd":0.0,"output_usd":0.0011400000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.502324500004761}
{"item_id":"senseval2.d002.s075.t007","gold_sense_keys":["subtract%2:31:00::"],"candidates":[{"index":1,"sense_key":"subtract%2:31:00::","synset_id":"subtract.v.01"},{"index":2,"sense_key":"subtract%2:30:00::","synset_id":"subtract.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"subtract%2:31:00::","call_ids":["senseval2.d002.s075.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"subtract%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":645,"cached_input_tokens":0,"output_tokens":75,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00204,"input_uncached_usd":0.00129,"input_cached_usd":0.0,"output_usd":0.00075,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3620068330783397}
{"item_id":"senseval2.d002.s075.t008","gold_sense_keys":["learn%2:31:00::"],"candidates":[{"index":1,"sense_key":"learn%2:31:00::","synset_id":"learn.v.01"},{"index":2,"sense_key":"learn%2:31:01::","synset_id":"learn.v.02"},{"index":3,"sense_key":"learn%2:31:03::","synset_id":"memorize.v.01"},{"index":4,"sense_key":"learn%2:31:02::","synset_id":"learn.v.04"},{"index":5,"sense_key":"learn%2:32:00::","synset_id":"teach.v.01"},{"index":6,"sense_key":"learn%2:32:02::","synset_id":"determine.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"learn%2:31:00::","call_ids":["senseval2.d002.s075.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"learn%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":972,"cached_input_tokens":0,"output_tokens":107,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003014,"input_uncached_usd":0.001944,"input_cached_usd":0.0,"output_usd":0.00107,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4509529169881716}
{"item_id":"senseval2.d002.s075.t010","gold_sense_keys":["basis%1:09:00::"],"candidates":[{"index":1,"sense_key":"basis%1:24:00::","synset_id":"footing.n.02"},{"index":2,"sense_key":"basis%1:09:00::","synset_id":"basis.n.02"},{"index":3,"sense_key":"basis%1:24:01::","synset_id":"basis.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"basis%1:09:00::","call_ids":["senseval2.d002.s075.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"basis%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":752,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002144,"input_uncached_usd":0.001504,"input_cached_usd":0.0,"output_usd":0.00064,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2421900000190362}
{"item_id":"senseval2.d002.s075.t011","gold_sense_keys":["mathematical%3:01:00::"],"candidates":[{"index":1,"sense_key":"mathematical%3:01:00::","synset_id":"mathematical.a.01"},{"index":2,"sense_key":"mathematical%3:00:00::","synset_id":"numerical.a.03"},{"index":3,"sense_key":"mathematical%5:00:00:unquestionable:00","synset_id":"mathematical.s.01"},{"index":4,"sense_key":"mathematical%5:00:00:possible:00","synset_id":"mathematical.s.02"},{"index":5,"sense_key":"mathematical%5:00:00:exact:00","synset_id":"mathematical.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"mathematical%3:01:00::","call_ids":["senseval2.d002.s075.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"mathematical%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":840,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0018,"input_uncached_usd":0.0016799999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.800458583049476}
{"item_id":"senseval2.d002.s075.t013","gold_sense_keys":["read%2:31:00::"],"candidates":[{"index":1,"sense_key":"read%2:31:00::","synset_id":"read.v.01"},{"index":2,"sense_key":"read%2:42:00::","synset_id":"read.v.02"},{"index":3,"sense_key":"read%2:31:01::","synset_id":"read.v.03"},{"index":4,"sense_key":"read%2:31:06::","synset_id":"read.v.04"},{"index":5,"sense_key":"read%2:31:02::","synset_id":"read.v.05"},{"index":6,"sense_key":"read%2:31:09::","synset_id":"take.v.06"},{"index":7,"sense_key":"read%2:31:05::","synset_id":"learn.v.04"},{"index":8,"sense_key":"read%2:32:02::","synset_id":"read.v.08"},{"index":9,"sense_key":"read%2:36:05::","synset_id":"read.v.09"},{"index":10,"sense_key":"read%2:31:08::","synset_id":"read.v.10"},{"index":11,"sense_key":"read%2:31:04::","synset_id":"understand.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"read%2:31:00::","call_ids":["senseval2.d002.s075.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"read%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1207,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002534,"input_uncached_usd":0.002414,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1368435840122402}
{"item_id":"senseval2.d002.s075.t014","gold_sense_keys":["article%1:10:00::"],"candidates":[{"index":1,"sense_key":"article%1:10:00::","synset_id":"article.n.01"},{"index":2,"sense_key":"article%1:03:00::","synset_id":"article.n.02"},{"index":3,"sense_key":"article%1:10:02::","synset_id":"article.n.03"},{"index":4,"sense_key":"article%1:10:01::","synset_id":"article.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"article%1:10:00::","call_ids":["senseval2.d002.s075.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"article%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":710,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0015400000000000001,"input_uncached_usd":0.00142,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1676001660525799}
{"item_id":"senseval2.d002.s075.t015","gold_sense_keys":["recent%5:00:00:new:00"],"candidates":[{"index":1,"sense_key":"recent%5:00:00:new:00","synset_id":"recent.s.01"},{"index":2,"sense_key":"recent%5:00:00:past:00","synset_id":"late.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"recent%5:00:00:new:00","call_ids":["senseval2.d002.s075.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"recent%5:00:00:new:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":640,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00201,"input_uncached_usd":0.0012799999999999999,"input_cached_usd":0.0,"output_usd":0.0007300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.636727291974239}
{"item_id":"senseval2.d002.s075.t017","gold_sense_keys":["major%1:18:01::"],"candidates":[{"index":1,"sense_key":"major%1:18:00::","synset_id":"major.n.01"},{"index":2,"sense_key":"major%1:18:02::","synset_id":"major.n.02"},{"index":3,"sense_key":"major%1:18:01::","synset_id":"major.n.03"},{"index":4,"sense_key":"major%1:09:00::","synset_id":"major.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"major%1:18:01::","call_ids":["senseval2.d002.s075.t017__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"major%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":756,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001632,"input_uncached_usd":0.0015119999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4708201249595731}
{"item_id":"senseval2.d002.s076.t000","gold_sense_keys":["most%4:02:00::"],"candidates":[{"index":1,"sense_key":"most%4:02:00::","synset_id":"most.r.01"},{"index":2,"sense_key":"most%4:02:01::","synset_id":"most.r.02"},{"index":3,"sense_key":"most%4:02:02::","synset_id":"about.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"most%4:02:00::","call_ids":["senseval2.d002.s076.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"most%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":751,"cached_input_tokens":0,"output_tokens":45,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001952,"input_uncached_usd":0.0015019999999999999,"input_cached_usd":0.0,"output_usd":0.00045000000000000004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6803354170406237}
{"item_id":"senseval2.d002.s076.t001","gold_sense_keys":["important%3:00:00::","important%3:00:02::","important%3:00:04::"],"candidates":[{"index":1,"sense_key":"important%3:00:00::","synset_id":"important.a.01"},{"index":2,"sense_key":"important%3:00:04::","synset_id":"significant.a.01"},{"index":3,"sense_key":"important%3:00:02::","synset_id":"crucial.a.01"},{"index":4,"sense_key":"important%5:00:00:influential:00","synset_id":"authoritative.s.01"},{"index":5,"sense_key":"important%5:00:00:immodest:02","synset_id":"important.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"important%3:00:00::","call_ids":["senseval2.d002.s076.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"important%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":870,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00186,"input_uncached_usd":0.00174,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0584145000902936}
{"item_id":"senseval2.d002.s076.t002","gold_sense_keys":["school%1:14:00::","school%1:14:03::"],"candidates":[{"index":1,"sense_key":"school%1:14:00::","synset_id":"school.n.01"},{"index":2,"sense_key":"school%1:06:00::","synset_id":"school.n.02"},{"index":3,"sense_key":"school%1:09:00::","synset_id":"school.n.03"},{"index":4,"sense_key":"school%1:14:02::","synset_id":"school.n.04"},{"index":5,"sense_key":"school%1:28:00::","synset_id":"school.n.05"},{"index":6,"sense_key":"school%1:14:03::","synset_id":"school.n.06"},{"index":7,"sense_key":"school%1:14:01::","synset_id":"school.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"school%1:14:00::","call_ids":["senseval2.d002.s076.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"school%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":961,"cached_input_tokens":0,"output_tokens":117,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003092,"input_uncached_usd":0.0019219999999999999,"input_cached_usd":0.0,"output_usd":0.00117,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7111349160550162}
{"item_id":"senseval2.d002.s076.t003","gold_sense_keys":["principal%1:18:00::"],"candidates":[{"index":1,"sense_key":"principal%1:21:01::","synset_id":"principal.n.01"},{"index":2,"sense_key":"principal%1:18:00::","synset_id":"principal.n.02"},{"index":3,"sense_key":"principal%1:18:01::","synset_id":"star.n.04"},{"index":4,"sense_key":"principal%1:21:02::","synset_id":"principal.n.04"},{"index":5,"sense_key":"principal%1:18:03::","synset_id":"principal.n.05"},{"index":6,"sense_key":"principal%1:18:02::","synset_id":"principal.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"principal%1:18:00::","call_ids":["senseval2.d002.s076.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"principal%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":874,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001868,"input_uncached_usd":0.001748,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1570470409933478}
{"item_id":"senseval2.d002.s076.t005","gold_sense_keys":["measure%1:03:00::"],"candidates":[{"index":1,"sense_key":"measure%1:04:01::","synset_id":"measure.n.01"},{"index":2,"sense_key":"measure%1:03:00::","synset_id":"measure.n.02"},{"index":3,"sense_key":"measure%1:10:02::","synset_id":"bill.n.01"},{"index":4,"sense_key":"measure%1:04:00::","synset_id":"measurement.n.01"},{"index":5,"sense_key":"measure%1:10:03::","synset_id":"standard.n.01"},{"index":6,"sense_key":"measure%1:10:01::","synset_id":"meter.n.03"},{"index":7,"sense_key":"measure%1:10:00::","synset_id":"measure.n.07"},{"index":8,"sense_key":"measure%1:06:00::","synset_id":"measuring_stick.n.01"},{"index":9,"sense_key":"measure%1:06:01::","synset_id":"measure.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"measure%1:03:00::","call_ids":["senseval2.d002.s076.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"measure%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1164,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002858,"input_uncached_usd":0.0023279999999999998,"input_cached_usd":0.0,"output_usd":0.0005300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.30947095900774}
{"item_id":"senseval2.d002.s076.t006","gold_sense_keys":["authority%1:07:00::"],"candidates":[{"index":1,"sense_key":"authority%1:07:00::","synset_id":"authority.n.01"},{"index":2,"sense_key":"authority%1:18:01::","synset_id":"authority.n.02"},{"index":3,"sense_key":"authority%1:18:00::","synset_id":"authority.n.03"},{"index":4,"sense_key":"authority%1:09:00::","synset_id":"assurance.n.01"},{"index":5,"sense_key":"authority%1:14:00::","synset_id":"agency.n.01"},{"index":6,"sense_key":"authority%1:07:02::","synset_id":"authority.n.06"},{"index":7,"sense_key":"authority%1:10:00::","synset_id":"authority.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"authority%1:07:00::","call_ids":["senseval2.d002.s076.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"authority%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1043,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0022059999999999996,"input_uncached_usd":0.0020859999999999997,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.128126542083919}
{"item_id":"senseval2.d002.s076.t007","gold_sense_keys":["faculty%1:14:00::"],"candidates":[{"index":1,"sense_key":"faculty%1:09:00::","synset_id":"faculty.n.01"},{"index":2,"sense_key":"faculty%1:14:00::","synset_id":"staff.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"faculty%1:14:00::","call_ids":["senseval2.d002.s076.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"faculty%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":688,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001496,"input_uncached_usd":0.0013759999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8565044169081375}
{"item_id":"senseval2.d002.s076.t009","gold_sense_keys":["matter%1:09:01::"],"candidates":[{"index":1,"sense_key":"matter%1:09:01::","synset_id":"matter.n.01"},{"index":2,"sense_key":"matter%1:09:00::","synset_id":"topic.n.02"},{"index":3,"sense_key":"matter%1:03:00::","synset_id":"matter.n.03"},{"index":4,"sense_key":"matter%1:09:02::","synset_id":"matter.n.04"},{"index":5,"sense_key":"matter%1:07:00::","synset_id":"matter.n.05"},{"index":6,"sense_key":"matter%1:10:00::","synset_id":"matter.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"matter%1:09:01::","call_ids":["senseval2.d002.s076.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"matter%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":891,"cached_input_tokens":0,"output_tokens":134,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0031219999999999998,"input_uncached_usd":0.001782,"input_cached_usd":0.0,"output_usd":0.00134,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2152061249362305}
{"item_id":"senseval2.d002.s076.t010","gold_sense_keys":["student%1:18:00::"],"candidates":[{"index":1,"sense_key":"student%1:18:00::","synset_id":"student.n.01"},{"index":2,"sense_key":"student%1:18:01::","synset_id":"scholar.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"student%1:18:00::","call_ids":["senseval2.d002.s076.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"student%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":683,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0014860000000000001,"input_uncached_usd":0.001366,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4601044160081074}
{"item_id":"senseval2.d002.s077.t000","gold_sense_keys":["study%1:10:00::"],"candidates":[{"index":1,"sense_key":"study%1:04:00::","synset_id":"survey.n.01"},{"index":2,"sense_key":"study%1:09:03::","synset_id":"study.n.02"},{"index":3,"sense_key":"study%1:10:00::","synset_id":"report.n.01"},{"index":4,"sense_key":"study%1:09:04::","synset_id":"study.n.04"},{"index":5,"sense_key":"study%1:06:00::","synset_id":"study.n.05"},{"index":6,"sense_key":"study%1:09:02::","synset_id":"discipline.n.01"},{"index":7,"sense_key":"study%1:06:01::","synset_id":"sketch.n.01"},{"index":8,"sense_key":"study%1:09:01::","synset_id":"cogitation.n.02"},{"index":9,"sense_key":"study%1:18:00::","synset_id":"study.n.09"},{"index":10,"sense_key":"study%1:10:01::","synset_id":"study.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"study%1:10:00::","call_ids":["senseval2.d002.s077.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"study%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1132,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002894,"input_uncached_usd":0.002264,"input_cached_usd":0.0,"output_usd":0.00063,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4042181669501588}
{"item_id":"senseval2.d002.s077.t001","gold_sense_keys":["study%1:10:00::"],"candidates":[{"index":1,"sense_key":"study%1:04:00::","synset_id":"survey.n.01"},{"index":2,"sense_key":"study%1:09:03::","synset_id":"study.n.02"},{"index":3,"sense_key":"study%1:10:00::","synset_id":"report.n.01"},{"index":4,"sense_key":"study%1:09:04::","synset_id":"study.n.04"},{"index":5,"sense_key":"study%1:06:00::","synset_id":"study.n.05"},{"index":6,"sense_key":"study%1:09:02::","synset_id":"discipline.n.01"},{"index":7,"sense_key":"study%1:06:01::","synset_id":"sketch.n.01"},{"index":8,"sense_key":"study%1:09:01::","synset_id":"cogitation.n.02"},{"index":9,"sense_key":"study%1:18:00::","synset_id":"study.n.09"},{"index":10,"sense_key":"study%1:10:01::","synset_id":"study.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"study%1:10:00::","call_ids":["senseval2.d002.s077.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"study%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1131,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0023819999999999996,"input_uncached_usd":0.0022619999999999997,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.056812292081304}
{"item_id":"senseval2.d002.s077.t002","gold_sense_keys":["most%4:02:00::"],"candidates":[{"index":1,"sense_key":"most%4:02:00::","synset_id":"most.r.01"},{"index":2,"sense_key":"most%4:02:01::","synset_id":"most.r.02"},{"index":3,"sense_key":"most%4:02:02::","synset_id":"about.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"most%4:02:00::","call_ids":["senseval2.d002.s077.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"most%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":732,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001994,"input_uncached_usd":0.001464,"input_cached_usd":0.0,"output_usd":0.0005300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.3517968750093132}
{"item_id":"senseval2.d002.s077.t004","gold_sense_keys":["tell%2:32:00::"],"candidates":[{"index":1,"sense_key":"tell%2:32:04::","synset_id":"state.v.01"},{"index":2,"sense_key":"tell%2:32:00::","synset_id":"tell.v.02"},{"index":3,"sense_key":"tell%2:32:01::","synset_id":"tell.v.03"},{"index":4,"sense_key":"tell%2:32:02::","synset_id":"order.v.01"},{"index":5,"sense_key":"tell%2:31:00::","synset_id":"tell.v.05"},{"index":6,"sense_key":"tell%2:32:08::","synset_id":"assure.v.02"},{"index":7,"sense_key":"tell%2:32:03::","synset_id":"tell.v.07"},{"index":8,"sense_key":"tell%2:31:01::","synset_id":"distinguish.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"tell%2:32:00::","call_ids":["senseval2.d002.s077.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"tell%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1009,"cached_input_tokens":0,"output_tokens":140,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003418,"input_uncached_usd":0.002018,"input_cached_usd":0.0,"output_usd":0.0014000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9033392500132322}
{"item_id":"senseval2.d002.s077.t005","gold_sense_keys":["best%3:00:00::"],"candidates":[{"index":1,"sense_key":"best%3:00:00::","synset_id":"best.a.01"},{"index":2,"sense_key":"best%5:00:00:advisable:00","synset_id":"better.s.01"},{"index":3,"sense_key":"good%3:00:01::","synset_id":"good.a.01"},{"index":4,"sense_key":"full%5:00:00:ample:00","synset_id":"full.s.04"},{"index":5,"sense_key":"good%3:00:02::","synset_id":"good.a.03"},{"index":6,"sense_key":"estimable%5:00:00:reputable:00","synset_id":"estimable.s.01"},{"index":7,"sense_key":"beneficial%5:00:00:advantageous:00","synset_id":"beneficial.s.01"},{"index":8,"sense_key":"good%5:00:00:nice:00","synset_id":"good.s.04"},{"index":9,"sense_key":"good%5:00:00:righteous:00","synset_id":"good.s.05"},{"index":10,"sense_key":"adept%5:00:00:skilled:00","synset_id":"adept.s.01"},{"index":11,"sense_key":"good%5:00:00:complete:00","synset_id":"good.s.07"},{"index":12,"sense_key":"dear%5:00:00:close:02","synset_id":"dear.s.02"},{"index":13,"sense_key":"dependable%5:00:00:sound:00","synset_id":"dependable.s.03"},{"index":14,"sense_key":"good%5:00:00:opportune:00","synset_id":"good.s.10"},{"index":15,"sense_key":"good%5:00:00:fortunate:00","synset_id":"good.s.11"},{"index":16,"sense_key":"effective%5:00:00:operative:00","synset_id":"effective.s.03"},{"index":17,"sense_key":"good%5:00:00:pleasing:00","synset_id":"good.s.13"},{"index":18,"sense_key":"good%5:00:00:intellectual:00","synset_id":"good.s.14"},{"index":19,"sense_key":"good%5:00:00:healthy:00","synset_id":"good.s.15"},{"index":20,"sense_key":"good%5:00:00:healthful:00","synset_id":"good.s.16"},{"index":21,"sense_key":"good%5:00:00:genuine:00","synset_id":"good.s.17"},{"index":22,"sense_key":"good%5:00:00:fresh:01","synset_id":"good.s.18"},{"index":23,"sense_key":"good%5:00:00:discriminating:00","synset_id":"good.s.19"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"best%3:00:00::","call_ids":["senseval2.d002.s077.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"best%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2012,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004144,"input_uncached_usd":0.004024,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1567331249825656}
{"item_id":"senseval2.d002.s077.t006","gold_sense_keys":["school%1:14:00::"],"candidates":[{"index":1,"sense_key":"school%1:14:00::","synset_id":"school.n.01"},{"index":2,"sense_key":"school%1:06:00::","synset_id":"school.n.02"},{"index":3,"sense_key":"school%1:09:00::","synset_id":"school.n.03"},{"index":4,"sense_key":"school%1:14:02::","synset_id":"school.n.04"},{"index":5,"sense_key":"school%1:28:00::","synset_id":"school.n.05"},{"index":6,"sense_key":"school%1:14:03::","synset_id":"school.n.06"},{"index":7,"sense_key":"school%1:14:01::","synset_id":"school.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"school%1:14:00::","call_ids":["senseval2.d002.s077.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"school%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":943,"cached_input_tokens":0,"output_tokens":105,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0029360000000000002,"input_uncached_usd":0.0018859999999999999,"input_cached_usd":0.0,"output_usd":0.0010500000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0370965000474826}
{"item_id":"senseval2.d002.s077.t007","gold_sense_keys":["outside%5:00:00:external:00","outside%5:00:00:extrinsic:00"],"candidates":[{"index":1,"sense_key":"outside%3:00:00::","synset_id":"outside.a.01"},{"index":2,"sense_key":"outside%5:00:00:extrinsic:00","synset_id":"external.s.01"},{"index":3,"sense_key":"outside%5:00:00:external:00","synset_id":"outside.s.02"},{"index":4,"sense_key":"outside%3:00:04::","synset_id":"outdoor.a.01"},{"index":5,"sense_key":"outside%5:00:00:extramural:00","synset_id":"outside.s.03"},{"index":6,"sense_key":"outside%5:00:00:exterior:00","synset_id":"outside.s.04"},{"index":7,"sense_key":"outside%5:00:00:foreign:02","synset_id":"external.s.02"},{"index":8,"sense_key":"outside%5:00:00:unlikely:00","synset_id":"outside.s.06"},{"index":9,"sense_key":"outside%5:00:00:outer:00","synset_id":"outside.s.07"},{"index":10,"sense_key":"outside%5:00:00:inaccurate:00","synset_id":"away.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"outside%5:00:00:external:00","call_ids":["senseval2.d002.s077.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"outside%5:00:00:external:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1147,"cached_input_tokens":0,"output_tokens":220,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004494,"input_uncached_usd":0.002294,"input_cached_usd":0.0,"output_usd":0.0022,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.811816415982321}
{"item_id":"senseval2.d002.s077.t008","gold_sense_keys":["interference%1:04:00::"],"candidates":[{"index":1,"sense_key":"interference%1:10:00::","synset_id":"intervention.n.02"},{"index":2,"sense_key":"interference%1:04:00::","synset_id":"hindrance.n.03"},{"index":3,"sense_key":"interference%1:11:00::","synset_id":"noise.n.03"},{"index":4,"sense_key":"interference%1:04:01::","synset_id":"interference.n.04"},{"index":5,"sense_key":"interference%1:06:00::","synset_id":"hindrance.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"interference%1:04:00::","call_ids":["senseval2.d002.s077.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"interference%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":817,"cached_input_tokens":0,"output_tokens":150,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0031339999999999996,"input_uncached_usd":0.0016339999999999998,"input_cached_usd":0.0,"output_usd":0.0015,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.887137624900788}
{"item_id":"senseval2.d002.s077.t009","gold_sense_keys":["govern%2:41:00::"],"candidates":[{"index":1,"sense_key":"govern%2:41:02::","synset_id":"regulate.v.02"},{"index":2,"sense_key":"govern%2:41:01::","synset_id":"govern.v.02"},{"index":3,"sense_key":"govern%2:41:00::","synset_id":"govern.v.03"},{"index":4,"sense_key":"govern%2:42:00::","synset_id":"govern.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"govern%2:41:00::","call_ids":["senseval2.d002.s077.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"govern%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":797,"cached_input_tokens":0,"output_tokens":75,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0023439999999999997,"input_uncached_usd":0.001594,"input_cached_usd":0.0,"output_usd":0.00075,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4456547499867156}
{"item_id":"senseval2.d002.s077.t010","gold_sense_keys":["powerful%3:00:00::","powerful%5:00:00:influential:00"],"candidates":[{"index":1,"sense_key":"powerful%3:00:00::","synset_id":"powerful.a.01"},{"index":2,"sense_key":"powerful%5:00:02:strong:00","synset_id":"knock-down.s.01"},{"index":3,"sense_key":"powerful%5:00:00:influential:00","synset_id":"potent.s.01"},{"index":4,"sense_key":"powerful%5:00:01:strong:00","synset_id":"brawny.s.01"},{"index":5,"sense_key":"powerful%5:00:00:superhuman:00","synset_id":"herculean.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"powerful%3:00:00::","call_ids":["senseval2.d002.s077.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"powerful%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":854,"cached_input_tokens":0,"output_tokens":179,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0034980000000000002,"input_uncached_usd":0.0017079999999999999,"input_cached_usd":0.0,"output_usd":0.0017900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.550276624970138}
{"item_id":"senseval2.d002.s077.t011","gold_sense_keys":["head%1:18:01::"],"candidates":[{"index":1,"sense_key":"head%1:08:00::","synset_id":"head.n.01"},{"index":2,"sense_key":"head%1:05:00::","synset_id":"head.n.02"},{"index":3,"sense_key":"head%1:09:00::","synset_id":"mind.n.01"},{"index":4,"sense_key":"head%1:18:00::","synset_id":"head.n.04"},{"index":5,"sense_key":"head%1:14:00::","synset_id":"head.n.05"},{"index":6,"sense_key":"head%1:19:00::","synset_id":"head.n.06"},{"index":7,"sense_key":"head%1:15:00::","synset_id":"head.n.07"},{"index":8,"sense_key":"head%1:15:01::","synset_id":"fountainhead.n.02"},{"index":9,"sense_key":"head%1:10:03::","synset_id":"head.n.09"},{"index":10,"sense_key":"head%1:26:00::","synset_id":"head.n.10"},{"index":11,"sense_key":"head%1:23:00::","synset_id":"head.n.11"},{"index":12,"sense_key":"head%1:20:00::","synset_id":"capitulum.n.01"},{"index":13,"sense_key":"head%1:18:01::","synset_id":"principal.n.02"},{"index":14,"sense_key":"head%1:18:03::","synset_id":"head.n.14"},{"index":15,"sense_key":"head%1:18:02::","synset_id":"head.n.15"},{"index":16,"sense_key":"head%1:17:02::","synset_id":"promontory.n.01"},{"index":17,"sense_key":"head%1:17:01::","synset_id":"head.n.17"},{"index":18,"sense_key":"head%1:17:00::","synset_id":"head.n.18"},{"index":19,"sense_key":"head%1:15:02::","synset_id":"forefront.n.01"},{"index":20,"sense_key":"head%1:11:01::","synset_id":"pass.n.09"},{"index":21,"sense_key":"head%1:11:00::","synset_id":"headway.n.02"},{"index":22,"sense_key":"head%1:10:01::","synset_id":"point.n.20"},{"index":23,"sense_key":"head%1:10:02::","synset_id":"question.n.02"},{"index":24,"sense_key":"head%1:10:00::","synset_id":"heading.n.01"},{"index":25,"sense_key":"head%1:08:01::","synset_id":"head.n.25"},{"index":26,"sense_key":"head%1:08:02::","synset_id":"head.n.26"},{"index":27,"sense_key":"head%1:06:05::","synset_id":"read/write_head.n.01"},{"index":28,"sense_key":"head%1:06:04::","synset_id":"head.n.28"},{"index":29,"sense_key":"head%1:06:03::","synset_id":"head.n.29"},{"index":30,"sense_key":"head%1:06:02::","synset_id":"head.n.30"},{"index":31,"sense_key":"head%1:06:00::","synset_id":"head.n.31"},{"index":32,"sense_key":"head%1:06:01::","synset_id":"drumhead.n.01"},{"index":33,"sense_key":"head%1:04:00::","synset_id":"oral_sex.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":13,"chosen_sense_key":"head%1:18:01::","call_ids":["senseval2.d002.s077.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":13,"predicted_sense_key":"head%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2365,"cached_input_tokens":0,"output_tokens":85,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00558,"input_uncached_usd":0.00473,"input_cached_usd":0.0,"output_usd":0.0008500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4042345840716735}
{"item_id":"senseval2.d002.s078.t000","gold_sense_keys":["authority%1:07:00::"],"candidates":[{"index":1,"sense_key":"authority%1:07:00::","synset_id":"authority.n.01"},{"index":2,"sense_key":"authority%1:18:01::","synset_id":"authority.n.02"},{"index":3,"sense_key":"authority%1:18:00::","synset_id":"authority.n.03"},{"index":4,"sense_key":"authority%1:09:00::","synset_id":"assurance.n.01"},{"index":5,"sense_key":"authority%1:14:00::","synset_id":"agency.n.01"},{"index":6,"sense_key":"authority%1:07:02::","synset_id":"authority.n.06"},{"index":7,"sense_key":"authority%1:10:00::","synset_id":"authority.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"authority%1:07:00::","call_ids":["senseval2.d002.s078.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"authority%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1025,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0021699999999999996,"input_uncached_usd":0.0020499999999999997,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0477794170146808}
{"item_id":"senseval2.d002.s078.t002","gold_sense_keys":["go_with%2:42:01::"],"candidates":[{"index":1,"sense_key":"go_with%2:42:01::","synset_id":"attach_to.v.01"},{"index":2,"sense_key":"go_with%2:42:00::","synset_id":"collocate_with.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"go_with%2:42:01::","call_ids":["senseval2.d002.s078.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"go_with%2:42:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":703,"cached_input_tokens":0,"output_tokens":87,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0022760000000000002,"input_uncached_usd":0.001406,"input_cached_usd":0.0,"output_usd":0.0008700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1843668749788776}
{"item_id":"senseval2.d002.s078.t003","gold_sense_keys":["unambiguous%3:00:04::"],"candidates":[{"index":1,"sense_key":"unambiguous%3:00:00::","synset_id":"unambiguous.a.01"},{"index":2,"sense_key":"unambiguous%3:00:04::","synset_id":"unequivocal.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"unambiguous%3:00:04::","call_ids":["senseval2.d002.s078.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"unambiguous%3:00:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":728,"cached_input_tokens":0,"output_tokens":149,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002946,"input_uncached_usd":0.0014559999999999998,"input_cached_usd":0.0,"output_usd":0.0014900000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7631257080938667}
{"item_id":"senseval2.d002.s079.t000","gold_sense_keys":["school%1:14:00::"],"candidates":[{"index":1,"sense_key":"school%1:14:00::","synset_id":"school.n.01"},{"index":2,"sense_key":"school%1:06:00::","synset_id":"school.n.02"},{"index":3,"sense_key":"school%1:09:00::","synset_id":"school.n.03"},{"index":4,"sense_key":"school%1:14:02::","synset_id":"school.n.04"},{"index":5,"sense_key":"school%1:28:00::","synset_id":"school.n.05"},{"index":6,"sense_key":"school%1:14:03::","synset_id":"school.n.06"},{"index":7,"sense_key":"school%1:14:01::","synset_id":"school.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"school%1:14:00::","call_ids":["senseval2.d002.s079.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"school%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":962,"cached_input_tokens":0,"output_tokens":124,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003164,"input_uncached_usd":0.001924,"input_cached_usd":0.0,"output_usd":0.00124,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.861586541053839}
{"item_id":"senseval2.d002.s079.t002","gold_sense_keys":["way%1:04:01::","way%1:07:01::"],"candidates":[{"index":1,"sense_key":"way%1:07:01::","synset_id":"manner.n.01"},{"index":2,"sense_key":"way%1:04:01::","synset_id":"means.n.01"},{"index":3,"sense_key":"way%1:15:00::","synset_id":"direction.n.01"},{"index":4,"sense_key":"way%1:26:00::","synset_id":"way.n.04"},{"index":5,"sense_key":"way%1:04:00::","synset_id":"way.n.05"},{"index":6,"sense_key":"way%1:06:00::","synset_id":"way.n.06"},{"index":7,"sense_key":"way%1:04:02::","synset_id":"way.n.07"},{"index":8,"sense_key":"way%1:23:00::","synset_id":"room.n.02"},{"index":9,"sense_key":"way%1:07:02::","synset_id":"way.n.09"},{"index":10,"sense_key":"way%1:09:00::","synset_id":"way.n.10"},{"index":11,"sense_key":"way%1:09:01::","synset_id":"way.n.11"},{"index":12,"sense_key":"way%1:21:00::","synset_id":"way.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"way%1:07:01::","call_ids":["senseval2.d002.s079.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"way%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1180,"cached_input_tokens":0,"output_tokens":99,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0033499999999999997,"input_uncached_usd":0.0023599999999999997,"input_cached_usd":0.0,"output_usd":0.00099,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6709602921037003}
{"item_id":"senseval2.d002.s079.t003","gold_sense_keys":["produce%2:36:03::"],"candidates":[{"index":1,"sense_key":"produce%2:36:02::","synset_id":"produce.v.01"},{"index":2,"sense_key":"produce%2:36:00::","synset_id":"produce.v.02"},{"index":3,"sense_key":"produce%2:36:03::","synset_id":"produce.v.03"},{"index":4,"sense_key":"produce%2:39:01::","synset_id":"produce.v.04"},{"index":5,"sense_key":"produce%2:36:05::","synset_id":"grow.v.07"},{"index":6,"sense_key":"produce%2:39:00::","synset_id":"produce.v.06"},{"index":7,"sense_key":"produce%2:29:00::","synset_id":"grow.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"produce%2:36:03::","call_ids":["senseval2.d002.s079.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"produce%2:36:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1064,"cached_input_tokens":0,"output_tokens":242,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004548,"input_uncached_usd":0.002128,"input_cached_usd":0.0,"output_usd":0.0024200000000000003,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.133727916982025}
{"item_id":"senseval2.d002.s079.t004","gold_sense_keys":["student%1:18:00::"],"candidates":[{"index":1,"sense_key":"student%1:18:00::","synset_id":"student.n.01"},{"index":2,"sense_key":"student%1:18:01::","synset_id":"scholar.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"student%1:18:00::","call_ids":["senseval2.d002.s079.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"student%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":683,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0014860000000000001,"input_uncached_usd":0.001366,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4616880000103265}
{"item_id":"senseval2.d002.s079.t005","gold_sense_keys":["high%3:00:02::"],"candidates":[{"index":1,"sense_key":"high%3:00:02::","synset_id":"high.a.01"},{"index":2,"sense_key":"high%3:00:01::","synset_id":"high.a.02"},{"index":3,"sense_key":"high%5:00:00:superior:01","synset_id":"eminent.s.01"},{"index":4,"sense_key":"high%3:00:03::","synset_id":"high.a.04"},{"index":5,"sense_key":"high%5:00:00:elated:00","synset_id":"high.s.02"},{"index":6,"sense_key":"high%5:00:00:malodorous:00","synset_id":"gamey.s.02"},{"index":7,"sense_key":"high%5:00:00:intoxicated:00","synset_id":"high.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"high%3:00:02::","call_ids":["senseval2.d002.s079.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"high%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":971,"cached_input_tokens":0,"output_tokens":110,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003042,"input_uncached_usd":0.001942,"input_cached_usd":0.0,"output_usd":0.0011,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6033224170096219}
{"item_id":"senseval2.d002.s079.t006","gold_sense_keys":["morale%1:26:00::"],"candidates":[{"index":1,"sense_key":"morale%1:26:00::","synset_id":"morale.n.01"},{"index":2,"sense_key":"morale%1:07:00::","synset_id":"esprit_de_corps.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"morale%1:26:00::","call_ids":["senseval2.d002.s079.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"morale%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":673,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002006,"input_uncached_usd":0.001346,"input_cached_usd":0.0,"output_usd":0.0006600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5210372499423102}
{"item_id":"senseval2.d002.s079.t007","gold_sense_keys":["superior%3:00:02::"],"candidates":[{"index":1,"sense_key":"superior%3:00:02::","synset_id":"superior.a.01"},{"index":2,"sense_key":"superior%3:00:01::","synset_id":"superior.a.02"},{"index":3,"sense_key":"superior%5:00:00:dominant:01","synset_id":"superior.s.01"},{"index":4,"sense_key":"superior%3:00:00::","synset_id":"superscript.a.01"},{"index":5,"sense_key":"superior%3:00:03::","synset_id":"superior.a.05"},{"index":6,"sense_key":"superior%5:00:00:senior:00","synset_id":"ranking.s.01"},{"index":7,"sense_key":"superior%5:00:00:unaffected:00","synset_id":"superior.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"superior%3:00:02::","call_ids":["senseval2.d002.s079.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"superior%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":986,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0020919999999999997,"input_uncached_usd":0.001972,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1401097499765456}
{"item_id":"senseval2.d002.s079.t008","gold_sense_keys":["academic%3:01:00::"],"candidates":[{"index":1,"sense_key":"academic%3:01:00::","synset_id":"academic.a.01"},{"index":2,"sense_key":"academic%5:00:00:theoretical:01","synset_id":"academic.s.01"},{"index":3,"sense_key":"academic%5:00:00:scholarly:00","synset_id":"academic.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"academic%3:01:00::","call_ids":["senseval2.d002.s079.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"academic%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":745,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00205,"input_uncached_usd":0.00149,"input_cached_usd":0.0,"output_usd":0.0005600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4808842089259997}
{"item_id":"senseval2.d002.s079.t009","gold_sense_keys":["performance%1:04:00::","performance%1:04:03::"],"candidates":[{"index":1,"sense_key":"performance%1:10:00::","synset_id":"performance.n.01"},{"index":2,"sense_key":"performance%1:04:01::","synset_id":"performance.n.02"},{"index":3,"sense_key":"performance%1:04:00::","synset_id":"performance.n.03"},{"index":4,"sense_key":"performance%1:04:03::","synset_id":"performance.n.04"},{"index":5,"sense_key":"performance%1:22:00::","synset_id":"operation.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"performance%1:04:00::","call_ids":["senseval2.d002.s079.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"performance%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":948,"cached_input_tokens":0,"output_tokens":121,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0031060000000000003,"input_uncached_usd":0.001896,"input_cached_usd":0.0,"output_usd":0.0012100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.244194042054005}
{"item_id":"senseval2.d002.s080.t000","gold_sense_keys":["fact%1:10:01::"],"candidates":[{"index":1,"sense_key":"fact%1:09:01::","synset_id":"fact.n.01"},{"index":2,"sense_key":"fact%1:10:01::","synset_id":"fact.n.02"},{"index":3,"sense_key":"fact%1:26:01::","synset_id":"fact.n.03"},{"index":4,"sense_key":"fact%1:09:02::","synset_id":"fact.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"fact%1:09:02::","call_ids":["senseval2.d002.s080.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"fact%1:09:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":788,"cached_input_tokens":0,"output_tokens":158,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003156,"input_uncached_usd":0.001576,"input_cached_usd":0.0,"output_usd":0.00158,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9086671659024432}
{"item_id":"senseval2.d002.s080.t001","gold_sense_keys":["ruffle%2:37:01::"],"candidates":[{"index":1,"sense_key":"ruffle%2:38:00::","synset_id":"ripple.v.01"},{"index":2,"sense_key":"ruffle%2:37:00::","synset_id":"ruffle.v.02"},{"index":3,"sense_key":"ruffle%2:38:01::","synset_id":"tittup.v.01"},{"index":4,"sense_key":"ruffle%2:37:01::","synset_id":"ruffle.v.04"},{"index":5,"sense_key":"ruffle%2:35:03::","synset_id":"flick.v.06"},{"index":6,"sense_key":"ruffle%2:35:04::","synset_id":"shuffle.v.03"},{"index":7,"sense_key":"ruffle%2:35:01::","synset_id":"ruffle.v.07"},{"index":8,"sense_key":"ruffle%2:35:02::","synset_id":"ruffle.v.08"},{"index":9,"sense_key":"ruffle%2:35:00::","synset_id":"ruffle.v.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"ruffle%2:37:01::","call_ids":["senseval2.d002.s080.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"ruffle%2:37:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1102,"cached_input_tokens":0,"output_tokens":139,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003594,"input_uncached_usd":0.002204,"input_cached_usd":0.0,"output_usd":0.0013900000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7645339579321444}
{"item_id":"senseval2.d002.s080.t002","gold_sense_keys":["fact%1:10:01::"],"candidates":[{"index":1,"sense_key":"fact%1:09:01::","synset_id":"fact.n.01"},{"index":2,"sense_key":"fact%1:10:01::","synset_id":"fact.n.02"},{"index":3,"sense_key":"fact%1:26:01::","synset_id":"fact.n.03"},{"index":4,"sense_key":"fact%1:09:02::","synset_id":"fact.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"fact%1:09:02::","call_ids":["senseval2.d002.s080.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"fact%1:09:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":788,"cached_input_tokens":0,"output_tokens":161,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003186,"input_uncached_usd":0.001576,"input_cached_usd":0.0,"output_usd":0.00161,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0091971250949427}
{"item_id":"senseval2.d002.s080.t004","gold_sense_keys":["one%1:09:00::"],"candidates":[{"index":1,"sense_key":"one%1:23:00::","synset_id":"one.n.01"},{"index":2,"sense_key":"one%1:09:00::","synset_id":"one.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"one%1:09:00::","call_ids":["senseval2.d002.s080.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"one%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":672,"cached_input_tokens":0,"output_tokens":108,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002424,"input_uncached_usd":0.001344,"input_cached_usd":0.0,"output_usd":0.00108,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.78191258397419}
{"item_id":"senseval2.d002.s080.t005","gold_sense_keys":["so%4:02:02::"],"candidates":[{"index":1,"sense_key":"so%4:02:02::","synset_id":"so.r.01"},{"index":2,"sense_key":"so%4:02:06::","synset_id":"so.r.02"},{"index":3,"sense_key":"so%4:02:05::","synset_id":"so.r.03"},{"index":4,"sense_key":"so%4:02:07::","synset_id":"so.r.04"},{"index":5,"sense_key":"so%4:02:08::","synset_id":"so.r.05"},{"index":6,"sense_key":"so%4:02:04::","synset_id":"thus.r.02"},{"index":7,"sense_key":"so%4:02:03::","synset_id":"so.r.07"},{"index":8,"sense_key":"so%4:02:09::","synset_id":"then.r.01"},{"index":9,"sense_key":"so%4:02:10::","synset_id":"therefore.r.01"},{"index":10,"sense_key":"so%4:02:00::","synset_id":"indeed.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"so%4:02:02::","call_ids":["senseval2.d002.s080.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"so%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1208,"cached_input_tokens":0,"output_tokens":120,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003616,"input_uncached_usd":0.0024159999999999997,"input_cached_usd":0.0,"output_usd":0.0012000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1077402079245076}
{"item_id":"senseval2.d002.s081.t000","gold_sense_keys":["publish%2:32:00::"],"candidates":[{"index":1,"sense_key":"publish%2:36:01::","synset_id":"print.v.01"},{"index":2,"sense_key":"publish%2:32:00::","synset_id":"publish.v.02"},{"index":3,"sense_key":"publish%2:36:00::","synset_id":"publish.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"publish%2:32:00::","call_ids":["senseval2.d002.s081.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"publish%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":657,"cached_input_tokens":0,"output_tokens":96,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002274,"input_uncached_usd":0.0013139999999999998,"input_cached_usd":0.0,"output_usd":0.0009600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0420109159313142}
{"item_id":"senseval3.d000.s000.t000","gold_sense_keys":["man%1:18:00::"],"candidates":[{"index":1,"sense_key":"man%1:18:00::","synset_id":"man.n.01"},{"index":2,"sense_key":"man%1:18:03::","synset_id":"serviceman.n.01"},{"index":3,"sense_key":"man%1:18:04::","synset_id":"man.n.03"},{"index":4,"sense_key":"man%1:05:01::","synset_id":"homo.n.02"},{"index":5,"sense_key":"man%1:18:08::","synset_id":"man.n.05"},{"index":6,"sense_key":"man%1:18:06::","synset_id":"man.n.06"},{"index":7,"sense_key":"man%1:18:05::","synset_id":"valet.n.01"},{"index":8,"sense_key":"man%1:18:07::","synset_id":"man.n.08"},{"index":9,"sense_key":"man%1:15:00::","synset_id":"man.n.09"},{"index":10,"sense_key":"man%1:06:00::","synset_id":"man.n.10"},{"index":11,"sense_key":"man%1:05:00::","synset_id":"world.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"man%1:18:00::","call_ids":["senseval3.d000.s000.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"man%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1009,"cached_input_tokens":0,"output_tokens":49,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002508,"input_uncached_usd":0.002018,"input_cached_usd":0.0,"output_usd":0.0004900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8125960000324994}
{"item_id":"senseval3.d000.s001.t003","gold_sense_keys":["eye%1:08:00::"],"candidates":[{"index":1,"sense_key":"eye%1:08:00::","synset_id":"eye.n.01"},{"index":2,"sense_key":"eye%1:09:00::","synset_id":"eye.n.02"},{"index":3,"sense_key":"eye%1:09:01::","synset_id":"eye.n.03"},{"index":4,"sense_key":"eye%1:15:00::","synset_id":"center.n.01"},{"index":5,"sense_key":"eye%1:06:00::","synset_id":"eye.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"eye%1:08:00::","call_ids":["senseval3.d000.s001.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"eye%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":567,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0012540000000000001,"input_uncached_usd":0.001134,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.7008065419504419}
{"item_id":"senseval3.d000.s002.t000","gold_sense_keys":["have%2:40:00::"],"candidates":[{"index":1,"sense_key":"have%2:40:00::","synset_id":"have.v.01"},{"index":2,"sense_key":"have%2:42:00::","synset_id":"have.v.02"},{"index":3,"sense_key":"have%2:39:06::","synset_id":"experience.v.03"},{"index":4,"sense_key":"have%2:40:04::","synset_id":"own.v.01"},{"index":5,"sense_key":"have%2:30:00::","synset_id":"get.v.03"},{"index":6,"sense_key":"have%2:34:00::","synset_id":"consume.v.02"},{"index":7,"sense_key":"have%2:41:00::","synset_id":"have.v.07"},{"index":8,"sense_key":"have%2:36:00::","synset_id":"hold.v.03"},{"index":9,"sense_key":"have%2:40:01::","synset_id":"have.v.09"},{"index":10,"sense_key":"have%2:42:12::","synset_id":"have.v.10"},{"index":11,"sense_key":"have%2:30:01::","synset_id":"have.v.11"},{"index":12,"sense_key":"have%2:29:02::","synset_id":"have.v.12"},{"index":13,"sense_key":"have%2:32:00::","synset_id":"induce.v.02"},{"index":14,"sense_key":"have%2:40:05::","synset_id":"accept.v.02"},{"index":15,"sense_key":"have%2:40:03::","synset_id":"receive.v.01"},{"index":16,"sense_key":"have%2:29:05::","synset_id":"suffer.v.02"},{"index":17,"sense_key":"have%2:40:02::","synset_id":"have.v.17"},{"index":18,"sense_key":"have%2:29:00::","synset_id":"give_birth.v.01"},{"index":19,"sense_key":"have%2:35:00::","synset_id":"take.v.35"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"have%2:40:00::","call_ids":["senseval3.d000.s002.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"have%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1475,"cached_input_tokens":0,"output_tokens":74,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00369,"input_uncached_usd":0.00295,"input_cached_usd":0.0,"output_usd":0.0007400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4034053329378366}
{"item_id":"senseval3.d000.s002.t002","gold_sense_keys":["answer%1:10:01::"],"candidates":[{"index":1,"sense_key":"answer%1:10:01::","synset_id":"answer.n.01"},{"index":2,"sense_key":"answer%1:10:02::","synset_id":"solution.n.02"},{"index":3,"sense_key":"answer%1:10:00::","synset_id":"answer.n.03"},{"index":4,"sense_key":"answer%1:10:03::","synset_id":"answer.n.04"},{"index":5,"sense_key":"answer%1:04:00::","synset_id":"answer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"answer%1:10:01::","call_ids":["senseval3.d000.s002.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"answer%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":712,"cached_input_tokens":0,"output_tokens":91,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002334,"input_uncached_usd":0.001424,"input_cached_usd":0.0,"output_usd":0.0009100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7096829160582274}
{"item_id":"senseval3.d000.s002.t004","gold_sense_keys":["surprise%1:12:00::"],"candidates":[{"index":1,"sense_key":"surprise%1:12:00::","synset_id":"surprise.n.01"},{"index":2,"sense_key":"surprise%1:11:00::","synset_id":"surprise.n.02"},{"index":3,"sense_key":"surprise%1:04:00::","synset_id":"surprise.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"surprise%1:12:00::","call_ids":["senseval3.d000.s002.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"surprise%1:12:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":426,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.000972,"input_uncached_usd":0.000852,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.966531457961537}
{"item_id":"senseval3.d000.s002.t005","gold_sense_keys":["fit%1:26:00::"],"candidates":[{"index":1,"sense_key":"fit%1:26:01::","synset_id":"fit.n.01"},{"index":2,"sense_key":"fit%1:26:00::","synset_id":"paroxysm.n.01"},{"index":3,"sense_key":"fit%1:07:00::","synset_id":"fit.n.03"},{"index":4,"sense_key":"fit%1:04:00::","synset_id":"burst.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"fit%1:26:00::","call_ids":["senseval3.d000.s002.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"fit%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":558,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0012360000000000001,"input_uncached_usd":0.001116,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0885858329711482}
{"item_id":"senseval3.d000.s003.t000","gold_sense_keys":["man%1:18:00::"],"candidates":[{"index":1,"sense_key":"man%1:18:00::","synset_id":"man.n.01"},{"index":2,"sense_key":"man%1:18:03::","synset_id":"serviceman.n.01"},{"index":3,"sense_key":"man%1:18:04::","synset_id":"man.n.03"},{"index":4,"sense_key":"man%1:05:01::","synset_id":"homo.n.02"},{"index":5,"sense_key":"man%1:18:08::","synset_id":"man.n.05"},{"index":6,"sense_key":"man%1:18:06::","synset_id":"man.n.06"},{"index":7,"sense_key":"man%1:18:05::","synset_id":"valet.n.01"},{"index":8,"sense_key":"man%1:18:07::","synset_id":"man.n.08"},{"index":9,"sense_key":"man%1:15:00::","synset_id":"man.n.09"},{"index":10,"sense_key":"man%1:06:00::","synset_id":"man.n.10"},{"index":11,"sense_key":"man%1:05:00::","synset_id":"world.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"man%1:18:00::","call_ids":["senseval3.d000.s003.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"man%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1082,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0022839999999999996,"input_uncached_usd":0.0021639999999999997,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.7546284589916468}
{"item_id":"senseval3.d000.s003.t001","gold_sense_keys":["drunk%3:00:00::"],"candidates":[{"index":1,"sense_key":"drunk%3:00:00::","synset_id":"intoxicated.a.01"},{"index":2,"sense_key":"drunk%5:00:00:excited:00","synset_id":"intoxicated.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"drunk%3:00:00::","call_ids":["senseval3.d000.s003.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"drunk%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":506,"cached_input_tokens":0,"output_tokens":49,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0015019999999999999,"input_uncached_usd":0.0010119999999999999,"input_cached_usd":0.0,"output_usd":0.0004900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1578720420366153}
{"item_id":"senseval3.d000.s003.t002","gold_sense_keys":["crazy%5:00:00:insane:00"],"candidates":[{"index":1,"sense_key":"crazy%5:00:00:insane:00","synset_id":"brainsick.s.01"},{"index":2,"sense_key":"crazy%5:00:00:impractical:00","synset_id":"crazy.s.02"},{"index":3,"sense_key":"crazy%5:00:00:excited:00","synset_id":"crazy.s.03"},{"index":4,"sense_key":"crazy%5:00:00:strange:00","synset_id":"crazy.s.04"},{"index":5,"sense_key":"crazy%5:00:00:enthusiastic:00","synset_id":"crazy.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"crazy%5:00:00:insane:00","call_ids":["senseval3.d000.s003.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"crazy%5:00:00:insane:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":701,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0015220000000000001,"input_uncached_usd":0.001402,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.7542053329525515}
{"item_id":"senseval3.d000.s004.t002","gold_sense_keys":["match%2:42:01::"],"candidates":[{"index":1,"sense_key":"match%2:42:00::","synset_id":"match.v.01"},{"index":2,"sense_key":"match%2:40:00::","synset_id":"match.v.02"},{"index":3,"sense_key":"match%2:35:08::","synset_id":"match.v.03"},{"index":4,"sense_key":"match%2:42:01::","synset_id":"equal.v.02"},{"index":5,"sense_key":"match%2:30:01::","synset_id":"match.v.05"},{"index":6,"sense_key":"match%2:41:00::","synset_id":"meet.v.06"},{"index":7,"sense_key":"match%2:35:10::","synset_id":"match.v.07"},{"index":8,"sense_key":"match%2:33:00::","synset_id":"pit.v.01"},{"index":9,"sense_key":"match%2:30:00::","synset_id":"match.v.09"},{"index":10,"sense_key":"match%2:30:07::","synset_id":"equal.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"match%2:42:01::","call_ids":["senseval3.d000.s004.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"match%2:42:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1023,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0027660000000000002,"input_uncached_usd":0.002046,"input_cached_usd":0.0,"output_usd":0.00072,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3246300409082323}
{"item_id":"senseval3.d000.s004.t003","gold_sense_keys":["drink%1:13:04::"],"candidates":[{"index":1,"sense_key":"drink%1:13:04::","synset_id":"drink.n.01"},{"index":2,"sense_key":"drink%1:04:00::","synset_id":"drink.n.02"},{"index":3,"sense_key":"drink%1:13:00::","synset_id":"beverage.n.01"},{"index":4,"sense_key":"drink%1:17:00::","synset_id":"drink.n.04"},{"index":5,"sense_key":"drink%1:04:01::","synset_id":"swallow.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"drink%1:13:04::","call_ids":["senseval3.d000.s004.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"drink%1:13:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":687,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001964,"input_uncached_usd":0.001374,"input_cached_usd":0.0,"output_usd":0.00059,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5702873750124127}
{"item_id":"senseval3.d000.s004.t004","gold_sense_keys":["drink%1:13:04::"],"candidates":[{"index":1,"sense_key":"drink%1:13:04::","synset_id":"drink.n.01"},{"index":2,"sense_key":"drink%1:04:00::","synset_id":"drink.n.02"},{"index":3,"sense_key":"drink%1:13:00::","synset_id":"beverage.n.01"},{"index":4,"sense_key":"drink%1:17:00::","synset_id":"drink.n.04"},{"index":5,"sense_key":"drink%1:04:01::","synset_id":"swallow.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"drink%1:13:04::","call_ids":["senseval3.d000.s004.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"drink%1:13:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":687,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001974,"input_uncached_usd":0.001374,"input_cached_usd":0.0,"output_usd":0.0006000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.866473958012648}
{"item_id":"senseval3.d000.s004.t005","gold_sense_keys":["lose%2:40:02::"],"candidates":[{"index":1,"sense_key":"lose%2:40:02::","synset_id":"lose.v.01"},{"index":2,"sense_key":"lose%2:33:00::","synset_id":"lose.v.02"},{"index":3,"sense_key":"lose%2:37:00::","synset_id":"lose.v.03"},{"index":4,"sense_key":"lose%2:35:00::","synset_id":"misplace.v.01"},{"index":5,"sense_key":"lose%2:40:01::","synset_id":"lose.v.05"},{"index":6,"sense_key":"lose%2:39:00::","synset_id":"lose.v.06"},{"index":7,"sense_key":"lose%2:40:06::","synset_id":"lose.v.07"},{"index":8,"sense_key":"lose%2:40:00::","synset_id":"lose.v.08"},{"index":9,"sense_key":"lose%2:33:01::","synset_id":"fall_back.v.04"},{"index":10,"sense_key":"lose%2:39:01::","synset_id":"miss.v.01"},{"index":11,"sense_key":"lose%2:30:03::","synset_id":"suffer.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"lose%2:40:02::","call_ids":["senseval3.d000.s004.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"lose%2:40:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1027,"cached_input_tokens":0,"output_tokens":146,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0035139999999999998,"input_uncached_usd":0.002054,"input_cached_usd":0.0,"output_usd":0.0014600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8725693749729544}
{"item_id":"senseval3.d000.s004.t006","gold_sense_keys":["count%1:23:00::"],"candidates":[{"index":1,"sense_key":"count%1:23:00::","synset_id":"count.n.01"},{"index":2,"sense_key":"count%1:04:00::","synset_id":"count.n.02"},{"index":3,"sense_key":"count%1:18:00::","synset_id":"count.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"count%1:04:00::","call_ids":["senseval3.d000.s004.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"count%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":522,"cached_input_tokens":0,"output_tokens":83,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0018740000000000002,"input_uncached_usd":0.001044,"input_cached_usd":0.0,"output_usd":0.0008300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7162414999911562}
{"item_id":"senseval3.d000.s004.t007","gold_sense_keys":["man%1:18:00::"],"candidates":[{"index":1,"sense_key":"man%1:18:00::","synset_id":"man.n.01"},{"index":2,"sense_key":"man%1:18:03::","synset_id":"serviceman.n.01"},{"index":3,"sense_key":"man%1:18:04::","synset_id":"man.n.03"},{"index":4,"sense_key":"man%1:05:01::","synset_id":"homo.n.02"},{"index":5,"sense_key":"man%1:18:08::","synset_id":"man.n.05"},{"index":6,"sense_key":"man%1:18:06::","synset_id":"man.n.06"},{"index":7,"sense_key":"man%1:18:05::","synset_id":"valet.n.01"},{"index":8,"sense_key":"man%1:18:07::","synset_id":"man.n.08"},{"index":9,"sense_key":"man%1:15:00::","synset_id":"man.n.09"},{"index":10,"sense_key":"man%1:06:00::","synset_id":"man.n.10"},{"index":11,"sense_key":"man%1:05:00::","synset_id":"world.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"man%1:18:00::","call_ids":["senseval3.d000.s004.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"man%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1112,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0023439999999999997,"input_uncached_usd":0.002224,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0532160829752684}
{"item_id":"senseval3.d000.s004.t008","gold_sense_keys":["eye%1:08:00::"],"candidates":[{"index":1,"sense_key":"eye%1:08:00::","synset_id":"eye.n.01"},{"index":2,"sense_key":"eye%1:09:00::","synset_id":"eye.n.02"},{"index":3,"sense_key":"eye%1:09:01::","synset_id":"eye.n.03"},{"index":4,"sense_key":"eye%1:15:00::","synset_id":"center.n.01"},{"index":5,"sense_key":"eye%1:06:00::","synset_id":"eye.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"eye%1:08:00::","call_ids":["senseval3.d000.s004.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"eye%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":645,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00141,"input_uncached_usd":0.00129,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0405633330810815}
{"item_id":"senseval3.d000.s005.t000","gold_sense_keys":["guy%1:18:00::"],"candidates":[{"index":1,"sense_key":"guy%1:18:00::","synset_id":"guy.n.01"},{"index":2,"sense_key":"guy%1:06:01::","synset_id":"guy.n.02"},{"index":3,"sense_key":"guy%1:06:00::","synset_id":"guy.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"guy%1:18:00::","call_ids":["senseval3.d000.s005.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"guy%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":599,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001318,"input_uncached_usd":0.0011979999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3947672500507906}
{"item_id":"senseval3.d000.s005.t002","gold_sense_keys":["think%2:31:01::","think%2:31:03::","think%2:31:12::"],"candidates":[{"index":1,"sense_key":"think%2:31:01::","synset_id":"think.v.01"},{"index":2,"sense_key":"think%2:31:03::","synset_id":"think.v.02"},{"index":3,"sense_key":"think%2:31:00::","synset_id":"think.v.03"},{"index":4,"sense_key":"think%2:31:02::","synset_id":"remember.v.01"},{"index":5,"sense_key":"think%2:36:00::","synset_id":"think.v.05"},{"index":6,"sense_key":"think%2:31:09::","synset_id":"think.v.06"},{"index":7,"sense_key":"think%2:31:06::","synset_id":"intend.v.01"},{"index":8,"sense_key":"think%2:31:12::","synset_id":"think.v.08"},{"index":9,"sense_key":"think%2:31:13::","synset_id":"think.v.09"},{"index":10,"sense_key":"think%2:31:15::","synset_id":"think.v.10"},{"index":11,"sense_key":"think%2:31:14::","synset_id":"think.v.11"},{"index":12,"sense_key":"think%2:31:08::","synset_id":"think.v.12"},{"index":13,"sense_key":"think%2:30:00::","synset_id":"think.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"think%2:31:01::","call_ids":["senseval3.d000.s005.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"think%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1196,"cached_input_tokens":0,"output_tokens":302,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.005412,"input_uncached_usd":0.002392,"input_cached_usd":0.0,"output_usd":0.00302,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.0042167080100626}
{"item_id":"senseval3.d000.s005.t004","gold_sense_keys":["eye%1:08:00::"],"candidates":[{"index":1,"sense_key":"eye%1:08:00::","synset_id":"eye.n.01"},{"index":2,"sense_key":"eye%1:09:00::","synset_id":"eye.n.02"},{"index":3,"sense_key":"eye%1:09:01::","synset_id":"eye.n.03"},{"index":4,"sense_key":"eye%1:15:00::","synset_id":"center.n.01"},{"index":5,"sense_key":"eye%1:06:00::","synset_id":"eye.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"eye%1:08:00::","call_ids":["senseval3.d000.s005.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"eye%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":680,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00148,"input_uncached_usd":0.0013599999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1124526250641793}
{"item_id":"senseval3.d000.s006.t000","gold_sense_keys":["eye%1:08:00::"],"candidates":[{"index":1,"sense_key":"eye%1:08:00::","synset_id":"eye.n.01"},{"index":2,"sense_key":"eye%1:09:00::","synset_id":"eye.n.02"},{"index":3,"sense_key":"eye%1:09:01::","synset_id":"eye.n.03"},{"index":4,"sense_key":"eye%1:15:00::","synset_id":"center.n.01"},{"index":5,"sense_key":"eye%1:06:00::","synset_id":"eye.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"eye%1:08:00::","call_ids":["senseval3.d000.s006.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"eye%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":686,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001492,"input_uncached_usd":0.001372,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7535704170586541}
{"item_id":"senseval3.d000.s006.t002","gold_sense_keys":["bright%3:00:00::"],"candidates":[{"index":1,"sense_key":"bright%3:00:00::","synset_id":"bright.a.01"},{"index":2,"sense_key":"bright%5:00:00:colorful:00","synset_id":"bright.s.01"},{"index":3,"sense_key":"bright%5:00:00:intelligent:00","synset_id":"bright.s.02"},{"index":4,"sense_key":"bright%5:00:00:light:06","synset_id":"bright.s.03"},{"index":5,"sense_key":"bright%5:00:00:polished:00","synset_id":"bright.s.04"},{"index":6,"sense_key":"bright%5:00:00:glorious:00","synset_id":"bright.s.05"},{"index":7,"sense_key":"bright%3:00:02::","synset_id":"undimmed.a.01"},{"index":8,"sense_key":"bright%5:00:00:reverberant:00","synset_id":"bright.s.06"},{"index":9,"sense_key":"bright%5:00:00:happy:00","synset_id":"bright.s.07"},{"index":10,"sense_key":"bright%5:00:00:auspicious:00","synset_id":"bright.s.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bright%3:00:00::","call_ids":["senseval3.d000.s006.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bright%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1111,"cached_input_tokens":0,"output_tokens":211,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004332000000000001,"input_uncached_usd":0.002222,"input_cached_usd":0.0,"output_usd":0.0021100000000000003,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.3689377499977127}
{"item_id":"senseval3.d000.s006.t003","gold_sense_keys":["strange%3:00:00::"],"candidates":[{"index":1,"sense_key":"strange%3:00:00::","synset_id":"strange.a.01"},{"index":2,"sense_key":"strange%5:00:01:unfamiliar:00","synset_id":"strange.s.01"},{"index":3,"sense_key":"strange%3:00:01::","synset_id":"foreign.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"strange%3:00:00::","call_ids":["senseval3.d000.s006.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"strange%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":630,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00138,"input_uncached_usd":0.0012599999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4826955000171438}
{"item_id":"senseval3.d000.s006.t004","gold_sense_keys":["intensity%1:07:00::"],"candidates":[{"index":1,"sense_key":"intensity%1:07:03::","synset_id":"intensity.n.01"},{"index":2,"sense_key":"intensity%1:07:00::","synset_id":"intensity.n.02"},{"index":3,"sense_key":"intensity%1:07:02::","synset_id":"volume.n.06"},{"index":4,"sense_key":"intensity%1:07:01::","synset_id":"saturation.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"intensity%1:07:00::","call_ids":["senseval3.d000.s006.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"intensity%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":684,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001488,"input_uncached_usd":0.0013679999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1761332500027493}
{"item_id":"senseval3.d000.s006.t005","gold_sense_keys":["cold%3:00:02::"],"candidates":[{"index":1,"sense_key":"cold%3:00:01::","synset_id":"cold.a.01"},{"index":2,"sense_key":"cold%3:00:02::","synset_id":"cold.a.02"},{"index":3,"sense_key":"cold%5:00:00:stale:00","synset_id":"cold.s.01"},{"index":4,"sense_key":"cold%5:00:00:cool:03","synset_id":"cold.s.02"},{"index":5,"sense_key":"cold%5:00:00:perfect:00","synset_id":"cold.s.03"},{"index":6,"sense_key":"cold%5:00:00:unoriginal:00","synset_id":"cold.s.04"},{"index":7,"sense_key":"cold%5:00:00:intense:00","synset_id":"cold.s.05"},{"index":8,"sense_key":"cold%5:00:00:unloving:00","synset_id":"cold.s.06"},{"index":9,"sense_key":"cold%5:00:00:inhumane:00","synset_id":"cold.s.07"},{"index":10,"sense_key":"cold%5:00:00:unenthusiastic:00","synset_id":"cold.s.08"},{"index":11,"sense_key":"cold%5:00:00:unconscious:00","synset_id":"cold.s.09"},{"index":12,"sense_key":"cold%5:00:00:far:00","synset_id":"cold.s.10"},{"index":13,"sense_key":"cold%5:00:00:dead:01","synset_id":"cold.s.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"cold%5:00:00:intense:00","call_ids":["senseval3.d000.s006.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"cold%5:00:00:intense:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1207,"cached_input_tokens":0,"output_tokens":130,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0037140000000000003,"input_uncached_usd":0.002414,"input_cached_usd":0.0,"output_usd":0.0013000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.014465417014435}
{"item_id":"senseval3.d000.s006.t006","gold_sense_keys":["fire%1:12:00::"],"candidates":[{"index":1,"sense_key":"fire%1:11:00::","synset_id":"fire.n.01"},{"index":2,"sense_key":"fire%1:04:00::","synset_id":"fire.n.02"},{"index":3,"sense_key":"fire%1:22:00::","synset_id":"fire.n.03"},{"index":4,"sense_key":"fire%1:06:00::","synset_id":"fire.n.04"},{"index":5,"sense_key":"fire%1:27:01::","synset_id":"fire.n.05"},{"index":6,"sense_key":"fire%1:12:00::","synset_id":"ardor.n.03"},{"index":7,"sense_key":"fire%1:27:00::","synset_id":"fire.n.07"},{"index":8,"sense_key":"fire%1:11:01::","synset_id":"fire.n.08"},{"index":9,"sense_key":"fire%1:10:00::","synset_id":"fire.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"fire%1:12:00::","call_ids":["senseval3.d000.s006.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"fire%1:12:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":974,"cached_input_tokens":0,"output_tokens":84,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0027879999999999997,"input_uncached_usd":0.0019479999999999999,"input_cached_usd":0.0,"output_usd":0.00084,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.388351249974221}
{"item_id":"senseval3.d000.s006.t007","gold_sense_keys":["burn%2:43:01::"],"candidates":[{"index":1,"sense_key":"burn%2:30:02::","synset_id":"burn.v.01"},{"index":2,"sense_key":"burn%2:43:01::","synset_id":"burn.v.02"},{"index":3,"sense_key":"burn%2:30:00::","synset_id":"burn.v.03"},{"index":4,"sense_key":"burn%2:39:00::","synset_id":"bite.v.02"},{"index":5,"sense_key":"burn%2:43:02::","synset_id":"burn.v.05"},{"index":6,"sense_key":"burn%2:37:00::","synset_id":"burn.v.06"},{"index":7,"sense_key":"burn%2:30:01::","synset_id":"burn.v.07"},{"index":8,"sense_key":"burn%2:41:00::","synset_id":"burn.v.08"},{"index":9,"sense_key":"burn%2:40:01::","synset_id":"burn.v.09"},{"index":10,"sense_key":"burn%2:39:04::","synset_id":"burn.v.10"},{"index":11,"sense_key":"burn%2:30:08::","synset_id":"cauterize.v.01"},{"index":12,"sense_key":"burn%2:29:00::","synset_id":"sunburn.v.01"},{"index":13,"sense_key":"burn%2:36:05::","synset_id":"cut.v.21"},{"index":14,"sense_key":"burn%2:34:00::","synset_id":"burn_off.v.01"},{"index":15,"sense_key":"burn%2:30:03::","synset_id":"burn.v.15"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"burn%2:43:01::","call_ids":["senseval3.d000.s006.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"burn%2:43:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1244,"cached_input_tokens":0,"output_tokens":152,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004007999999999999,"input_uncached_usd":0.0024879999999999998,"input_cached_usd":0.0,"output_usd":0.00152,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2493545000907034}
{"item_id":"senseval3.d000.s007.t000","gold_sense_keys":["notice%2:39:00::"],"candidates":[{"index":1,"sense_key":"notice%2:39:04::","synset_id":"detect.v.01"},{"index":2,"sense_key":"notice%2:39:00::","synset_id":"notice.v.02"},{"index":3,"sense_key":"notice%2:32:00::","synset_id":"comment.v.01"},{"index":4,"sense_key":"notice%2:32:01::","synset_id":"notice.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"notice%2:39:00::","call_ids":["senseval3.d000.s007.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"notice%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":683,"cached_input_tokens":0,"output_tokens":130,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002666,"input_uncached_usd":0.001366,"input_cached_usd":0.0,"output_usd":0.0013000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.849915915983729}
{"item_id":"senseval3.d000.s008.t000","gold_sense_keys":["man%1:18:00::"],"candidates":[{"index":1,"sense_key":"man%1:18:00::","synset_id":"man.n.01"},{"index":2,"sense_key":"man%1:18:03::","synset_id":"serviceman.n.01"},{"index":3,"sense_key":"man%1:18:04::","synset_id":"man.n.03"},{"index":4,"sense_key":"man%1:05:01::","synset_id":"homo.n.02"},{"index":5,"sense_key":"man%1:18:08::","synset_id":"man.n.05"},{"index":6,"sense_key":"man%1:18:06::","synset_id":"man.n.06"},{"index":7,"sense_key":"man%1:18:05::","synset_id":"valet.n.01"},{"index":8,"sense_key":"man%1:18:07::","synset_id":"man.n.08"},{"index":9,"sense_key":"man%1:15:00::","synset_id":"man.n.09"},{"index":10,"sense_key":"man%1:06:00::","synset_id":"man.n.10"},{"index":11,"sense_key":"man%1:05:00::","synset_id":"world.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"man%1:18:00::","call_ids":["senseval3.d000.s008.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"man%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1121,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002362,"input_uncached_usd":0.002242,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":4.8572730829473585}
{"item_id":"senseval3.d000.s008.t001","gold_sense_keys":["drunk%3:00:00::"],"candidates":[{"index":1,"sense_key":"drunk%3:00:00::","synset_id":"intoxicated.a.01"},{"index":2,"sense_key":"drunk%5:00:00:excited:00","synset_id":"intoxicated.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"drunk%3:00:00::","call_ids":["senseval3.d000.s008.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"drunk%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":545,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00181,"input_uncached_usd":0.00109,"input_cached_usd":0.0,"output_usd":0.00072,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":5.973928416962735}
{"item_id":"senseval3.d000.s009.t000","gold_sense_keys":["wonder%2:31:00::"],"candidates":[{"index":1,"sense_key":"wonder%2:31:00::","synset_id":"wonder.v.01"},{"index":2,"sense_key":"wonder%2:32:01::","synset_id":"wonder.v.02"},{"index":3,"sense_key":"wonder%2:32:12::","synset_id":"wonder.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"wonder%2:31:00::","call_ids":["senseval3.d000.s009.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"wonder%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":577,"cached_input_tokens":0,"output_tokens":163,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002784,"input_uncached_usd":0.0011539999999999999,"input_cached_usd":0.0,"output_usd":0.0016300000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.3923122079577297}
{"item_id":"senseval3.d000.s009.t001","gold_sense_keys":["get%2:30:00::"],"candidates":[{"index":1,"sense_key":"get%2:40:00::","synset_id":"get.v.01"},{"index":2,"sense_key":"get%2:30:00::","synset_id":"become.v.01"},{"index":3,"sense_key":"get%2:30:02::","synset_id":"get.v.03"},{"index":4,"sense_key":"get%2:30:03::","synset_id":"receive.v.02"},{"index":5,"sense_key":"get%2:38:00::","synset_id":"arrive.v.01"},{"index":6,"sense_key":"get%2:35:03::","synset_id":"bring.v.04"},{"index":7,"sense_key":"get%2:39:00::","synset_id":"experience.v.03"},{"index":8,"sense_key":"get%2:33:00::","synset_id":"pay_back.v.02"},{"index":9,"sense_key":"get%2:40:01::","synset_id":"have.v.17"},{"index":10,"sense_key":"get%2:32:00::","synset_id":"induce.v.02"},{"index":11,"sense_key":"get%2:35:09::","synset_id":"get.v.11"},{"index":12,"sense_key":"get%2:29:01::","synset_id":"grow.v.08"},{"index":13,"sense_key":"get%2:29:00::","synset_id":"contract.v.04"},{"index":14,"sense_key":"get%2:32:03::","synset_id":"get.v.14"},{"index":15,"sense_key":"get%2:30:01::","synset_id":"make.v.02"},{"index":16,"sense_key":"get%2:32:08::","synset_id":"drive.v.11"},{"index":17,"sense_key":"get%2:31:09::","synset_id":"catch.v.18"},{"index":18,"sense_key":"get%2:35:00::","synset_id":"catch.v.07"},{"index":19,"sense_key":"get%2:35:10::","synset_id":"get.v.19"},{"index":20,"sense_key":"get%2:31:03::","synset_id":"get.v.20"},{"index":21,"sense_key":"get%2:40:14::","synset_id":"get.v.21"},{"index":22,"sense_key":"get%2:40:13::","synset_id":"get.v.22"},{"index":23,"sense_key":"get%2:39:13::","synset_id":"catch.v.21"},{"index":24,"sense_key":"get%2:39:09::","synset_id":"catch.v.22"},{"index":25,"sense_key":"get%2:39:14::","synset_id":"get.v.25"},{"index":26,"sense_key":"get%2:38:15::","synset_id":"scram.v.01"},{"index":27,"sense_key":"get%2:38:12::","synset_id":"get.v.27"},{"index":28,"sense_key":"get%2:37:12::","synset_id":"get.v.28"},{"index":29,"sense_key":"get%2:37:11::","synset_id":"get.v.29"},{"index":30,"sense_key":"get%2:36:14::","synset_id":"catch.v.24"},{"index":31,"sense_key":"get%2:36:00::","synset_id":"draw.v.15"},{"index":32,"sense_key":"get%2:35:12::","synset_id":"get.v.32"},{"index":33,"sense_key":"get%2:31:10::","synset_id":"perplex.v.01"},{"index":34,"sense_key":"get%2:30:12::","synset_id":"get_down.v.07"},{"index":35,"sense_key":"get%2:29:11::","synset_id":"suffer.v.02"},{"index":36,"sense_key":"get%2:29:08::","synset_id":"beget.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"get%2:30:00::","call_ids":["senseval3.d000.s009.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"get%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2720,"cached_input_tokens":0,"output_tokens":116,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0066,"input_uncached_usd":0.0054399999999999995,"input_cached_usd":0.0,"output_usd":0.00116,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9485703749815002}
{"item_id":"senseval3.d000.s009.t003","gold_sense_keys":["stranger%1:18:01::"],"candidates":[{"index":1,"sense_key":"stranger%1:18:00::","synset_id":"stranger.n.01"},{"index":2,"sense_key":"stranger%1:18:01::","synset_id":"stranger.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"stranger%1:18:01::","call_ids":["senseval3.d000.s009.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"stranger%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":468,"cached_input_tokens":0,"output_tokens":82,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0017560000000000002,"input_uncached_usd":0.000936,"input_cached_usd":0.0,"output_usd":0.0008200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4065083749592304}
{"item_id":"senseval3.d000.s010.t000","gold_sense_keys":["remember%2:31:00::"],"candidates":[{"index":1,"sense_key":"remember%2:31:00::","synset_id":"remember.v.01"},{"index":2,"sense_key":"remember%2:31:01::","synset_id":"remember.v.02"},{"index":3,"sense_key":"remember%2:31:02::","synset_id":"remember.v.03"},{"index":4,"sense_key":"remember%2:40:00::","synset_id":"remember.v.04"},{"index":5,"sense_key":"remember%2:32:07::","synset_id":"remember.v.05"},{"index":6,"sense_key":"remember%2:32:00::","synset_id":"commend.v.05"},{"index":7,"sense_key":"remember%2:31:08::","synset_id":"remember.v.07"},{"index":8,"sense_key":"remember%2:31:09::","synset_id":"commemorate.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"remember%2:31:00::","call_ids":["senseval3.d000.s010.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"remember%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":890,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0019,"input_uncached_usd":0.00178,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1988175420556217}
{"item_id":"senseval3.d000.s011.t001","gold_sense_keys":["hour%1:28:00::"],"candidates":[{"index":1,"sense_key":"hour%1:28:00::","synset_id":"hour.n.01"},{"index":2,"sense_key":"hour%1:28:01::","synset_id":"hour.n.02"},{"index":3,"sense_key":"hour%1:28:02::","synset_id":"hour.n.03"},{"index":4,"sense_key":"hour%1:07:00::","synset_id":"hour.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"hour%1:28:00::","call_ids":["senseval3.d000.s011.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"hour%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":603,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0013260000000000001,"input_uncached_usd":0.001206,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0202399590052664}
{"item_id":"senseval3.d000.s011.t002","gold_sense_keys":["steady%3:00:00::"],"candidates":[{"index":1,"sense_key":"steady%3:00:00::","synset_id":"steady.a.01"},{"index":2,"sense_key":"steady%5:00:00:stable:00","synset_id":"firm.s.06"},{"index":3,"sense_key":"steady%5:00:00:secure:03","synset_id":"steady.s.02"},{"index":4,"sense_key":"steady%5:00:00:resolute:00","synset_id":"firm.s.01"},{"index":5,"sense_key":"steady%5:00:00:frequent:00","synset_id":"regular.s.06"},{"index":6,"sense_key":"steady%5:00:00:unexcitable:00","synset_id":"steady.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"steady%5:00:00:frequent:00","call_ids":["senseval3.d000.s011.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"steady%5:00:00:frequent:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":750,"cached_input_tokens":0,"output_tokens":93,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00243,"input_uncached_usd":0.0015,"input_cached_usd":0.0,"output_usd":0.00093,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4868247079430148}
{"item_id":"senseval3.d000.s011.t003","gold_sense_keys":["drinking%1:04:01::"],"candidates":[{"index":1,"sense_key":"drinking%1:04:00::","synset_id":"drinking.n.01"},{"index":2,"sense_key":"drinking%1:04:01::","synset_id":"drink.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"drinking%1:04:01::","call_ids":["senseval3.d000.s011.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"drinking%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":510,"cached_input_tokens":0,"output_tokens":252,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0035399999999999997,"input_uncached_usd":0.0010199999999999999,"input_cached_usd":0.0,"output_usd":0.00252,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.0683427919866517}
{"item_id":"senseval3.d000.s012.t000","gold_sense_keys":["stranger%1:18:01::"],"candidates":[{"index":1,"sense_key":"stranger%1:18:00::","synset_id":"stranger.n.01"},{"index":2,"sense_key":"stranger%1:18:01::","synset_id":"stranger.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"stranger%1:18:01::","call_ids":["senseval3.d000.s012.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"stranger%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":481,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001632,"input_uncached_usd":0.000962,"input_cached_usd":0.0,"output_usd":0.00067,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5572317910846323}
{"item_id":"senseval3.d000.s012.t001","gold_sense_keys":["head%1:08:00::"],"candidates":[{"index":1,"sense_key":"head%1:08:00::","synset_id":"head.n.01"},{"index":2,"sense_key":"head%1:05:00::","synset_id":"head.n.02"},{"index":3,"sense_key":"head%1:09:00::","synset_id":"mind.n.01"},{"index":4,"sense_key":"head%1:18:00::","synset_id":"head.n.04"},{"index":5,"sense_key":"head%1:14:00::","synset_id":"head.n.05"},{"index":6,"sense_key":"head%1:19:00::","synset_id":"head.n.06"},{"index":7,"sense_key":"head%1:15:00::","synset_id":"head.n.07"},{"index":8,"sense_key":"head%1:15:01::","synset_id":"fountainhead.n.02"},{"index":9,"sense_key":"head%1:10:03::","synset_id":"head.n.09"},{"index":10,"sense_key":"head%1:26:00::","synset_id":"head.n.10"},{"index":11,"sense_key":"head%1:23:00::","synset_id":"head.n.11"},{"index":12,"sense_key":"head%1:20:00::","synset_id":"capitulum.n.01"},{"index":13,"sense_key":"head%1:18:01::","synset_id":"principal.n.02"},{"index":14,"sense_key":"head%1:18:03::","synset_id":"head.n.14"},{"index":15,"sense_key":"head%1:18:02::","synset_id":"head.n.15"},{"index":16,"sense_key":"head%1:17:02::","synset_id":"promontory.n.01"},{"index":17,"sense_key":"head%1:17:01::","synset_id":"head.n.17"},{"index":18,"sense_key":"head%1:17:00::","synset_id":"head.n.18"},{"index":19,"sense_key":"head%1:15:02::","synset_id":"forefront.n.01"},{"index":20,"sense_key":"head%1:11:01::","synset_id":"pass.n.09"},{"index":21,"sense_key":"head%1:11:00::","synset_id":"headway.n.02"},{"index":22,"sense_key":"head%1:10:01::","synset_id":"point.n.20"},{"index":23,"sense_key":"head%1:10:02::","synset_id":"question.n.02"},{"index":24,"sense_key":"head%1:10:00::","synset_id":"heading.n.01"},{"index":25,"sense_key":"head%1:08:01::","synset_id":"head.n.25"},{"index":26,"sense_key":"head%1:08:02::","synset_id":"head.n.26"},{"index":27,"sense_key":"head%1:06:05::","synset_id":"read/write_head.n.01"},{"index":28,"sense_key":"head%1:06:04::","synset_id":"head.n.28"},{"index":29,"sense_key":"head%1:06:03::","synset_id":"head.n.29"},{"index":30,"sense_key":"head%1:06:02::","synset_id":"head.n.30"},{"index":31,"sense_key":"head%1:06:00::","synset_id":"head.n.31"},{"index":32,"sense_key":"head%1:06:01::","synset_id":"drumhead.n.01"},{"index":33,"sense_key":"head%1:04:00::","synset_id":"oral_sex.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"head%1:08:00::","call_ids":["senseval3.d000.s012.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"head%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2212,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.005084,"input_uncached_usd":0.004424,"input_cached_usd":0.0,"output_usd":0.0006600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4491473339730874}
{"item_id":"senseval3.d000.s012.t003","gold_sense_keys":["thought%1:09:00::"],"candidates":[{"index":1,"sense_key":"thought%1:09:01::","synset_id":"idea.n.01"},{"index":2,"sense_key":"thought%1:09:00::","synset_id":"thinking.n.01"},{"index":3,"sense_key":"thought%1:09:02::","synset_id":"thought.n.03"},{"index":4,"sense_key":"thought%1:09:03::","synset_id":"opinion.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"thought%1:09:00::","call_ids":["senseval3.d000.s012.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"thought%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":682,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001484,"input_uncached_usd":0.001364,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0145243749720976}
{"item_id":"senseval3.d000.s012.t004","gold_sense_keys":["start%2:30:00::"],"candidates":[{"index":1,"sense_key":"start%2:30:00::","synset_id":"get_down.v.07"},{"index":2,"sense_key":"start%2:30:01::","synset_id":"begin.v.03"},{"index":3,"sense_key":"start%2:38:02::","synset_id":"depart.v.03"},{"index":4,"sense_key":"start%2:42:00::","synset_id":"begin.v.02"},{"index":5,"sense_key":"start%2:36:00::","synset_id":"originate.v.02"},{"index":6,"sense_key":"start%2:36:01::","synset_id":"start.v.06"},{"index":7,"sense_key":"start%2:38:04::","synset_id":"startle.v.02"},{"index":8,"sense_key":"start%2:38:01::","synset_id":"start.v.08"},{"index":9,"sense_key":"start%2:38:00::","synset_id":"start.v.09"},{"index":10,"sense_key":"start%2:41:00::","synset_id":"start.v.10"},{"index":11,"sense_key":"start%2:33:00::","synset_id":"start.v.11"},{"index":12,"sense_key":"start%2:42:03::","synset_id":"begin.v.07"},{"index":13,"sense_key":"start%2:41:04::","synset_id":"begin.v.08"},{"index":14,"sense_key":"start%2:38:03::","synset_id":"start.v.14"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"start%2:30:00::","call_ids":["senseval3.d000.s012.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"start%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1275,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0031799999999999997,"input_uncached_usd":0.0025499999999999997,"input_cached_usd":0.0,"output_usd":0.00063,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1559288339922205}
{"item_id":"senseval3.d000.s012.t005","gold_sense_keys":["cross%2:38:00::"],"candidates":[{"index":1,"sense_key":"cross%2:38:00::","synset_id":"traverse.v.01"},{"index":2,"sense_key":"cross%2:38:03::","synset_id":"intersect.v.01"},{"index":3,"sense_key":"cross%2:41:00::","synset_id":"thwart.v.01"},{"index":4,"sense_key":"cross%2:38:04::","synset_id":"cross.v.04"},{"index":5,"sense_key":"cross%2:42:00::","synset_id":"cross.v.05"},{"index":6,"sense_key":"cross%2:38:01::","synset_id":"cross.v.06"},{"index":7,"sense_key":"cross%2:36:09::","synset_id":"cross.v.07"},{"index":8,"sense_key":"cross%2:35:00::","synset_id":"crossbreed.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cross%2:38:00::","call_ids":["senseval3.d000.s012.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cross%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":899,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001918,"input_uncached_usd":0.001798,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0344448750838637}
{"item_id":"senseval3.d000.s012.t006","gold_sense_keys":["street%1:06:01::"],"candidates":[{"index":1,"sense_key":"street%1:06:00::","synset_id":"street.n.01"},{"index":2,"sense_key":"street%1:06:01::","synset_id":"street.n.02"},{"index":3,"sense_key":"street%1:26:00::","synset_id":"street.n.03"},{"index":4,"sense_key":"street%1:26:01::","synset_id":"street.n.04"},{"index":5,"sense_key":"street%1:14:00::","synset_id":"street.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"street%1:06:01::","call_ids":["senseval3.d000.s012.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"street%1:06:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":702,"cached_input_tokens":0,"output_tokens":82,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002224,"input_uncached_usd":0.0014039999999999999,"input_cached_usd":0.0,"output_usd":0.0008200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5269885000307113}
{"item_id":"senseval3.d000.s012.t007","gold_sense_keys":["light%1:10:00::"],"candidates":[{"index":1,"sense_key":"light%1:19:00::","synset_id":"light.n.01"},{"index":2,"sense_key":"light%1:06:00::","synset_id":"light.n.02"},{"index":3,"sense_key":"light%1:09:01::","synset_id":"light.n.03"},{"index":4,"sense_key":"light%1:07:01::","synset_id":"luminosity.n.01"},{"index":5,"sense_key":"light%1:15:00::","synset_id":"light.n.05"},{"index":6,"sense_key":"light%1:26:01::","synset_id":"light.n.06"},{"index":7,"sense_key":"light%1:07:00::","synset_id":"light.n.07"},{"index":8,"sense_key":"light%1:18:00::","synset_id":"light.n.08"},{"index":9,"sense_key":"light%1:26:00::","synset_id":"light.n.09"},{"index":10,"sense_key":"light%1:09:00::","synset_id":"light.n.10"},{"index":11,"sense_key":"light%1:07:02::","synset_id":"sparkle.n.01"},{"index":12,"sense_key":"light%1:09:02::","synset_id":"light.n.12"},{"index":13,"sense_key":"light%1:16:00::","synset_id":"inner_light.n.01"},{"index":14,"sense_key":"light%1:10:00::","synset_id":"light.n.14"},{"index":15,"sense_key":"light%1:06:01::","synset_id":"lighter.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":14,"chosen_sense_key":"light%1:10:00::","call_ids":["senseval3.d000.s012.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":14,"predicted_sense_key":"light%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1303,"cached_input_tokens":0,"output_tokens":69,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003296,"input_uncached_usd":0.002606,"input_cached_usd":0.0,"output_usd":0.0006900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6888826669892296}
{"item_id":"senseval3.d000.s012.t010","gold_sense_keys":["roar%2:32:01::"],"candidates":[{"index":1,"sense_key":"roar%2:32:01::","synset_id":"roar.v.01"},{"index":2,"sense_key":"roar%2:32:03::","synset_id":"thunder.v.02"},{"index":3,"sense_key":"roar%2:32:02::","synset_id":"howl.v.01"},{"index":4,"sense_key":"roar%2:38:00::","synset_id":"roar.v.04"},{"index":5,"sense_key":"roar%2:32:05::","synset_id":"bellow.v.02"},{"index":6,"sense_key":"roar%2:29:00::","synset_id":"roar.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"roar%2:32:01::","call_ids":["senseval3.d000.s012.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"roar%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":766,"cached_input_tokens":0,"output_tokens":97,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002502,"input_uncached_usd":0.001532,"input_cached_usd":0.0,"output_usd":0.00097,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.4243981670588255}
{"item_id":"senseval3.d000.s012.t011","gold_sense_keys":["intersection%1:06:00::"],"candidates":[{"index":1,"sense_key":"intersection%1:09:00::","synset_id":"intersection.n.01"},{"index":2,"sense_key":"intersection%1:06:00::","synset_id":"intersection.n.02"},{"index":3,"sense_key":"intersection%1:25:00::","synset_id":"intersection.n.03"},{"index":4,"sense_key":"intersection%1:14:00::","synset_id":"intersection.n.04"},{"index":5,"sense_key":"intersection%1:09:01::","synset_id":"overlap.n.01"},{"index":6,"sense_key":"intersection%1:04:00::","synset_id":"intersection.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"intersection%1:06:00::","call_ids":["senseval3.d000.s012.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"intersection%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":729,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001578,"input_uncached_usd":0.0014579999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.046904040966183}
{"item_id":"senseval3.d000.s013.t000","gold_sense_keys":["brake%1:06:00::"],"candidates":[{"index":1,"sense_key":"brake%1:06:00::","synset_id":"brake.n.01"},{"index":2,"sense_key":"brake%1:20:02::","synset_id":"brake.n.02"},{"index":3,"sense_key":"brake%1:20:01::","synset_id":"bracken.n.02"},{"index":4,"sense_key":"brake%1:14:00::","synset_id":"brake.n.04"},{"index":5,"sense_key":"brake%1:06:01::","synset_id":"brake.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"brake%1:06:00::","call_ids":["senseval3.d000.s013.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"brake%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":692,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0015040000000000001,"input_uncached_usd":0.001384,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9608832499943674}
{"item_id":"senseval3.d000.s013.t001","gold_sense_keys":["howl%2:32:01::"],"candidates":[{"index":1,"sense_key":"howl%2:32:02::","synset_id":"howl.v.01"},{"index":2,"sense_key":"howl%2:32:03::","synset_id":"howl.v.02"},{"index":3,"sense_key":"howl%2:32:01::","synset_id":"roar.v.01"},{"index":4,"sense_key":"howl%2:29:00::","synset_id":"roar.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"howl%2:32:01::","call_ids":["senseval3.d000.s013.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"howl%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":663,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0019459999999999998,"input_uncached_usd":0.001326,"input_cached_usd":0.0,"output_usd":0.00062,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4691614579642192}
{"item_id":"senseval3.d000.s013.t002","gold_sense_keys":["horn%1:06:05::"],"candidates":[{"index":1,"sense_key":"horn%1:06:06::","synset_id":"horn.n.01"},{"index":2,"sense_key":"horn%1:05:01::","synset_id":"horn.n.02"},{"index":3,"sense_key":"horn%1:10:02::","synset_id":"horn.n.03"},{"index":4,"sense_key":"horn%1:06:04::","synset_id":"horn.n.04"},{"index":5,"sense_key":"horn%1:06:01::","synset_id":"cornet.n.01"},{"index":6,"sense_key":"horn%1:05:02::","synset_id":"horn.n.06"},{"index":7,"sense_key":"horn%1:27:00::","synset_id":"horn.n.07"},{"index":8,"sense_key":"horn%1:06:07::","synset_id":"horn.n.08"},{"index":9,"sense_key":"horn%1:06:03::","synset_id":"horn.n.09"},{"index":10,"sense_key":"horn%1:06:00::","synset_id":"french_horn.n.01"},{"index":11,"sense_key":"horn%1:06:05::","synset_id":"automobile_horn.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"horn%1:06:05::","call_ids":["senseval3.d000.s013.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"horn%1:06:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":998,"cached_input_tokens":0,"output_tokens":143,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0034260000000000002,"input_uncached_usd":0.001996,"input_cached_usd":0.0,"output_usd":0.00143,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.4521569999633357}
{"item_id":"senseval3.d000.s013.t003","gold_sense_keys":["blare%2:39:00::"],"candidates":[{"index":1,"sense_key":"blare%2:39:01::","synset_id":"blast.v.01"},{"index":2,"sense_key":"blare%2:39:00::","synset_id":"honk.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"blare%2:39:00::","call_ids":["senseval3.d000.s013.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"blare%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":525,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00117,"input_uncached_usd":0.00105,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4089610829250887}
{"item_id":"senseval3.d000.s013.t004","gold_sense_keys":["man%1:18:00::"],"candidates":[{"index":1,"sense_key":"man%1:18:00::","synset_id":"man.n.01"},{"index":2,"sense_key":"man%1:18:03::","synset_id":"serviceman.n.01"},{"index":3,"sense_key":"man%1:18:04::","synset_id":"man.n.03"},{"index":4,"sense_key":"man%1:05:01::","synset_id":"homo.n.02"},{"index":5,"sense_key":"man%1:18:08::","synset_id":"man.n.05"},{"index":6,"sense_key":"man%1:18:06::","synset_id":"man.n.06"},{"index":7,"sense_key":"man%1:18:05::","synset_id":"valet.n.01"},{"index":8,"sense_key":"man%1:18:07::","synset_id":"man.n.08"},{"index":9,"sense_key":"man%1:15:00::","synset_id":"man.n.09"},{"index":10,"sense_key":"man%1:06:00::","synset_id":"man.n.10"},{"index":11,"sense_key":"man%1:05:00::","synset_id":"world.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"man%1:18:00::","call_ids":["senseval3.d000.s013.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"man%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1149,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0024179999999999996,"input_uncached_usd":0.0022979999999999997,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0336068329634145}
{"item_id":"senseval3.d000.s013.t005","gold_sense_keys":["hit%2:35:01::"],"candidates":[{"index":1,"sense_key":"hit%2:35:03::","synset_id":"hit.v.01"},{"index":2,"sense_key":"hit%2:35:01::","synset_id":"hit.v.02"},{"index":3,"sense_key":"hit%2:35:00::","synset_id":"hit.v.03"},{"index":4,"sense_key":"hit%2:38:00::","synset_id":"reach.v.01"},{"index":5,"sense_key":"hit%2:35:05::","synset_id":"hit.v.05"},{"index":6,"sense_key":"hit%2:33:01::","synset_id":"shoot.v.01"},{"index":7,"sense_key":"hit%2:40:05::","synset_id":"stumble.v.03"},{"index":8,"sense_key":"hit%2:33:02::","synset_id":"score.v.01"},{"index":9,"sense_key":"hit%2:39:00::","synset_id":"hit.v.09"},{"index":10,"sense_key":"hit%2:33:00::","synset_id":"strike.v.04"},{"index":11,"sense_key":"hit%2:41:00::","synset_id":"murder.v.01"},{"index":12,"sense_key":"hit%2:38:02::","synset_id":"hit.v.12"},{"index":13,"sense_key":"hit%2:38:07::","synset_id":"reach.v.02"},{"index":14,"sense_key":"hit%2:35:02::","synset_id":"strike.v.10"},{"index":15,"sense_key":"hit%2:34:00::","synset_id":"hit.v.15"},{"index":16,"sense_key":"hit%2:33:03::","synset_id":"hit.v.16"},{"index":17,"sense_key":"hit%2:32:12::","synset_id":"hit.v.17"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"hit%2:35:01::","call_ids":["senseval3.d000.s013.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"hit%2:35:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1490,"cached_input_tokens":0,"output_tokens":104,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00402,"input_uncached_usd":0.00298,"input_cached_usd":0.0,"output_usd":0.0010400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.543788625043817}
{"item_id":"senseval3.d000.s013.t006","gold_sense_keys":["call_out%2:32:01::"],"candidates":[{"index":1,"sense_key":"call_out%2:32:01::","synset_id":"exclaim.v.01"},{"index":2,"sense_key":"call_out%2:32:00::","synset_id":"call_out.v.02"},{"index":3,"sense_key":"call_out%2:32:12::","synset_id":"call_out.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"call_out%2:32:01::","call_ids":["senseval3.d000.s013.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"call_out%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":588,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001296,"input_uncached_usd":0.001176,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8510737500619143}
{"item_id":"senseval3.d000.s013.t007","gold_sense_keys":["second%1:28:00::","second%1:28:01::"],"candidates":[{"index":1,"sense_key":"second%1:28:00::","synset_id":"second.n.01"},{"index":2,"sense_key":"second%1:28:01::","synset_id":"moment.n.02"},{"index":3,"sense_key":"second%1:04:00::","synset_id":"second_base.n.02"},{"index":4,"sense_key":"second%1:28:02::","synset_id":"moment.n.01"},{"index":5,"sense_key":"second%1:24:00::","synset_id":"second.n.05"},{"index":6,"sense_key":"second%1:23:00::","synset_id":"second.n.06"},{"index":7,"sense_key":"second%1:18:00::","synset_id":"second.n.07"},{"index":8,"sense_key":"second%1:10:00::","synset_id":"second.n.08"},{"index":9,"sense_key":"second%1:06:00::","synset_id":"second_gear.n.01"},{"index":10,"sense_key":"second%1:06:01::","synset_id":"irregular.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"second%1:28:01::","call_ids":["senseval3.d000.s013.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"second%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":988,"cached_input_tokens":0,"output_tokens":189,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003866,"input_uncached_usd":0.001976,"input_cached_usd":0.0,"output_usd":0.0018900000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.266280167037621}
{"item_id":"senseval3.d000.s014.t002","gold_sense_keys":["do%2:36:01::"],"candidates":[{"index":1,"sense_key":"do%2:41:01::","synset_id":"make.v.01"},{"index":2,"sense_key":"do%2:36:01::","synset_id":"perform.v.01"},{"index":3,"sense_key":"do%2:41:04::","synset_id":"do.v.03"},{"index":4,"sense_key":"do%2:42:00::","synset_id":"do.v.04"},{"index":5,"sense_key":"do%2:36:02::","synset_id":"cause.v.01"},{"index":6,"sense_key":"do%2:41:02::","synset_id":"practice.v.01"},{"index":7,"sense_key":"do%2:42:01::","synset_id":"suffice.v.01"},{"index":8,"sense_key":"do%2:36:00::","synset_id":"do.v.08"},{"index":9,"sense_key":"do%2:29:09::","synset_id":"act.v.02"},{"index":10,"sense_key":"do%2:42:02::","synset_id":"serve.v.09"},{"index":11,"sense_key":"do%2:41:03::","synset_id":"do.v.11"},{"index":12,"sense_key":"do%2:29:04::","synset_id":"dress.v.16"},{"index":13,"sense_key":"do%2:38:00::","synset_id":"do.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"do%2:36:01::","call_ids":["senseval3.d000.s014.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"do%2:36:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1238,"cached_input_tokens":0,"output_tokens":105,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003526,"input_uncached_usd":0.002476,"input_cached_usd":0.0,"output_usd":0.0010500000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8951355000026524}
{"item_id":"senseval3.d000.s015.t001","gold_sense_keys":["care%2:37:02::"],"candidates":[{"index":1,"sense_key":"care%2:37:02::","synset_id":"care.v.01"},{"index":2,"sense_key":"care%2:41:13::","synset_id":"care.v.02"},{"index":3,"sense_key":"care%2:37:00::","synset_id":"wish.v.02"},{"index":4,"sense_key":"care%2:41:11::","synset_id":"manage.v.02"},{"index":5,"sense_key":"care%2:37:12::","synset_id":"worry.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"care%2:37:02::","call_ids":["senseval3.d000.s015.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"care%2:37:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":737,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001594,"input_uncached_usd":0.0014739999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.929132541990839}
{"item_id":"senseval3.d000.s015.t002","gold_sense_keys":["guy%1:18:00::"],"candidates":[{"index":1,"sense_key":"guy%1:18:00::","synset_id":"guy.n.01"},{"index":2,"sense_key":"guy%1:06:01::","synset_id":"guy.n.02"},{"index":3,"sense_key":"guy%1:06:00::","synset_id":"guy.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"guy%1:18:00::","call_ids":["senseval3.d000.s015.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"guy%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":651,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001422,"input_uncached_usd":0.001302,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.986784707987681}
{"item_id":"senseval3.d000.s015.t003","gold_sense_keys":["hit%2:35:01::"],"candidates":[{"index":1,"sense_key":"hit%2:35:03::","synset_id":"hit.v.01"},{"index":2,"sense_key":"hit%2:35:01::","synset_id":"hit.v.02"},{"index":3,"sense_key":"hit%2:35:00::","synset_id":"hit.v.03"},{"index":4,"sense_key":"hit%2:38:00::","synset_id":"reach.v.01"},{"index":5,"sense_key":"hit%2:35:05::","synset_id":"hit.v.05"},{"index":6,"sense_key":"hit%2:33:01::","synset_id":"shoot.v.01"},{"index":7,"sense_key":"hit%2:40:05::","synset_id":"stumble.v.03"},{"index":8,"sense_key":"hit%2:33:02::","synset_id":"score.v.01"},{"index":9,"sense_key":"hit%2:39:00::","synset_id":"hit.v.09"},{"index":10,"sense_key":"hit%2:33:00::","synset_id":"strike.v.04"},{"index":11,"sense_key":"hit%2:41:00::","synset_id":"murder.v.01"},{"index":12,"sense_key":"hit%2:38:02::","synset_id":"hit.v.12"},{"index":13,"sense_key":"hit%2:38:07::","synset_id":"reach.v.02"},{"index":14,"sense_key":"hit%2:35:02::","synset_id":"strike.v.10"},{"index":15,"sense_key":"hit%2:34:00::","synset_id":"hit.v.15"},{"index":16,"sense_key":"hit%2:33:03::","synset_id":"hit.v.16"},{"index":17,"sense_key":"hit%2:32:12::","synset_id":"hit.v.17"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"hit%2:35:01::","call_ids":["senseval3.d000.s015.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"hit%2:35:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1539,"cached_input_tokens":0,"output_tokens":148,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004558,"input_uncached_usd":0.003078,"input_cached_usd":0.0,"output_usd":0.0014800000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.4008375420235097}
{"item_id":"senseval3.d000.s016.t000","gold_sense_keys":["regret%2:37:00::"],"candidates":[{"index":1,"sense_key":"regret%2:37:00::","synset_id":"repent.v.02"},{"index":2,"sense_key":"regret%2:37:01::","synset_id":"regret.v.02"},{"index":3,"sense_key":"regret%2:32:00::","synset_id":"regret.v.03"},{"index":4,"sense_key":"regret%2:32:02::","synset_id":"regret.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"regret%2:37:00::","call_ids":["senseval3.d000.s016.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"regret%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":663,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001446,"input_uncached_usd":0.001326,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1425080419285223}
{"item_id":"senseval3.d000.s016.t001","gold_sense_keys":["open%2:35:00::"],"candidates":[{"index":1,"sense_key":"open%2:35:00::","synset_id":"open.v.01"},{"index":2,"sense_key":"open%2:41:00::","synset_id":"open.v.02"},{"index":3,"sense_key":"open%2:35:06::","synset_id":"open.v.03"},{"index":4,"sense_key":"open%2:41:01::","synset_id":"open.v.04"},{"index":5,"sense_key":"open%2:35:08::","synset_id":"unfold.v.04"},{"index":6,"sense_key":"open%2:30:00::","synset_id":"open.v.06"},{"index":7,"sense_key":"open%2:30:01::","synset_id":"open.v.07"},{"index":8,"sense_key":"open%2:42:00::","synset_id":"open.v.08"},{"index":9,"sense_key":"open%2:33:00::","synset_id":"open.v.09"},{"index":10,"sense_key":"open%2:40:04::","synset_id":"afford.v.04"},{"index":11,"sense_key":"open%2:39:10::","synset_id":"open.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"open%2:35:00::","call_ids":["senseval3.d000.s016.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"open%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1021,"cached_input_tokens":0,"output_tokens":81,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002852,"input_uncached_usd":0.002042,"input_cached_usd":0.0,"output_usd":0.0008100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6568997079739347}
{"item_id":"senseval3.d000.s016.t002","gold_sense_keys":["mouth%1:08:01::"],"candidates":[{"index":1,"sense_key":"mouth%1:08:01::","synset_id":"mouth.n.01"},{"index":2,"sense_key":"mouth%1:08:00::","synset_id":"mouth.n.02"},{"index":3,"sense_key":"mouth%1:17:01::","synset_id":"mouth.n.03"},{"index":4,"sense_key":"mouth%1:17:00::","synset_id":"mouth.n.04"},{"index":5,"sense_key":"mouth%1:18:00::","synset_id":"mouth.n.05"},{"index":6,"sense_key":"mouth%1:18:01::","synset_id":"mouthpiece.n.03"},{"index":7,"sense_key":"mouth%1:10:00::","synset_id":"sass.n.01"},{"index":8,"sense_key":"mouth%1:06:00::","synset_id":"mouth.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"mouth%1:08:01::","call_ids":["senseval3.d000.s016.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"mouth%1:08:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":984,"cached_input_tokens":0,"output_tokens":126,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003228,"input_uncached_usd":0.0019679999999999997,"input_cached_usd":0.0,"output_usd":0.00126,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.940756290918216}
{"item_id":"senseval3.d000.s016.t003","gold_sense_keys":["truck%1:06:00::"],"candidates":[{"index":1,"sense_key":"truck%1:06:00::","synset_id":"truck.n.01"},{"index":2,"sense_key":"truck%1:06:01::","synset_id":"hand_truck.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"truck%1:06:00::","call_ids":["senseval3.d000.s016.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"truck%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":608,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001336,"input_uncached_usd":0.0012159999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8260326660238206}
{"item_id":"senseval3.d000.s016.t005","gold_sense_keys":["stop%1:11:00::"],"candidates":[{"index":1,"sense_key":"stop%1:11:00::","synset_id":"stop.n.01"},{"index":2,"sense_key":"stop%1:04:00::","synset_id":"stop.n.02"},{"index":3,"sense_key":"stop%1:04:02::","synset_id":"stop.n.03"},{"index":4,"sense_key":"stop%1:26:00::","synset_id":"arrest.n.02"},{"index":5,"sense_key":"stop%1:15:00::","synset_id":"stop.n.05"},{"index":6,"sense_key":"stop%1:10:00::","synset_id":"stop_consonant.n.01"},{"index":7,"sense_key":"stop%1:10:01::","synset_id":"period.n.07"},{"index":8,"sense_key":"stop%1:06:03::","synset_id":"stop.n.08"},{"index":9,"sense_key":"stop%1:06:02::","synset_id":"diaphragm.n.01"},{"index":10,"sense_key":"stop%1:06:00::","synset_id":"catch.n.06"},{"index":11,"sense_key":"stop%1:06:01::","synset_id":"blockage.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"stop%1:11:00::","call_ids":["senseval3.d000.s016.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"stop%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1278,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0031859999999999996,"input_uncached_usd":0.0025559999999999997,"input_cached_usd":0.0,"output_usd":0.00063,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.953671875060536}
{"item_id":"senseval3.d000.s016.t006","gold_sense_keys":["angry%3:00:00::"],"candidates":[{"index":1,"sense_key":"angry%3:00:00::","synset_id":"angry.a.01"},{"index":2,"sense_key":"angry%5:00:00:stormy:00","synset_id":"angry.s.01"},{"index":3,"sense_key":"angry%5:00:00:unhealthy:00","synset_id":"angry.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"angry%3:00:00::","call_ids":["senseval3.d000.s016.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"angry%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":657,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001434,"input_uncached_usd":0.0013139999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.113876459072344}
{"item_id":"senseval3.d000.s016.t007","gold_sense_keys":["driver%1:18:00::"],"candidates":[{"index":1,"sense_key":"driver%1:18:00::","synset_id":"driver.n.01"},{"index":2,"sense_key":"driver%1:18:02::","synset_id":"driver.n.02"},{"index":3,"sense_key":"driver%1:18:01::","synset_id":"driver.n.03"},{"index":4,"sense_key":"driver%1:10:00::","synset_id":"driver.n.04"},{"index":5,"sense_key":"driver%1:06:00::","synset_id":"driver.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"driver%1:18:00::","call_ids":["senseval3.d000.s016.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"driver%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":711,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001542,"input_uncached_usd":0.0014219999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4553282500710338}
{"item_id":"senseval3.d000.s016.t008","gold_sense_keys":["jump%2:38:11::"],"candidates":[{"index":1,"sense_key":"jump%2:38:00::","synset_id":"jump.v.01"},{"index":2,"sense_key":"jump%2:38:04::","synset_id":"startle.v.02"},{"index":3,"sense_key":"jump%2:33:00::","synset_id":"jump.v.03"},{"index":4,"sense_key":"jump%2:30:00::","synset_id":"jump.v.04"},{"index":5,"sense_key":"jump%2:42:00::","synset_id":"leap_out.v.01"},{"index":6,"sense_key":"jump%2:33:03::","synset_id":"jump.v.06"},{"index":7,"sense_key":"jump%2:30:01::","synset_id":"rise.v.11"},{"index":8,"sense_key":"jump%2:38:11::","synset_id":"jump.v.08"},{"index":9,"sense_key":"jump%2:38:05::","synset_id":"derail.v.02"},{"index":10,"sense_key":"jump%2:38:12::","synset_id":"chute.v.01"},{"index":11,"sense_key":"jump%2:38:01::","synset_id":"jump.v.11"},{"index":12,"sense_key":"jump%2:38:07::","synset_id":"jumpstart.v.01"},{"index":13,"sense_key":"jump%2:31:00::","synset_id":"jump.v.13"},{"index":14,"sense_key":"jump%2:30:05::","synset_id":"leap.v.02"},{"index":15,"sense_key":"jump%2:30:03::","synset_id":"alternate.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"jump%2:38:11::","call_ids":["senseval3.d000.s016.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"jump%2:38:11::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1357,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003284,"input_uncached_usd":0.002714,"input_cached_usd":0.0,"output_usd":0.0005700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1597588339354843}
{"item_id":"senseval3.d000.s016.t009","gold_sense_keys":["cab%1:06:02::"],"candidates":[{"index":1,"sense_key":"cab%1:06:02::","synset_id":"cab.n.01"},{"index":2,"sense_key":"cab%1:06:01::","synset_id":"cab.n.02"},{"index":3,"sense_key":"cab%1:06:00::","synset_id":"cab.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cab%1:06:02::","call_ids":["senseval3.d000.s016.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cab%1:06:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":658,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001436,"input_uncached_usd":0.001316,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4069253749912605}
{"item_id":"senseval3.d000.s016.t010","gold_sense_keys":["walk%2:38:00::"],"candidates":[{"index":1,"sense_key":"walk%2:38:00::","synset_id":"walk.v.01"},{"index":2,"sense_key":"walk%2:38:01::","synset_id":"walk.v.02"},{"index":3,"sense_key":"walk%2:33:00::","synset_id":"walk.v.03"},{"index":4,"sense_key":"walk%2:38:03::","synset_id":"walk.v.04"},{"index":5,"sense_key":"walk%2:33:01::","synset_id":"walk.v.05"},{"index":6,"sense_key":"walk%2:41:00::","synset_id":"walk.v.06"},{"index":7,"sense_key":"walk%2:41:01::","synset_id":"walk.v.07"},{"index":8,"sense_key":"walk%2:38:02::","synset_id":"walk.v.08"},{"index":9,"sense_key":"walk%2:38:05::","synset_id":"walk.v.09"},{"index":10,"sense_key":"walk%2:38:04::","synset_id":"walk.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"walk%2:38:00::","call_ids":["senseval3.d000.s016.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"walk%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":964,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002048,"input_uncached_usd":0.0019279999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9033648329786956}
{"item_id":"senseval3.d000.s017.t000","gold_sense_keys":["stranger%1:18:01::"],"candidates":[{"index":1,"sense_key":"stranger%1:18:00::","synset_id":"stranger.n.01"},{"index":2,"sense_key":"stranger%1:18:01::","synset_id":"stranger.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"stranger%1:18:01::","call_ids":["senseval3.d000.s017.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"stranger%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":579,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001878,"input_uncached_usd":0.001158,"input_cached_usd":0.0,"output_usd":0.00072,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4812662499025464}
{"item_id":"senseval3.d000.s017.t003","gold_sense_keys":["arm%1:08:00::"],"candidates":[{"index":1,"sense_key":"arm%1:08:00::","synset_id":"arm.n.01"},{"index":2,"sense_key":"arm%1:06:03::","synset_id":"arm.n.02"},{"index":3,"sense_key":"arm%1:06:01::","synset_id":"weapon.n.01"},{"index":4,"sense_key":"arm%1:06:02::","synset_id":"arm.n.04"},{"index":5,"sense_key":"arm%1:14:00::","synset_id":"branch.n.01"},{"index":6,"sense_key":"arm%1:06:00::","synset_id":"sleeve.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"arm%1:08:00::","call_ids":["senseval3.d000.s017.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"arm%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":870,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00186,"input_uncached_usd":0.00174,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8403699999907985}
{"item_id":"senseval3.d000.s017.t004","gold_sense_keys":["shoulder%1:08:00::"],"candidates":[{"index":1,"sense_key":"shoulder%1:08:00::","synset_id":"shoulder.n.01"},{"index":2,"sense_key":"shoulder%1:13:00::","synset_id":"shoulder.n.02"},{"index":3,"sense_key":"shoulder%1:08:02::","synset_id":"shoulder.n.03"},{"index":4,"sense_key":"shoulder%1:06:01::","synset_id":"shoulder.n.04"},{"index":5,"sense_key":"shoulder%1:06:00::","synset_id":"shoulder.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"shoulder%1:08:00::","call_ids":["senseval3.d000.s017.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"shoulder%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":784,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001688,"input_uncached_usd":0.001568,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0023926249705255}
{"item_id":"senseval3.d000.s017.t005","gold_sense_keys":["old%3:00:01::"],"candidates":[{"index":1,"sense_key":"old%3:00:02::","synset_id":"old.a.01"},{"index":2,"sense_key":"old%3:00:01::","synset_id":"old.a.02"},{"index":3,"sense_key":"old%5:00:00:familiar:00","synset_id":"old.s.01"},{"index":4,"sense_key":"old%5:00:00:experienced:00","synset_id":"old.s.02"},{"index":5,"sense_key":"old%5:00:03:past:00","synset_id":"erstwhile.s.01"},{"index":6,"sense_key":"old%5:00:00:genuine:00","synset_id":"honest-to-god.s.01"},{"index":7,"sense_key":"old%5:00:00:early:01","synset_id":"old.s.05"},{"index":8,"sense_key":"old%5:00:00:preceding:00","synset_id":"previous.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"old%5:00:00:familiar:00","call_ids":["senseval3.d000.s017.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"old%5:00:00:familiar:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":993,"cached_input_tokens":0,"output_tokens":292,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004906000000000001,"input_uncached_usd":0.001986,"input_cached_usd":0.0,"output_usd":0.0029200000000000003,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.841327750007622}
{"item_id":"senseval3.d000.s017.t006","gold_sense_keys":["friend%1:18:00::"],"candidates":[{"index":1,"sense_key":"friend%1:18:00::","synset_id":"friend.n.01"},{"index":2,"sense_key":"friend%1:18:03::","synset_id":"ally.n.02"},{"index":3,"sense_key":"friend%1:18:02::","synset_id":"acquaintance.n.03"},{"index":4,"sense_key":"friend%1:18:04::","synset_id":"supporter.n.01"},{"index":5,"sense_key":"friend%1:18:01::","synset_id":"friend.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"friend%1:18:00::","call_ids":["senseval3.d000.s017.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"friend%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":820,"cached_input_tokens":0,"output_tokens":101,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00265,"input_uncached_usd":0.00164,"input_cached_usd":0.0,"output_usd":0.00101,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.213380708009936}
{"item_id":"senseval3.d000.s018.t000","gold_sense_keys":["driver%1:18:00::"],"candidates":[{"index":1,"sense_key":"driver%1:18:00::","synset_id":"driver.n.01"},{"index":2,"sense_key":"driver%1:18:02::","synset_id":"driver.n.02"},{"index":3,"sense_key":"driver%1:18:01::","synset_id":"driver.n.03"},{"index":4,"sense_key":"driver%1:10:00::","synset_id":"driver.n.04"},{"index":5,"sense_key":"driver%1:06:00::","synset_id":"driver.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"driver%1:18:00::","call_ids":["senseval3.d000.s018.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"driver%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":703,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001526,"input_uncached_usd":0.001406,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5773816249566153}
{"item_id":"senseval3.d000.s018.t001","gold_sense_keys":["start%2:30:00::"],"candidates":[{"index":1,"sense_key":"start%2:30:00::","synset_id":"get_down.v.07"},{"index":2,"sense_key":"start%2:30:01::","synset_id":"begin.v.03"},{"index":3,"sense_key":"start%2:38:02::","synset_id":"depart.v.03"},{"index":4,"sense_key":"start%2:42:00::","synset_id":"begin.v.02"},{"index":5,"sense_key":"start%2:36:00::","synset_id":"originate.v.02"},{"index":6,"sense_key":"start%2:36:01::","synset_id":"start.v.06"},{"index":7,"sense_key":"start%2:38:04::","synset_id":"startle.v.02"},{"index":8,"sense_key":"start%2:38:01::","synset_id":"start.v.08"},{"index":9,"sense_key":"start%2:38:00::","synset_id":"start.v.09"},{"index":10,"sense_key":"start%2:41:00::","synset_id":"start.v.10"},{"index":11,"sense_key":"start%2:33:00::","synset_id":"start.v.11"},{"index":12,"sense_key":"start%2:42:03::","synset_id":"begin.v.07"},{"index":13,"sense_key":"start%2:41:04::","synset_id":"begin.v.08"},{"index":14,"sense_key":"start%2:38:03::","synset_id":"start.v.14"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"start%2:30:00::","call_ids":["senseval3.d000.s018.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"start%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1360,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00345,"input_uncached_usd":0.0027199999999999998,"input_cached_usd":0.0,"output_usd":0.0007300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3246947079896927}
{"item_id":"senseval3.d000.s018.t002","gold_sense_keys":["curse%2:32:02::"],"candidates":[{"index":1,"sense_key":"curse%2:32:00::","synset_id":"curse.v.01"},{"index":2,"sense_key":"curse%2:32:02::","synset_id":"curse.v.02"},{"index":3,"sense_key":"curse%2:32:01::","synset_id":"curse.v.03"},{"index":4,"sense_key":"curse%2:32:03::","synset_id":"excommunicate.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"curse%2:32:02::","call_ids":["senseval3.d000.s018.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"curse%2:32:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":765,"cached_input_tokens":0,"output_tokens":182,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00335,"input_uncached_usd":0.00153,"input_cached_usd":0.0,"output_usd":0.0018200000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.675274792010896}
{"item_id":"senseval3.d000.s018.t003","gold_sense_keys":["plot%1:09:00::"],"candidates":[{"index":1,"sense_key":"plot%1:09:00::","synset_id":"plot.n.01"},{"index":2,"sense_key":"plot%1:15:00::","synset_id":"plot.n.02"},{"index":3,"sense_key":"plot%1:10:00::","synset_id":"plot.n.03"},{"index":4,"sense_key":"plot%1:10:01::","synset_id":"plot.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"plot%1:09:00::","call_ids":["senseval3.d000.s018.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"plot%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":730,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00158,"input_uncached_usd":0.00146,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.456968207960017}
{"item_id":"senseval3.d000.s018.t004","gold_sense_keys":["ruin%2:41:00::"],"candidates":[{"index":1,"sense_key":"ruin%2:35:00::","synset_id":"destroy.v.02"},{"index":2,"sense_key":"ruin%2:41:00::","synset_id":"ruin.v.02"},{"index":3,"sense_key":"ruin%2:40:00::","synset_id":"bankrupt.v.01"},{"index":4,"sense_key":"ruin%2:35:01::","synset_id":"ruin.v.04"},{"index":5,"sense_key":"ruin%2:35:02::","synset_id":"deflower.v.01"},{"index":6,"sense_key":"ruin%2:30:00::","synset_id":"ruin.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"ruin%2:41:00::","call_ids":["senseval3.d000.s018.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"ruin%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":826,"cached_input_tokens":0,"output_tokens":123,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002882,"input_uncached_usd":0.0016519999999999998,"input_cached_usd":0.0,"output_usd":0.0012300000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7660231250338256}
{"item_id":"senseval3.d000.s018.t006","gold_sense_keys":["record%1:04:01::"],"candidates":[{"index":1,"sense_key":"record%1:10:03::","synset_id":"record.n.01"},{"index":2,"sense_key":"record%1:06:00::","synset_id":"phonograph_record.n.01"},{"index":3,"sense_key":"record%1:23:00::","synset_id":"record.n.03"},{"index":4,"sense_key":"record%1:04:01::","synset_id":"record.n.04"},{"index":5,"sense_key":"record%1:10:02::","synset_id":"record.n.05"},{"index":6,"sense_key":"record%1:04:00::","synset_id":"record.n.06"},{"index":7,"sense_key":"record%1:21:00::","synset_id":"record.n.07"},{"index":8,"sense_key":"record%1:10:00::","synset_id":"criminal_record.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"record%1:04:01::","call_ids":["senseval3.d000.s018.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"record%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1061,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0027819999999999998,"input_uncached_usd":0.0021219999999999998,"input_cached_usd":0.0,"output_usd":0.0006600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8361402919981629}
{"item_id":"senseval3.d000.s019.t000","gold_sense_keys":["man%1:18:00::"],"candidates":[{"index":1,"sense_key":"man%1:18:00::","synset_id":"man.n.01"},{"index":2,"sense_key":"man%1:18:03::","synset_id":"serviceman.n.01"},{"index":3,"sense_key":"man%1:18:04::","synset_id":"man.n.03"},{"index":4,"sense_key":"man%1:05:01::","synset_id":"homo.n.02"},{"index":5,"sense_key":"man%1:18:08::","synset_id":"man.n.05"},{"index":6,"sense_key":"man%1:18:06::","synset_id":"man.n.06"},{"index":7,"sense_key":"man%1:18:05::","synset_id":"valet.n.01"},{"index":8,"sense_key":"man%1:18:07::","synset_id":"man.n.08"},{"index":9,"sense_key":"man%1:15:00::","synset_id":"man.n.09"},{"index":10,"sense_key":"man%1:06:00::","synset_id":"man.n.10"},{"index":11,"sense_key":"man%1:05:00::","synset_id":"world.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"man%1:18:00::","call_ids":["senseval3.d000.s019.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"man%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1209,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002538,"input_uncached_usd":0.002418,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1161577920429409}
{"item_id":"senseval3.d000.s019.t001","gold_sense_keys":["save%2:41:00::"],"candidates":[{"index":1,"sense_key":"save%2:41:01::","synset_id":"salvage.v.01"},{"index":2,"sense_key":"save%2:40:00::","synset_id":"save.v.02"},{"index":3,"sense_key":"save%2:41:00::","synset_id":"save.v.03"},{"index":4,"sense_key":"save%2:40:02::","synset_id":"save.v.04"},{"index":5,"sense_key":"save%2:40:01::","synset_id":"save.v.05"},{"index":6,"sense_key":"save%2:30:00::","synset_id":"save.v.06"},{"index":7,"sense_key":"save%2:41:02::","synset_id":"deliver.v.08"},{"index":8,"sense_key":"save%2:41:05::","synset_id":"spare.v.01"},{"index":9,"sense_key":"save%2:40:04::","synset_id":"save.v.09"},{"index":10,"sense_key":"save%2:40:03::","synset_id":"keep_open.v.01"},{"index":11,"sense_key":"save%2:32:04::","synset_id":"write.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"save%2:41:01::","call_ids":["senseval3.d000.s019.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"save%2:41:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1055,"cached_input_tokens":0,"output_tokens":112,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0032300000000000002,"input_uncached_usd":0.00211,"input_cached_usd":0.0,"output_usd":0.0011200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.160781415994279}
{"item_id":"senseval3.d000.s019.t002","gold_sense_keys":["turn%2:38:00::"],"candidates":[{"index":1,"sense_key":"turn%2:38:00::","synset_id":"turn.v.01"},{"index":2,"sense_key":"turn%2:30:04::","synset_id":"change_state.v.01"},{"index":3,"sense_key":"turn%2:42:00::","synset_id":"become.v.02"},{"index":4,"sense_key":"turn%2:38:02::","synset_id":"turn.v.04"},{"index":5,"sense_key":"turn%2:30:00::","synset_id":"change_by_reversal.v.01"},{"index":6,"sense_key":"turn%2:38:03::","synset_id":"turn.v.06"},{"index":7,"sense_key":"turn%2:30:03::","synset_id":"turn.v.07"},{"index":8,"sense_key":"turn%2:35:08::","synset_id":"turn.v.08"},{"index":9,"sense_key":"turn%2:38:04::","synset_id":"turn.v.09"},{"index":10,"sense_key":"turn%2:38:01::","synset_id":"turn.v.10"},{"index":11,"sense_key":"turn%2:38:06::","synset_id":"turn.v.11"},{"index":12,"sense_key":"turn%2:36:00::","synset_id":"plow.v.01"},{"index":13,"sense_key":"turn%2:35:06::","synset_id":"turn.v.13"},{"index":14,"sense_key":"turn%2:30:09::","synset_id":"turn.v.14"},{"index":15,"sense_key":"turn%2:29:00::","synset_id":"twist.v.10"},{"index":16,"sense_key":"turn%2:42:13::","synset_id":"turn.v.16"},{"index":17,"sense_key":"turn%2:41:10::","synset_id":"turn.v.17"},{"index":18,"sense_key":"turn%2:40:13::","synset_id":"turn.v.18"},{"index":19,"sense_key":"turn%2:38:13::","synset_id":"turn.v.19"},{"index":20,"sense_key":"turn%2:38:14::","synset_id":"turn.v.20"},{"index":21,"sense_key":"turn%2:35:11::","synset_id":"flex.v.05"},{"index":22,"sense_key":"turn%2:35:10::","synset_id":"turn.v.22"},{"index":23,"sense_key":"turn%2:33:13::","synset_id":"turn.v.23"},{"index":24,"sense_key":"turn%2:32:14::","synset_id":"call_on.v.01"},{"index":25,"sense_key":"turn%2:30:01::","synset_id":"sour.v.01"},{"index":26,"sense_key":"turn%2:30:14::","synset_id":"turn.v.26"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"turn%2:38:00::","call_ids":["senseval3.d000.s019.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"turn%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2031,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004742,"input_uncached_usd":0.004062,"input_cached_usd":0.0,"output_usd":0.00068,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3904649999458343}
{"item_id":"senseval3.d000.s019.t003","gold_sense_keys":["look%2:39:00::"],"candidates":[{"index":1,"sense_key":"look%2:39:00::","synset_id":"look.v.01"},{"index":2,"sense_key":"look%2:39:01::","synset_id":"look.v.02"},{"index":3,"sense_key":"look%2:29:00::","synset_id":"look.v.03"},{"index":4,"sense_key":"look%2:39:02::","synset_id":"search.v.02"},{"index":5,"sense_key":"look%2:42:00::","synset_id":"front.v.01"},{"index":6,"sense_key":"look%2:41:13::","synset_id":"attend.v.02"},{"index":7,"sense_key":"look%2:32:00::","synset_id":"look.v.07"},{"index":8,"sense_key":"look%2:31:00::","synset_id":"expect.v.03"},{"index":9,"sense_key":"look%2:42:02::","synset_id":"look.v.09"},{"index":10,"sense_key":"look%2:31:02::","synset_id":"count.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"look%2:39:00::","call_ids":["senseval3.d000.s019.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"look%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1084,"cached_input_tokens":0,"output_tokens":106,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003228,"input_uncached_usd":0.002168,"input_cached_usd":0.0,"output_usd":0.0010600000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8207221670309082}
{"item_id":"senseval3.d000.s019.t005","gold_sense_keys":["face%1:08:00::"],"candidates":[{"index":1,"sense_key":"face%1:08:00::","synset_id":"face.n.01"},{"index":2,"sense_key":"face%1:07:03::","synset_id":"expression.n.01"},{"index":3,"sense_key":"face%1:07:00::","synset_id":"face.n.03"},{"index":4,"sense_key":"face%1:06:02::","synset_id":"face.n.04"},{"index":5,"sense_key":"face%1:18:00::","synset_id":"face.n.05"},{"index":6,"sense_key":"face%1:15:00::","synset_id":"side.n.04"},{"index":7,"sense_key":"face%1:08:01::","synset_id":"face.n.07"},{"index":8,"sense_key":"face%1:06:01::","synset_id":"face.n.08"},{"index":9,"sense_key":"face%1:10:01::","synset_id":"grimace.n.01"},{"index":10,"sense_key":"face%1:10:00::","synset_id":"font.n.01"},{"index":11,"sense_key":"face%1:07:01::","synset_id":"face.n.11"},{"index":12,"sense_key":"face%1:07:02::","synset_id":"boldness.n.02"},{"index":13,"sense_key":"face%1:06:00::","synset_id":"face.n.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"face%1:08:00::","call_ids":["senseval3.d000.s019.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"face%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1169,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0024579999999999997,"input_uncached_usd":0.002338,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0594737500650808}
{"item_id":"senseval3.d000.s019.t006","gold_sense_keys":["say%2:32:15::"],"candidates":[{"index":1,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":6,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":9,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":10,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":11,"sense_key":"say%2:32:09::","synset_id":"say.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"say%2:32:15::","call_ids":["senseval3.d000.s019.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"say%2:32:15::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1128,"cached_input_tokens":0,"output_tokens":97,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0032259999999999997,"input_uncached_usd":0.0022559999999999998,"input_cached_usd":0.0,"output_usd":0.00097,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5795453749597073}
{"item_id":"senseval3.d000.s019.t007","gold_sense_keys":["word%1:10:03::"],"candidates":[{"index":1,"sense_key":"word%1:10:00::","synset_id":"word.n.01"},{"index":2,"sense_key":"word%1:10:03::","synset_id":"word.n.02"},{"index":3,"sense_key":"word%1:10:02::","synset_id":"news.n.01"},{"index":4,"sense_key":"word%1:10:06::","synset_id":"word.n.04"},{"index":5,"sense_key":"word%1:10:04::","synset_id":"discussion.n.02"},{"index":6,"sense_key":"word%1:10:01::","synset_id":"parole.n.01"},{"index":7,"sense_key":"word%1:23:00::","synset_id":"word.n.07"},{"index":8,"sense_key":"word%1:18:00::","synset_id":"son.n.02"},{"index":9,"sense_key":"word%1:10:05::","synset_id":"password.n.01"},{"index":10,"sense_key":"word%1:10:07::","synset_id":"bible.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"word%1:10:03::","call_ids":["senseval3.d000.s019.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"word%1:10:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1049,"cached_input_tokens":0,"output_tokens":120,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003298,"input_uncached_usd":0.002098,"input_cached_usd":0.0,"output_usd":0.0012000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9459420829080045}
{"item_id":"senseval3.d000.s020.t000","gold_sense_keys":["driver%1:18:00::"],"candidates":[{"index":1,"sense_key":"driver%1:18:00::","synset_id":"driver.n.01"},{"index":2,"sense_key":"driver%1:18:02::","synset_id":"driver.n.02"},{"index":3,"sense_key":"driver%1:18:01::","synset_id":"driver.n.03"},{"index":4,"sense_key":"driver%1:10:00::","synset_id":"driver.n.04"},{"index":5,"sense_key":"driver%1:06:00::","synset_id":"driver.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"driver%1:18:00::","call_ids":["senseval3.d000.s020.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"driver%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":681,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001482,"input_uncached_usd":0.001362,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1916561250109226}
{"item_id":"senseval3.d000.s020.t001","gold_sense_keys":["stop%2:42:00::"],"candidates":[{"index":1,"sense_key":"stop%2:38:00::","synset_id":"stop.v.01"},{"index":2,"sense_key":"stop%2:42:00::","synset_id":"discontinue.v.01"},{"index":3,"sense_key":"stop%2:41:00::","synset_id":"stop.v.03"},{"index":4,"sense_key":"stop%2:38:02::","synset_id":"stop.v.04"},{"index":5,"sense_key":"stop%2:38:01::","synset_id":"stop.v.05"},{"index":6,"sense_key":"stop%2:30:05::","synset_id":"break.v.10"},{"index":7,"sense_key":"stop%2:33:00::","synset_id":"check.v.18"},{"index":8,"sense_key":"stop%2:35:00::","synset_id":"intercept.v.01"},{"index":9,"sense_key":"stop%2:42:13::","synset_id":"end.v.01"},{"index":10,"sense_key":"stop%2:35:12::","synset_id":"barricade.v.01"},{"index":11,"sense_key":"stop%2:30:12::","synset_id":"hold_on.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"stop%2:42:00::","call_ids":["senseval3.d000.s020.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"stop%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1217,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003064,"input_uncached_usd":0.002434,"input_cached_usd":0.0,"output_usd":0.00063,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.101457041921094}
{"item_id":"senseval3.d000.s020.t002","gold_sense_keys":["swear%2:32:01::"],"candidates":[{"index":1,"sense_key":"swear%2:32:01::","synset_id":"curse.v.01"},{"index":2,"sense_key":"swear%2:32:03::","synset_id":"affirm.v.02"},{"index":3,"sense_key":"swear%2:32:02::","synset_id":"swear.v.03"},{"index":4,"sense_key":"swear%2:32:00::","synset_id":"swear.v.04"},{"index":5,"sense_key":"swear%2:31:10::","synset_id":"trust.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"swear%2:32:01::","call_ids":["senseval3.d000.s020.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"swear%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":746,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001612,"input_uncached_usd":0.0014919999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1377143330173567}
{"item_id":"senseval3.d000.s020.t003","gold_sense_keys":["turn%2:38:00::"],"candidates":[{"index":1,"sense_key":"turn%2:38:00::","synset_id":"turn.v.01"},{"index":2,"sense_key":"turn%2:30:04::","synset_id":"change_state.v.01"},{"index":3,"sense_key":"turn%2:42:00::","synset_id":"become.v.02"},{"index":4,"sense_key":"turn%2:38:02::","synset_id":"turn.v.04"},{"index":5,"sense_key":"turn%2:30:00::","synset_id":"change_by_reversal.v.01"},{"index":6,"sense_key":"turn%2:38:03::","synset_id":"turn.v.06"},{"index":7,"sense_key":"turn%2:30:03::","synset_id":"turn.v.07"},{"index":8,"sense_key":"turn%2:35:08::","synset_id":"turn.v.08"},{"index":9,"sense_key":"turn%2:38:04::","synset_id":"turn.v.09"},{"index":10,"sense_key":"turn%2:38:01::","synset_id":"turn.v.10"},{"index":11,"sense_key":"turn%2:38:06::","synset_id":"turn.v.11"},{"index":12,"sense_key":"turn%2:36:00::","synset_id":"plow.v.01"},{"index":13,"sense_key":"turn%2:35:06::","synset_id":"turn.v.13"},{"index":14,"sense_key":"turn%2:30:09::","synset_id":"turn.v.14"},{"index":15,"sense_key":"turn%2:29:00::","synset_id":"twist.v.10"},{"index":16,"sense_key":"turn%2:42:13::","synset_id":"turn.v.16"},{"index":17,"sense_key":"turn%2:41:10::","synset_id":"turn.v.17"},{"index":18,"sense_key":"turn%2:40:13::","synset_id":"turn.v.18"},{"index":19,"sense_key":"turn%2:38:13::","synset_id":"turn.v.19"},{"index":20,"sense_key":"turn%2:38:14::","synset_id":"turn.v.20"},{"index":21,"sense_key":"turn%2:35:11::","synset_id":"flex.v.05"},{"index":22,"sense_key":"turn%2:35:10::","synset_id":"turn.v.22"},{"index":23,"sense_key":"turn%2:33:13::","synset_id":"turn.v.23"},{"index":24,"sense_key":"turn%2:32:14::","synset_id":"call_on.v.01"},{"index":25,"sense_key":"turn%2:30:01::","synset_id":"sour.v.01"},{"index":26,"sense_key":"turn%2:30:14::","synset_id":"turn.v.26"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"turn%2:38:00::","call_ids":["senseval3.d000.s020.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"turn%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2016,"cached_input_tokens":0,"output_tokens":145,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.005482,"input_uncached_usd":0.004032,"input_cached_usd":0.0,"output_usd":0.0014500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.20278154197149}
{"item_id":"senseval3.d000.s020.t004","gold_sense_keys":["heel%1:08:00::"],"candidates":[{"index":1,"sense_key":"heel%1:06:00::","synset_id":"heel.n.01"},{"index":2,"sense_key":"heel%1:08:00::","synset_id":"heel.n.02"},{"index":3,"sense_key":"heel%1:18:00::","synset_id":"cad.n.01"},{"index":4,"sense_key":"heel%1:13:00::","synset_id":"heel.n.04"},{"index":5,"sense_key":"heel%1:06:03::","synset_id":"heel.n.05"},{"index":6,"sense_key":"heel%1:06:02::","synset_id":"heel.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"heel%1:08:00::","call_ids":["senseval3.d000.s020.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"heel%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":722,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002064,"input_uncached_usd":0.001444,"input_cached_usd":0.0,"output_usd":0.00062,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3186054170364514}
{"item_id":"senseval3.d000.s020.t005","gold_sense_keys":["truck%1:06:00::"],"candidates":[{"index":1,"sense_key":"truck%1:06:00::","synset_id":"truck.n.01"},{"index":2,"sense_key":"truck%1:06:01::","synset_id":"hand_truck.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"truck%1:06:00::","call_ids":["senseval3.d000.s020.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"truck%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":578,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001276,"input_uncached_usd":0.001156,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0251653330633417}
{"item_id":"senseval3.d000.s021.t000","gold_sense_keys":["give%2:31:02::"],"candidates":[{"index":1,"sense_key":"give%2:40:03::","synset_id":"give.v.01"},{"index":2,"sense_key":"give%2:40:05::","synset_id":"yield.v.01"},{"index":3,"sense_key":"give%2:40:00::","synset_id":"give.v.03"},{"index":4,"sense_key":"give%2:40:07::","synset_id":"give.v.04"},{"index":5,"sense_key":"give%2:32:01::","synset_id":"give.v.05"},{"index":6,"sense_key":"give%2:36:02::","synset_id":"hold.v.03"},{"index":7,"sense_key":"give%2:32:00::","synset_id":"give.v.07"},{"index":8,"sense_key":"give%2:40:01::","synset_id":"give.v.08"},{"index":9,"sense_key":"give%2:36:03::","synset_id":"give.v.09"},{"index":10,"sense_key":"give%2:31:02::","synset_id":"give.v.10"},{"index":11,"sense_key":"give%2:36:00::","synset_id":"render.v.04"},{"index":12,"sense_key":"give%2:40:09::","synset_id":"impart.v.01"},{"index":13,"sense_key":"give%2:36:01::","synset_id":"establish.v.05"},{"index":14,"sense_key":"give%2:40:02::","synset_id":"give.v.14"},{"index":15,"sense_key":"give%2:41:00::","synset_id":"give.v.15"},{"index":16,"sense_key":"give%2:40:10::","synset_id":"sacrifice.v.01"},{"index":17,"sense_key":"give%2:40:06::","synset_id":"pass.v.05"},{"index":18,"sense_key":"give%2:32:03::","synset_id":"give.v.18"},{"index":19,"sense_key":"give%2:40:13::","synset_id":"give.v.19"},{"index":20,"sense_key":"give%2:40:14::","synset_id":"give.v.20"},{"index":21,"sense_key":"give%2:32:02::","synset_id":"give.v.21"},{"index":22,"sense_key":"give%2:40:11::","synset_id":"grant.v.05"},{"index":23,"sense_key":"give%2:38:00::","synset_id":"move_over.v.01"},{"index":24,"sense_key":"give%2:34:00::","synset_id":"feed.v.02"},{"index":25,"sense_key":"give%2:40:08::","synset_id":"contribute.v.02"},{"index":26,"sense_key":"give%2:38:13::","synset_id":"collapse.v.01"},{"index":27,"sense_key":"give%2:31:00::","synset_id":"give.v.27"},{"index":28,"sense_key":"give%2:41:10::","synset_id":"give.v.28"},{"index":29,"sense_key":"give%2:40:12::","synset_id":"give.v.29"},{"index":30,"sense_key":"give%2:40:15::","synset_id":"afford.v.04"},{"index":31,"sense_key":"give%2:36:13::","synset_id":"give.v.31"},{"index":32,"sense_key":"give%2:36:12::","synset_id":"give.v.32"},{"index":33,"sense_key":"give%2:35:00::","synset_id":"give.v.33"},{"index":34,"sense_key":"give%2:34:10::","synset_id":"give.v.34"},{"index":35,"sense_key":"give%2:32:11::","synset_id":"give.v.35"},{"index":36,"sense_key":"give%2:32:08::","synset_id":"give.v.36"},{"index":37,"sense_key":"give%2:32:15::","synset_id":"give.v.37"},{"index":38,"sense_key":"give%2:32:14::","synset_id":"give.v.38"},{"index":39,"sense_key":"give%2:32:09::","synset_id":"give.v.39"},{"index":40,"sense_key":"give%2:32:07::","synset_id":"give.v.40"},{"index":41,"sense_key":"give%2:32:10::","synset_id":"give.v.41"},{"index":42,"sense_key":"give%2:30:13::","synset_id":"give.v.42"},{"index":43,"sense_key":"give%2:29:11::","synset_id":"give.v.43"},{"index":44,"sense_key":"give%2:29:10::","synset_id":"give.v.44"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"give%2:31:02::","call_ids":["senseval3.d000.s021.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"give%2:31:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2861,"cached_input_tokens":0,"output_tokens":129,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.007012,"input_uncached_usd":0.005722,"input_cached_usd":0.0,"output_usd":0.0012900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9791675000451505}
{"item_id":"senseval3.d000.s021.t002","gold_sense_keys":["thought%1:09:00::"],"candidates":[{"index":1,"sense_key":"thought%1:09:01::","synset_id":"idea.n.01"},{"index":2,"sense_key":"thought%1:09:00::","synset_id":"thinking.n.01"},{"index":3,"sense_key":"thought%1:09:02::","synset_id":"thought.n.03"},{"index":4,"sense_key":"thought%1:09:03::","synset_id":"opinion.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"thought%1:09:00::","call_ids":["senseval3.d000.s021.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"thought%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":753,"cached_input_tokens":0,"output_tokens":48,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001986,"input_uncached_usd":0.001506,"input_cached_usd":0.0,"output_usd":0.00048000000000000007,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8097770829917863}
{"item_id":"senseval3.d000.s022.t000","gold_sense_keys":["recall%2:31:00::"],"candidates":[{"index":1,"sense_key":"recall%2:31:00::","synset_id":"remember.v.01"},{"index":2,"sense_key":"recall%2:32:00::","synset_id":"hark_back.v.01"},{"index":3,"sense_key":"recall%2:42:00::","synset_id":"echo.v.03"},{"index":4,"sense_key":"recall%2:32:02::","synset_id":"recall.v.04"},{"index":5,"sense_key":"recall%2:31:06::","synset_id":"recall.v.05"},{"index":6,"sense_key":"recall%2:41:00::","synset_id":"recall.v.06"},{"index":7,"sense_key":"recall%2:40:00::","synset_id":"recall.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"recall%2:31:00::","call_ids":["senseval3.d000.s022.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"recall%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":908,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001936,"input_uncached_usd":0.001816,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1779932079371065}
{"item_id":"senseval3.d000.s022.t001","gold_sense_keys":["wonder%2:31:00::"],"candidates":[{"index":1,"sense_key":"wonder%2:31:00::","synset_id":"wonder.v.01"},{"index":2,"sense_key":"wonder%2:32:01::","synset_id":"wonder.v.02"},{"index":3,"sense_key":"wonder%2:32:12::","synset_id":"wonder.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"wonder%2:31:00::","call_ids":["senseval3.d000.s022.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"wonder%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":637,"cached_input_tokens":0,"output_tokens":113,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002404,"input_uncached_usd":0.001274,"input_cached_usd":0.0,"output_usd":0.0011300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6453639169922099}
{"item_id":"senseval3.d000.s022.t002","gold_sense_keys":["see%2:36:00::","see%2:39:00::"],"candidates":[{"index":1,"sense_key":"see%2:39:00::","synset_id":"see.v.01"},{"index":2,"sense_key":"see%2:31:01::","synset_id":"understand.v.02"},{"index":3,"sense_key":"see%2:39:02::","synset_id":"witness.v.02"},{"index":4,"sense_key":"see%2:36:00::","synset_id":"visualize.v.01"},{"index":5,"sense_key":"see%2:31:00::","synset_id":"see.v.05"},{"index":6,"sense_key":"see%2:31:02::","synset_id":"learn.v.02"},{"index":7,"sense_key":"see%2:39:03::","synset_id":"watch.v.03"},{"index":8,"sense_key":"see%2:38:00::","synset_id":"meet.v.01"},{"index":9,"sense_key":"see%2:32:00::","synset_id":"determine.v.08"},{"index":10,"sense_key":"see%2:31:03::","synset_id":"see.v.10"},{"index":11,"sense_key":"see%2:41:11::","synset_id":"see.v.11"},{"index":12,"sense_key":"see%2:41:12::","synset_id":"see.v.12"},{"index":13,"sense_key":"see%2:41:10::","synset_id":"visit.v.01"},{"index":14,"sense_key":"see%2:41:14::","synset_id":"attend.v.02"},{"index":15,"sense_key":"see%2:35:09::","synset_id":"see.v.15"},{"index":16,"sense_key":"see%2:41:00::","synset_id":"go_steady.v.01"},{"index":17,"sense_key":"see%2:39:05::","synset_id":"see.v.17"},{"index":18,"sense_key":"see%2:31:08::","synset_id":"see.v.18"},{"index":19,"sense_key":"see%2:39:11::","synset_id":"see.v.19"},{"index":20,"sense_key":"see%2:39:13::","synset_id":"examine.v.02"},{"index":21,"sense_key":"see%2:39:12::","synset_id":"experience.v.01"},{"index":22,"sense_key":"see%2:38:03::","synset_id":"see.v.22"},{"index":23,"sense_key":"see%2:33:00::","synset_id":"see.v.23"},{"index":24,"sense_key":"see%2:31:13::","synset_id":"interpret.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"see%2:39:00::","call_ids":["senseval3.d000.s022.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"see%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1993,"cached_input_tokens":0,"output_tokens":125,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.005236,"input_uncached_usd":0.003986,"input_cached_usd":0.0,"output_usd":0.00125,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0938087919494137}
{"item_id":"senseval3.d000.s022.t003","gold_sense_keys":["eye%1:08:00::"],"candidates":[{"index":1,"sense_key":"eye%1:08:00::","synset_id":"eye.n.01"},{"index":2,"sense_key":"eye%1:09:00::","synset_id":"eye.n.02"},{"index":3,"sense_key":"eye%1:09:01::","synset_id":"eye.n.03"},{"index":4,"sense_key":"eye%1:15:00::","synset_id":"center.n.01"},{"index":5,"sense_key":"eye%1:06:00::","synset_id":"eye.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"eye%1:08:00::","call_ids":["senseval3.d000.s022.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"eye%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":712,"cached_input_tokens":0,"output_tokens":100,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002424,"input_uncached_usd":0.001424,"input_cached_usd":0.0,"output_usd":0.001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9471700409194455}
{"item_id":"senseval3.d000.s022.t004","gold_sense_keys":["make%2:32:00::","make%2:36:15::"],"candidates":[{"index":1,"sense_key":"make%2:41:00::","synset_id":"make.v.01"},{"index":2,"sense_key":"make%2:30:00::","synset_id":"make.v.02"},{"index":3,"sense_key":"make%2:36:00::","synset_id":"make.v.03"},{"index":4,"sense_key":"make%2:32:00::","synset_id":"induce.v.02"},{"index":5,"sense_key":"make%2:36:08::","synset_id":"cause.v.01"},{"index":6,"sense_key":"make%2:36:01::","synset_id":"produce.v.02"},{"index":7,"sense_key":"make%2:31:00::","synset_id":"draw.v.04"},{"index":8,"sense_key":"make%2:36:15::","synset_id":"make.v.08"},{"index":9,"sense_key":"make%2:36:09::","synset_id":"create.v.05"},{"index":10,"sense_key":"make%2:40:01::","synset_id":"gain.v.08"},{"index":11,"sense_key":"make%2:36:05::","synset_id":"do.v.08"},{"index":12,"sense_key":"make%2:42:05::","synset_id":"form.v.02"},{"index":13,"sense_key":"make%2:38:02::","synset_id":"reach.v.07"},{"index":14,"sense_key":"make%2:42:00::","synset_id":"make.v.14"},{"index":15,"sense_key":"make%2:36:11::","synset_id":"make.v.15"},{"index":16,"sense_key":"make%2:36:12::","synset_id":"make.v.16"},{"index":17,"sense_key":"make%2:36:13::","synset_id":"construct.v.01"},{"index":18,"sense_key":"make%2:30:15::","synset_id":"make.v.18"},{"index":19,"sense_key":"make%2:29:08::","synset_id":"make.v.19"},{"index":20,"sense_key":"make%2:41:03::","synset_id":"name.v.03"},{"index":21,"sense_key":"make%2:40:02::","synset_id":"have.v.17"},{"index":22,"sense_key":"make%2:38:00::","synset_id":"reach.v.01"},{"index":23,"sense_key":"make%2:31:13::","synset_id":"lay_down.v.01"},{"index":24,"sense_key":"make%2:41:13::","synset_id":"make.v.24"},{"index":25,"sense_key":"make%2:41:01::","synset_id":"make.v.25"},{"index":26,"sense_key":"make%2:36:04::","synset_id":"hold.v.03"},{"index":27,"sense_key":"make%2:30:02::","synset_id":"make.v.27"},{"index":28,"sense_key":"make%2:38:05::","synset_id":"take.v.27"},{"index":29,"sense_key":"make%2:29:00::","synset_id":"stool.v.04"},{"index":30,"sense_key":"make%2:42:09::","synset_id":"make.v.30"},{"index":31,"sense_key":"make%2:42:13::","synset_id":"make.v.31"},{"index":32,"sense_key":"make%2:42:12::","synset_id":"make.v.32"},{"index":33,"sense_key":"make%2:42:11::","synset_id":"make.v.33"},{"index":34,"sense_key":"make%2:42:15::","synset_id":"make.v.34"},{"index":35,"sense_key":"make%2:39:12::","synset_id":"make.v.35"},{"index":36,"sense_key":"make%2:38:10::","synset_id":"make.v.36"},{"index":37,"sense_key":"make%2:38:15::","synset_id":"make.v.37"},{"index":38,"sense_key":"make%2:36:10::","synset_id":"make.v.38"},{"index":39,"sense_key":"make%2:36:07::","synset_id":"cook.v.02"},{"index":40,"sense_key":"make%2:35:13::","synset_id":"seduce.v.01"},{"index":41,"sense_key":"make%2:32:14::","synset_id":"make.v.41"},{"index":42,"sense_key":"make%2:32:09::","synset_id":"make.v.42"},{"index":43,"sense_key":"make%2:31:10::","synset_id":"make.v.43"},{"index":44,"sense_key":"make%2:31:12::","synset_id":"make.v.44"},{"index":45,"sense_key":"make%2:30:10::","synset_id":"make.v.45"},{"index":46,"sense_key":"make%2:30:14::","synset_id":"make.v.46"},{"index":47,"sense_key":"make%2:30:12::","synset_id":"make.v.47"},{"index":48,"sense_key":"make%2:29:06::","synset_id":"make.v.48"},{"index":49,"sense_key":"make%2:29:01::","synset_id":"make.v.49"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"make%2:36:15::","call_ids":["senseval3.d000.s022.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"make%2:36:15::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":3104,"cached_input_tokens":0,"output_tokens":252,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.008728,"input_uncached_usd":0.006208,"input_cached_usd":0.0,"output_usd":0.00252,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.429789250018075}
{"item_id":"senseval3.d000.s023.t000","gold_sense_keys":["sort%1:09:00::"],"candidates":[{"index":1,"sense_key":"sort%1:09:00::","synset_id":"kind.n.01"},{"index":2,"sense_key":"sort%1:07:00::","synset_id":"sort.n.02"},{"index":3,"sense_key":"sort%1:18:00::","synset_id":"sort.n.03"},{"index":4,"sense_key":"sort%1:22:00::","synset_id":"sort.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"sort%1:09:00::","call_ids":["senseval3.d000.s023.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"sort%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":679,"cached_input_tokens":0,"output_tokens":124,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0025979999999999996,"input_uncached_usd":0.0013579999999999998,"input_cached_usd":0.0,"output_usd":0.00124,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6896544999908656}
{"item_id":"senseval3.d000.s023.t001","gold_sense_keys":["look%1:07:01::"],"candidates":[{"index":1,"sense_key":"look%1:07:01::","synset_id":"expression.n.01"},{"index":2,"sense_key":"look%1:04:00::","synset_id":"look.n.02"},{"index":3,"sense_key":"look%1:07:00::","synset_id":"look.n.03"},{"index":4,"sense_key":"look%1:26:00::","synset_id":"spirit.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"look%1:07:01::","call_ids":["senseval3.d000.s023.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"look%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":697,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002074,"input_uncached_usd":0.0013939999999999998,"input_cached_usd":0.0,"output_usd":0.00068,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.414642250048928}
{"item_id":"senseval3.d000.s023.t003","gold_sense_keys":["bluff%1:10:00::"],"candidates":[{"index":1,"sense_key":"bluff%1:17:00::","synset_id":"bluff.n.01"},{"index":2,"sense_key":"bluff%1:10:00::","synset_id":"bluff.n.02"},{"index":3,"sense_key":"bluff%1:04:00::","synset_id":"bluff.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bluff%1:10:00::","call_ids":["senseval3.d000.s023.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bluff%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":575,"cached_input_tokens":0,"output_tokens":82,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00197,"input_uncached_usd":0.00115,"input_cached_usd":0.0,"output_usd":0.0008200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5654067079303786}
{"item_id":"senseval3.d000.s023.t004","gold_sense_keys":["say%2:32:15::"],"candidates":[{"index":1,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":6,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":9,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":10,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":11,"sense_key":"say%2:32:09::","synset_id":"say.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"say%2:32:15::","call_ids":["senseval3.d000.s023.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"say%2:32:15::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1074,"cached_input_tokens":0,"output_tokens":111,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0032579999999999996,"input_uncached_usd":0.0021479999999999997,"input_cached_usd":0.0,"output_usd":0.00111,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6698271670611575}
{"item_id":"senseval3.d000.s023.t005","gold_sense_keys":["word%1:10:03::"],"candidates":[{"index":1,"sense_key":"word%1:10:00::","synset_id":"word.n.01"},{"index":2,"sense_key":"word%1:10:03::","synset_id":"word.n.02"},{"index":3,"sense_key":"word%1:10:02::","synset_id":"news.n.01"},{"index":4,"sense_key":"word%1:10:06::","synset_id":"word.n.04"},{"index":5,"sense_key":"word%1:10:04::","synset_id":"discussion.n.02"},{"index":6,"sense_key":"word%1:10:01::","synset_id":"parole.n.01"},{"index":7,"sense_key":"word%1:23:00::","synset_id":"word.n.07"},{"index":8,"sense_key":"word%1:18:00::","synset_id":"son.n.02"},{"index":9,"sense_key":"word%1:10:05::","synset_id":"password.n.01"},{"index":10,"sense_key":"word%1:10:07::","synset_id":"bible.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"word%1:10:03::","call_ids":["senseval3.d000.s023.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"word%1:10:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":995,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00252,"input_uncached_usd":0.00199,"input_cached_usd":0.0,"output_usd":0.0005300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6269210419850424}
{"item_id":"senseval3.d000.s024.t000","gold_sense_keys":["light%1:10:00::"],"candidates":[{"index":1,"sense_key":"light%1:19:00::","synset_id":"light.n.01"},{"index":2,"sense_key":"light%1:06:00::","synset_id":"light.n.02"},{"index":3,"sense_key":"light%1:09:01::","synset_id":"light.n.03"},{"index":4,"sense_key":"light%1:07:01::","synset_id":"luminosity.n.01"},{"index":5,"sense_key":"light%1:15:00::","synset_id":"light.n.05"},{"index":6,"sense_key":"light%1:26:01::","synset_id":"light.n.06"},{"index":7,"sense_key":"light%1:07:00::","synset_id":"light.n.07"},{"index":8,"sense_key":"light%1:18:00::","synset_id":"light.n.08"},{"index":9,"sense_key":"light%1:26:00::","synset_id":"light.n.09"},{"index":10,"sense_key":"light%1:09:00::","synset_id":"light.n.10"},{"index":11,"sense_key":"light%1:07:02::","synset_id":"sparkle.n.01"},{"index":12,"sense_key":"light%1:09:02::","synset_id":"light.n.12"},{"index":13,"sense_key":"light%1:16:00::","synset_id":"inner_light.n.01"},{"index":14,"sense_key":"light%1:10:00::","synset_id":"light.n.14"},{"index":15,"sense_key":"light%1:06:01::","synset_id":"lighter.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":14,"chosen_sense_key":"light%1:10:00::","call_ids":["senseval3.d000.s024.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":14,"predicted_sense_key":"light%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1337,"cached_input_tokens":0,"output_tokens":108,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0037539999999999995,"input_uncached_usd":0.0026739999999999997,"input_cached_usd":0.0,"output_usd":0.00108,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6345000839792192}
{"item_id":"senseval3.d000.s024.t002","gold_sense_keys":["way%1:15:00::"],"candidates":[{"index":1,"sense_key":"way%1:07:01::","synset_id":"manner.n.01"},{"index":2,"sense_key":"way%1:04:01::","synset_id":"means.n.01"},{"index":3,"sense_key":"way%1:15:00::","synset_id":"direction.n.01"},{"index":4,"sense_key":"way%1:26:00::","synset_id":"way.n.04"},{"index":5,"sense_key":"way%1:04:00::","synset_id":"way.n.05"},{"index":6,"sense_key":"way%1:06:00::","synset_id":"way.n.06"},{"index":7,"sense_key":"way%1:04:02::","synset_id":"way.n.07"},{"index":8,"sense_key":"way%1:23:00::","synset_id":"room.n.02"},{"index":9,"sense_key":"way%1:07:02::","synset_id":"way.n.09"},{"index":10,"sense_key":"way%1:09:00::","synset_id":"way.n.10"},{"index":11,"sense_key":"way%1:09:01::","synset_id":"way.n.11"},{"index":12,"sense_key":"way%1:21:00::","synset_id":"way.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"way%1:15:00::","call_ids":["senseval3.d000.s024.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"way%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1043,"cached_input_tokens":0,"output_tokens":143,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003516,"input_uncached_usd":0.0020859999999999997,"input_cached_usd":0.0,"output_usd":0.00143,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9817122920649126}
{"item_id":"senseval3.d000.s024.t003","gold_sense_keys":["go_on%2:38:00::"],"candidates":[{"index":1,"sense_key":"go_on%2:42:00::","synset_id":"continue.v.01"},{"index":2,"sense_key":"go_on%2:30:00::","synset_id":"happen.v.01"},{"index":3,"sense_key":"go_on%2:38:00::","synset_id":"advance.v.01"},{"index":4,"sense_key":"go_on%2:32:00::","synset_id":"continue.v.02"},{"index":5,"sense_key":"go_on%2:35:00::","synset_id":"go_on.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"go_on%2:38:00::","call_ids":["senseval3.d000.s024.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"go_on%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":767,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004254,"input_uncached_usd":0.001534,"input_cached_usd":0.0,"output_usd":0.00272,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.8907136659836397}
{"item_id":"senseval3.d000.s024.t004","gold_sense_keys":["street%1:06:01::"],"candidates":[{"index":1,"sense_key":"street%1:06:00::","synset_id":"street.n.01"},{"index":2,"sense_key":"street%1:06:01::","synset_id":"street.n.02"},{"index":3,"sense_key":"street%1:26:00::","synset_id":"street.n.03"},{"index":4,"sense_key":"street%1:26:01::","synset_id":"street.n.04"},{"index":5,"sense_key":"street%1:14:00::","synset_id":"street.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"street%1:06:01::","call_ids":["senseval3.d000.s024.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"street%1:06:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":736,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002092,"input_uncached_usd":0.001472,"input_cached_usd":0.0,"output_usd":0.00062,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.740938000031747}
{"item_id":"senseval3.d000.s025.t000","gold_sense_keys":["stranger%1:18:01::"],"candidates":[{"index":1,"sense_key":"stranger%1:18:00::","synset_id":"stranger.n.01"},{"index":2,"sense_key":"stranger%1:18:01::","synset_id":"stranger.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"stranger%1:18:01::","call_ids":["senseval3.d000.s025.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"stranger%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":524,"cached_input_tokens":0,"output_tokens":99,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002038,"input_uncached_usd":0.0010479999999999999,"input_cached_usd":0.0,"output_usd":0.00099,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4884830419905484}
{"item_id":"senseval3.d000.s025.t001","gold_sense_keys":["find_out%2:31:00::"],"candidates":[{"index":1,"sense_key":"find_out%2:32:00::","synset_id":"determine.v.01"},{"index":2,"sense_key":"find_out%2:31:00::","synset_id":"learn.v.02"},{"index":3,"sense_key":"find_out%2:32:01::","synset_id":"determine.v.08"},{"index":4,"sense_key":"find_out%2:31:01::","synset_id":"find_out.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"find_out%2:31:00::","call_ids":["senseval3.d000.s025.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"find_out%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":789,"cached_input_tokens":0,"output_tokens":85,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002428,"input_uncached_usd":0.001578,"input_cached_usd":0.0,"output_usd":0.0008500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.728086707997136}
{"item_id":"senseval3.d000.s025.t003","gold_sense_keys":["favorite%5:00:00:loved:00"],"candidates":[{"index":1,"sense_key":"favorite%5:00:00:popular:00","synset_id":"favorite.s.01"},{"index":2,"sense_key":"favorite%5:00:00:loved:00","synset_id":"favored.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"favorite%5:00:00:loved:00","call_ids":["senseval3.d000.s025.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"favorite%5:00:00:loved:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":570,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0017699999999999999,"input_uncached_usd":0.00114,"input_cached_usd":0.0,"output_usd":0.00063,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8469759169965982}
{"item_id":"senseval3.d000.s025.t004","gold_sense_keys":["bar%1:06:04::"],"candidates":[{"index":1,"sense_key":"bar%1:06:04::","synset_id":"barroom.n.01"},{"index":2,"sense_key":"bar%1:06:05::","synset_id":"bar.n.02"},{"index":3,"sense_key":"bar%1:06:00::","synset_id":"bar.n.03"},{"index":4,"sense_key":"bar%1:10:00::","synset_id":"measure.n.07"},{"index":5,"sense_key":"bar%1:06:02::","synset_id":"bar.n.05"},{"index":6,"sense_key":"bar%1:04:00::","synset_id":"prevention.n.01"},{"index":7,"sense_key":"bar%1:23:00::","synset_id":"bar.n.07"},{"index":8,"sense_key":"bar%1:17:00::","synset_id":"bar.n.08"},{"index":9,"sense_key":"bar%1:14:00::","synset_id":"legal_profession.n.01"},{"index":10,"sense_key":"bar%1:07:00::","synset_id":"stripe.n.05"},{"index":11,"sense_key":"bar%1:06:06::","synset_id":"cake.n.01"},{"index":12,"sense_key":"bar%1:06:01::","synset_id":"browning_automatic_rifle.n.01"},{"index":13,"sense_key":"bar%1:06:09::","synset_id":"bar.n.13"},{"index":14,"sense_key":"bar%1:06:08::","synset_id":"bar.n.14"},{"index":15,"sense_key":"bar%1:06:07::","synset_id":"bar.n.15"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bar%1:06:04::","call_ids":["senseval3.d000.s025.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bar%1:06:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1470,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00306,"input_uncached_usd":0.00294,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4841175000183284}
{"item_id":"senseval3.d000.s025.t005","gold_sense_keys":["insist%2:32:00::"],"candidates":[{"index":1,"sense_key":"insist%2:32:00::","synset_id":"insist.v.01"},{"index":2,"sense_key":"insist%2:32:03::","synset_id":"importune.v.01"},{"index":3,"sense_key":"insist%2:31:00::","synset_id":"insist.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"insist%2:32:00::","call_ids":["senseval3.d000.s025.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"insist%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":593,"cached_input_tokens":0,"output_tokens":105,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002236,"input_uncached_usd":0.001186,"input_cached_usd":0.0,"output_usd":0.0010500000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6462838340085}
{"item_id":"senseval3.d000.s025.t006","gold_sense_keys":["offer%2:41:00::"],"candidates":[{"index":1,"sense_key":"offer%2:40:02::","synset_id":"offer.v.01"},{"index":2,"sense_key":"offer%2:40:00::","synset_id":"offer.v.02"},{"index":3,"sense_key":"offer%2:41:00::","synset_id":"volunteer.v.02"},{"index":4,"sense_key":"offer%2:31:00::","synset_id":"offer.v.04"},{"index":5,"sense_key":"offer%2:40:09::","synset_id":"offer.v.05"},{"index":6,"sense_key":"offer%2:40:03::","synset_id":"offer.v.06"},{"index":7,"sense_key":"offer%2:40:01::","synset_id":"offer.v.07"},{"index":8,"sense_key":"offer%2:39:06::","synset_id":"offer.v.08"},{"index":9,"sense_key":"offer%2:42:00::","synset_id":"offer.v.09"},{"index":10,"sense_key":"offer%2:41:03::","synset_id":"put_up.v.02"},{"index":11,"sense_key":"offer%2:40:10::","synset_id":"extend.v.04"},{"index":12,"sense_key":"offer%2:32:12::","synset_id":"propose.v.05"},{"index":13,"sense_key":"offer%2:32:06::","synset_id":"offer.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"offer%2:41:00::","call_ids":["senseval3.d000.s025.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"offer%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1184,"cached_input_tokens":0,"output_tokens":85,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003218,"input_uncached_usd":0.002368,"input_cached_usd":0.0,"output_usd":0.0008500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5273504169890657}
{"item_id":"senseval3.d000.s025.t007","gold_sense_keys":["buy%2:40:00::"],"candidates":[{"index":1,"sense_key":"buy%2:40:00::","synset_id":"buy.v.01"},{"index":2,"sense_key":"buy%2:40:02::","synset_id":"bribe.v.01"},{"index":3,"sense_key":"buy%2:42:00::","synset_id":"buy.v.03"},{"index":4,"sense_key":"buy%2:40:03::","synset_id":"buy.v.04"},{"index":5,"sense_key":"buy%2:31:09::","synset_id":"buy.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"buy%2:40:00::","call_ids":["senseval3.d000.s025.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"buy%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":729,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001578,"input_uncached_usd":0.0014579999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9829914160072803}
{"item_id":"senseval3.d000.s025.t008","gold_sense_keys":["drink%1:13:04::"],"candidates":[{"index":1,"sense_key":"drink%1:13:04::","synset_id":"drink.n.01"},{"index":2,"sense_key":"drink%1:04:00::","synset_id":"drink.n.02"},{"index":3,"sense_key":"drink%1:13:00::","synset_id":"beverage.n.01"},{"index":4,"sense_key":"drink%1:17:00::","synset_id":"drink.n.04"},{"index":5,"sense_key":"drink%1:04:01::","synset_id":"swallow.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"drink%1:13:04::","call_ids":["senseval3.d000.s025.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"drink%1:13:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":751,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002142,"input_uncached_usd":0.0015019999999999999,"input_cached_usd":0.0,"output_usd":0.00064,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2363912500441074}
{"item_id":"senseval3.d000.s026.t000","gold_sense_keys":["go%2:38:00::"],"candidates":[{"index":1,"sense_key":"go%2:38:00::","synset_id":"travel.v.01"},{"index":2,"sense_key":"go%2:41:00::","synset_id":"go.v.02"},{"index":3,"sense_key":"go%2:38:01::","synset_id":"go.v.03"},{"index":4,"sense_key":"go%2:30:04::","synset_id":"become.v.01"},{"index":5,"sense_key":"go%2:42:06::","synset_id":"go.v.05"},{"index":6,"sense_key":"go%2:42:02::","synset_id":"run.v.05"},{"index":7,"sense_key":"go%2:42:00::","synset_id":"run.v.03"},{"index":8,"sense_key":"go%2:30:01::","synset_id":"proceed.v.04"},{"index":9,"sense_key":"go%2:30:08::","synset_id":"go.v.09"},{"index":10,"sense_key":"go%2:42:07::","synset_id":"go.v.10"},{"index":11,"sense_key":"go%2:39:00::","synset_id":"sound.v.02"},{"index":12,"sense_key":"go%2:35:00::","synset_id":"function.v.01"},{"index":13,"sense_key":"go%2:34:00::","synset_id":"run_low.v.01"},{"index":14,"sense_key":"go%2:30:02::","synset_id":"move.v.13"},{"index":15,"sense_key":"go%2:42:14::","synset_id":"survive.v.01"},{"index":16,"sense_key":"go%2:42:03::","synset_id":"go.v.16"},{"index":17,"sense_key":"go%2:30:00::","synset_id":"die.v.01"},{"index":18,"sense_key":"go%2:42:04::","synset_id":"belong.v.03"},{"index":19,"sense_key":"go%2:42:10::","synset_id":"go.v.19"},{"index":20,"sense_key":"go%2:38:09::","synset_id":"start.v.09"},{"index":21,"sense_key":"go%2:33:00::","synset_id":"move.v.15"},{"index":22,"sense_key":"go%2:42:13::","synset_id":"go.v.22"},{"index":23,"sense_key":"go%2:42:12::","synset_id":"go.v.23"},{"index":24,"sense_key":"go%2:42:11::","synset_id":"blend.v.02"},{"index":25,"sense_key":"go%2:42:08::","synset_id":"go.v.25"},{"index":26,"sense_key":"go%2:42:05::","synset_id":"fit.v.02"},{"index":27,"sense_key":"go%2:35:13::","synset_id":"rifle.v.02"},{"index":28,"sense_key":"go%2:34:09::","synset_id":"go.v.28"},{"index":29,"sense_key":"go%2:31:12::","synset_id":"plump.v.04"},{"index":30,"sense_key":"go%2:30:03::","synset_id":"fail.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"go%2:38:00::","call_ids":["senseval3.d000.s026.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"go%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2334,"cached_input_tokens":0,"output_tokens":102,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0056879999999999995,"input_uncached_usd":0.004667999999999999,"input_cached_usd":0.0,"output_usd":0.00102,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.638573250034824}
{"item_id":"senseval3.d000.s026.t001","gold_sense_keys":["keep_to_oneself%2:41:00::"],"candidates":[{"index":1,"sense_key":"keep_to_oneself%2:41:00::","synset_id":"keep_to_oneself.v.01"},{"index":2,"sense_key":"keep_to_oneself%2:40:00::","synset_id":"keep_to_oneself.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"keep_to_oneself%2:41:00::","call_ids":["senseval3.d000.s026.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"keep_to_oneself%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":538,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001196,"input_uncached_usd":0.001076,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1469067090656608}
{"item_id":"senseval3.d000.s026.t002","gold_sense_keys":["sit%2:35:00::","sit%2:38:00::"],"candidates":[{"index":1,"sense_key":"sit%2:35:00::","synset_id":"sit.v.01"},{"index":2,"sense_key":"sit%2:42:00::","synset_id":"sit.v.02"},{"index":3,"sense_key":"sit%2:38:00::","synset_id":"sit_down.v.01"},{"index":4,"sense_key":"sit%2:38:01::","synset_id":"sit.v.04"},{"index":5,"sense_key":"sit%2:39:00::","synset_id":"model.v.03"},{"index":6,"sense_key":"sit%2:38:03::","synset_id":"ride.v.01"},{"index":7,"sense_key":"sit%2:42:05::","synset_id":"sit.v.07"},{"index":8,"sense_key":"sit%2:41:05::","synset_id":"baby-sit.v.02"},{"index":9,"sense_key":"sit%2:35:01::","synset_id":"seat.v.01"},{"index":10,"sense_key":"sit%2:33:00::","synset_id":"sit.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"sit%2:35:00::","call_ids":["senseval3.d000.s026.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"sit%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":993,"cached_input_tokens":0,"output_tokens":121,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003196,"input_uncached_usd":0.001986,"input_cached_usd":0.0,"output_usd":0.0012100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6322572919307277}
{"item_id":"senseval3.d000.s026.t003","gold_sense_keys":["corner%1:06:02::"],"candidates":[{"index":1,"sense_key":"corner%1:15:02::","synset_id":"corner.n.01"},{"index":2,"sense_key":"corner%1:15:00::","synset_id":"corner.n.02"},{"index":3,"sense_key":"corner%1:06:02::","synset_id":"corner.n.03"},{"index":4,"sense_key":"corner%1:06:00::","synset_id":"corner.n.04"},{"index":5,"sense_key":"corner%1:15:01::","synset_id":"corner.n.05"},{"index":6,"sense_key":"corner%1:25:00::","synset_id":"recess.n.02"},{"index":7,"sense_key":"corner%1:26:00::","synset_id":"corner.n.07"},{"index":8,"sense_key":"corner%1:26:01::","synset_id":"corner.n.08"},{"index":9,"sense_key":"corner%1:17:00::","synset_id":"corner.n.09"},{"index":10,"sense_key":"corner%1:15:03::","synset_id":"corner.n.10"},{"index":11,"sense_key":"corner%1:06:01::","synset_id":"corner.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"corner%1:06:02::","call_ids":["senseval3.d000.s026.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"corner%1:06:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1021,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002672,"input_uncached_usd":0.002042,"input_cached_usd":0.0,"output_usd":0.00063,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8138803330948576}
{"item_id":"senseval3.d000.s026.t004","gold_sense_keys":["pass%2:42:00::"],"candidates":[{"index":1,"sense_key":"pass%2:38:00::","synset_id":"pass.v.01"},{"index":2,"sense_key":"pass%2:38:05::","synset_id":"travel_by.v.01"},{"index":3,"sense_key":"pass%2:41:02::","synset_id":"legislate.v.01"},{"index":4,"sense_key":"pass%2:38:03::","synset_id":"elapse.v.01"},{"index":5,"sense_key":"pass%2:40:00::","synset_id":"pass.v.05"},{"index":6,"sense_key":"pass%2:42:08::","synset_id":"run.v.03"},{"index":7,"sense_key":"pass%2:38:01::","synset_id":"pass.v.07"},{"index":8,"sense_key":"pass%2:30:00::","synset_id":"happen.v.01"},{"index":9,"sense_key":"pass%2:41:04::","synset_id":"pass.v.09"},{"index":10,"sense_key":"pass%2:42:00::","synset_id":"spend.v.01"},{"index":11,"sense_key":"pass%2:35:00::","synset_id":"guide.v.05"},{"index":12,"sense_key":"pass%2:32:01::","synset_id":"communicate.v.01"},{"index":13,"sense_key":"pass%2:30:01::","synset_id":"evanesce.v.01"},{"index":14,"sense_key":"pass%2:41:08::","synset_id":"pass.v.14"},{"index":15,"sense_key":"pass%2:42:04::","synset_id":"exceed.v.02"},{"index":16,"sense_key":"pass%2:41:07::","synset_id":"pass.v.16"},{"index":17,"sense_key":"pass%2:41:11::","synset_id":"pass.v.17"},{"index":18,"sense_key":"pass%2:40:02::","synset_id":"pass.v.18"},{"index":19,"sense_key":"pass%2:38:02::","synset_id":"sink.v.03"},{"index":20,"sense_key":"pass%2:40:01::","synset_id":"pass.v.20"},{"index":21,"sense_key":"pass%2:40:13::","synset_id":"fall.v.21"},{"index":22,"sense_key":"pass%2:38:04::","synset_id":"pass.v.22"},{"index":23,"sense_key":"pass%2:32:00::","synset_id":"authorize.v.01"},{"index":24,"sense_key":"pass%2:30:11::","synset_id":"die.v.01"},{"index":25,"sense_key":"pass%2:29:00::","synset_id":"excrete.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"pass%2:42:00::","call_ids":["senseval3.d000.s026.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"pass%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2049,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004218,"input_uncached_usd":0.004098,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.506276458967477}
{"item_id":"senseval3.d000.s026.t005","gold_sense_keys":["time%1:28:05::"],"candidates":[{"index":1,"sense_key":"time%1:11:00::","synset_id":"time.n.01"},{"index":2,"sense_key":"time%1:28:05::","synset_id":"time.n.02"},{"index":3,"sense_key":"time%1:28:00::","synset_id":"time.n.03"},{"index":4,"sense_key":"time%1:28:06::","synset_id":"time.n.04"},{"index":5,"sense_key":"time%1:03:00::","synset_id":"time.n.05"},{"index":6,"sense_key":"time%1:11:01::","synset_id":"time.n.06"},{"index":7,"sense_key":"time%1:28:03::","synset_id":"clock_time.n.01"},{"index":8,"sense_key":"time%1:28:04::","synset_id":"fourth_dimension.n.01"},{"index":9,"sense_key":"time%1:07:01::","synset_id":"meter.n.04"},{"index":10,"sense_key":"time%1:28:01::","synset_id":"prison_term.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"time%1:28:00::","call_ids":["senseval3.d000.s026.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"time%1:28:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1002,"cached_input_tokens":0,"output_tokens":140,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003404,"input_uncached_usd":0.0020039999999999997,"input_cached_usd":0.0,"output_usd":0.0014000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.907381874974817}
{"item_id":"senseval3.d000.s026.t006","gold_sense_keys":["nurse%2:37:00::"],"candidates":[{"index":1,"sense_key":"nurse%2:29:00::","synset_id":"nurse.v.01"},{"index":2,"sense_key":"nurse%2:37:00::","synset_id":"harbor.v.01"},{"index":3,"sense_key":"nurse%2:41:00::","synset_id":"nurse.v.03"},{"index":4,"sense_key":"nurse%2:41:03::","synset_id":"nurse.v.04"},{"index":5,"sense_key":"nurse%2:34:00::","synset_id":"breastfeed.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"nurse%2:37:00::","call_ids":["senseval3.d000.s026.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"nurse%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":785,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00169,"input_uncached_usd":0.00157,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3435297920368612}
{"item_id":"senseval3.d000.s026.t007","gold_sense_keys":["favorite%5:00:00:loved:00"],"candidates":[{"index":1,"sense_key":"favorite%5:00:00:popular:00","synset_id":"favorite.s.01"},{"index":2,"sense_key":"favorite%5:00:00:loved:00","synset_id":"favored.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"favorite%5:00:00:loved:00","call_ids":["senseval3.d000.s026.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"favorite%5:00:00:loved:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":568,"cached_input_tokens":0,"output_tokens":70,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001836,"input_uncached_usd":0.0011359999999999999,"input_cached_usd":0.0,"output_usd":0.0007000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1020409170305356}
{"item_id":"senseval3.d000.s027.t000","gold_sense_keys":["decide%2:31:00::"],"candidates":[{"index":1,"sense_key":"decide%2:31:00::","synset_id":"decide.v.01"},{"index":2,"sense_key":"decide%2:31:01::","synset_id":"decide.v.02"},{"index":3,"sense_key":"decide%2:31:02::","synset_id":"decide.v.03"},{"index":4,"sense_key":"decide%2:31:06::","synset_id":"decide.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"decide%2:31:00::","call_ids":["senseval3.d000.s027.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"decide%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":666,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001452,"input_uncached_usd":0.0013319999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.7864332080353051}
{"item_id":"senseval3.d000.s027.t001","gold_sense_keys":["mind%2:32:00::"],"candidates":[{"index":1,"sense_key":"mind%2:32:00::","synset_id":"mind.v.01"},{"index":2,"sense_key":"mind%2:31:02::","synset_id":"mind.v.02"},{"index":3,"sense_key":"mind%2:41:01::","synset_id":"take_care.v.02"},{"index":4,"sense_key":"mind%2:41:00::","synset_id":"heed.v.01"},{"index":5,"sense_key":"mind%2:31:03::","synset_id":"beware.v.01"},{"index":6,"sense_key":"mind%2:31:00::","synset_id":"mind.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"mind%2:32:00::","call_ids":["senseval3.d000.s027.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"mind%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":730,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00201,"input_uncached_usd":0.00146,"input_cached_usd":0.0,"output_usd":0.00055,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2316250000149012}
{"item_id":"senseval3.d000.s027.t002","gold_sense_keys":["company%1:26:00::"],"candidates":[{"index":1,"sense_key":"company%1:14:01::","synset_id":"company.n.01"},{"index":2,"sense_key":"company%1:14:03::","synset_id":"company.n.02"},{"index":3,"sense_key":"company%1:26:00::","synset_id":"company.n.03"},{"index":4,"sense_key":"company%1:14:02::","synset_id":"company.n.04"},{"index":5,"sense_key":"company%1:18:00::","synset_id":"caller.n.01"},{"index":6,"sense_key":"company%1:14:00::","synset_id":"company.n.06"},{"index":7,"sense_key":"company%1:14:06::","synset_id":"party.n.03"},{"index":8,"sense_key":"company%1:14:04::","synset_id":"ship's_company.n.01"},{"index":9,"sense_key":"company%1:14:05::","synset_id":"company.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"company%1:26:00::","call_ids":["senseval3.d000.s027.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"company%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":919,"cached_input_tokens":0,"output_tokens":51,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0023480000000000003,"input_uncached_usd":0.001838,"input_cached_usd":0.0,"output_usd":0.00051,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3640560830244794}
{"item_id":"senseval3.d000.s027.t004","gold_sense_keys":["free%5:00:00:unpaid:00"],"candidates":[{"index":1,"sense_key":"free%3:00:00::","synset_id":"free.a.01"},{"index":2,"sense_key":"free%3:00:02::","synset_id":"free.a.02"},{"index":3,"sense_key":"free%5:00:00:unpaid:00","synset_id":"complimentary.s.01"},{"index":4,"sense_key":"free%5:00:02:unoccupied:00","synset_id":"free.s.02"},{"index":5,"sense_key":"free%5:00:00:unfixed:00","synset_id":"detached.s.05"},{"index":6,"sense_key":"free%3:00:01::","synset_id":"free.a.06"},{"index":7,"sense_key":"free%5:00:01:unoccupied:00","synset_id":"spare.s.03"},{"index":8,"sense_key":"free%5:00:00:nonexistent:00","synset_id":"barren.s.03"},{"index":9,"sense_key":"free%5:00:00:inexact:00","synset_id":"free.s.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"free%5:00:00:unpaid:00","call_ids":["senseval3.d000.s027.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"free%5:00:00:unpaid:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":987,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002094,"input_uncached_usd":0.001974,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0142215420491993}
{"item_id":"senseval3.d000.s027.t005","gold_sense_keys":["drink%1:13:04::"],"candidates":[{"index":1,"sense_key":"drink%1:13:04::","synset_id":"drink.n.01"},{"index":2,"sense_key":"drink%1:04:00::","synset_id":"drink.n.02"},{"index":3,"sense_key":"drink%1:13:00::","synset_id":"beverage.n.01"},{"index":4,"sense_key":"drink%1:17:00::","synset_id":"drink.n.04"},{"index":5,"sense_key":"drink%1:04:01::","synset_id":"swallow.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"drink%1:13:04::","call_ids":["senseval3.d000.s027.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"drink%1:13:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":740,"cached_input_tokens":0,"output_tokens":48,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00196,"input_uncached_usd":0.00148,"input_cached_usd":0.0,"output_usd":0.00048000000000000007,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.691186249954626}
{"item_id":"senseval3.d000.s027.t006","gold_sense_keys":["make%2:40:01::"],"candidates":[{"index":1,"sense_key":"make%2:41:00::","synset_id":"make.v.01"},{"index":2,"sense_key":"make%2:30:00::","synset_id":"make.v.02"},{"index":3,"sense_key":"make%2:36:00::","synset_id":"make.v.03"},{"index":4,"sense_key":"make%2:32:00::","synset_id":"induce.v.02"},{"index":5,"sense_key":"make%2:36:08::","synset_id":"cause.v.01"},{"index":6,"sense_key":"make%2:36:01::","synset_id":"produce.v.02"},{"index":7,"sense_key":"make%2:31:00::","synset_id":"draw.v.04"},{"index":8,"sense_key":"make%2:36:15::","synset_id":"make.v.08"},{"index":9,"sense_key":"make%2:36:09::","synset_id":"create.v.05"},{"index":10,"sense_key":"make%2:40:01::","synset_id":"gain.v.08"},{"index":11,"sense_key":"make%2:36:05::","synset_id":"do.v.08"},{"index":12,"sense_key":"make%2:42:05::","synset_id":"form.v.02"},{"index":13,"sense_key":"make%2:38:02::","synset_id":"reach.v.07"},{"index":14,"sense_key":"make%2:42:00::","synset_id":"make.v.14"},{"index":15,"sense_key":"make%2:36:11::","synset_id":"make.v.15"},{"index":16,"sense_key":"make%2:36:12::","synset_id":"make.v.16"},{"index":17,"sense_key":"make%2:36:13::","synset_id":"construct.v.01"},{"index":18,"sense_key":"make%2:30:15::","synset_id":"make.v.18"},{"index":19,"sense_key":"make%2:29:08::","synset_id":"make.v.19"},{"index":20,"sense_key":"make%2:41:03::","synset_id":"name.v.03"},{"index":21,"sense_key":"make%2:40:02::","synset_id":"have.v.17"},{"index":22,"sense_key":"make%2:38:00::","synset_id":"reach.v.01"},{"index":23,"sense_key":"make%2:31:13::","synset_id":"lay_down.v.01"},{"index":24,"sense_key":"make%2:41:13::","synset_id":"make.v.24"},{"index":25,"sense_key":"make%2:41:01::","synset_id":"make.v.25"},{"index":26,"sense_key":"make%2:36:04::","synset_id":"hold.v.03"},{"index":27,"sense_key":"make%2:30:02::","synset_id":"make.v.27"},{"index":28,"sense_key":"make%2:38:05::","synset_id":"take.v.27"},{"index":29,"sense_key":"make%2:29:00::","synset_id":"stool.v.04"},{"index":30,"sense_key":"make%2:42:09::","synset_id":"make.v.30"},{"index":31,"sense_key":"make%2:42:13::","synset_id":"make.v.31"},{"index":32,"sense_key":"make%2:42:12::","synset_id":"make.v.32"},{"index":33,"sense_key":"make%2:42:11::","synset_id":"make.v.33"},{"index":34,"sense_key":"make%2:42:15::","synset_id":"make.v.34"},{"index":35,"sense_key":"make%2:39:12::","synset_id":"make.v.35"},{"index":36,"sense_key":"make%2:38:10::","synset_id":"make.v.36"},{"index":37,"sense_key":"make%2:38:15::","synset_id":"make.v.37"},{"index":38,"sense_key":"make%2:36:10::","synset_id":"make.v.38"},{"index":39,"sense_key":"make%2:36:07::","synset_id":"cook.v.02"},{"index":40,"sense_key":"make%2:35:13::","synset_id":"seduce.v.01"},{"index":41,"sense_key":"make%2:32:14::","synset_id":"make.v.41"},{"index":42,"sense_key":"make%2:32:09::","synset_id":"make.v.42"},{"index":43,"sense_key":"make%2:31:10::","synset_id":"make.v.43"},{"index":44,"sense_key":"make%2:31:12::","synset_id":"make.v.44"},{"index":45,"sense_key":"make%2:30:10::","synset_id":"make.v.45"},{"index":46,"sense_key":"make%2:30:14::","synset_id":"make.v.46"},{"index":47,"sense_key":"make%2:30:12::","synset_id":"make.v.47"},{"index":48,"sense_key":"make%2:29:06::","synset_id":"make.v.48"},{"index":49,"sense_key":"make%2:29:01::","synset_id":"make.v.49"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"make%2:40:01::","call_ids":["senseval3.d000.s027.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"make%2:40:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":3089,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.006298,"input_uncached_usd":0.0061779999999999995,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9547042499762028}
{"item_id":"senseval3.d000.s027.t009","gold_sense_keys":["job%1:04:00::"],"candidates":[{"index":1,"sense_key":"job%1:04:00::","synset_id":"occupation.n.01"},{"index":2,"sense_key":"job%1:04:01::","synset_id":"job.n.02"},{"index":3,"sense_key":"job%1:06:00::","synset_id":"job.n.03"},{"index":4,"sense_key":"job%1:06:01::","synset_id":"job.n.04"},{"index":5,"sense_key":"job%1:04:04::","synset_id":"job.n.05"},{"index":6,"sense_key":"job%1:04:05::","synset_id":"job.n.06"},{"index":7,"sense_key":"job%1:04:03::","synset_id":"job.n.07"},{"index":8,"sense_key":"job%1:26:00::","synset_id":"problem.n.01"},{"index":9,"sense_key":"job%1:18:00::","synset_id":"job.n.09"},{"index":10,"sense_key":"job%1:18:01::","synset_id":"job.n.10"},{"index":11,"sense_key":"job%1:10:01::","synset_id":"job.n.11"},{"index":12,"sense_key":"job%1:10:00::","synset_id":"job.n.12"},{"index":13,"sense_key":"job%1:04:02::","synset_id":"caper.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"job%1:04:00::","call_ids":["senseval3.d000.s027.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"job%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1163,"cached_input_tokens":0,"output_tokens":50,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002826,"input_uncached_usd":0.002326,"input_cached_usd":0.0,"output_usd":0.0005,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.414432416902855}
{"item_id":"senseval3.d000.s029.t000","gold_sense_keys":["wonder%2:32:01::"],"candidates":[{"index":1,"sense_key":"wonder%2:31:00::","synset_id":"wonder.v.01"},{"index":2,"sense_key":"wonder%2:32:01::","synset_id":"wonder.v.02"},{"index":3,"sense_key":"wonder%2:32:12::","synset_id":"wonder.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"wonder%2:32:01::","call_ids":["senseval3.d000.s029.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"wonder%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":609,"cached_input_tokens":0,"output_tokens":121,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002428,"input_uncached_usd":0.001218,"input_cached_usd":0.0,"output_usd":0.0012100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9814928330015391}
{"item_id":"senseval3.d000.s029.t002","gold_sense_keys":["screwball%1:18:00::"],"candidates":[{"index":1,"sense_key":"screwball%1:18:00::","synset_id":"crackpot.n.01"},{"index":2,"sense_key":"screwball%1:04:00::","synset_id":"screwball.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"screwball%1:18:00::","call_ids":["senseval3.d000.s029.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"screwball%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":523,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0011660000000000002,"input_uncached_usd":0.001046,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0518754170043394}
{"item_id":"senseval3.d000.s030.t000","gold_sense_keys":["take%2:31:07::"],"candidates":[{"index":1,"sense_key":"take%2:41:04::","synset_id":"take.v.01"},{"index":2,"sense_key":"take%2:40:06::","synset_id":"take.v.02"},{"index":3,"sense_key":"take%2:38:09::","synset_id":"lead.v.01"},{"index":4,"sense_key":"take%2:35:00::","synset_id":"take.v.04"},{"index":5,"sense_key":"take%2:30:01::","synset_id":"assume.v.03"},{"index":6,"sense_key":"take%2:31:07::","synset_id":"take.v.06"},{"index":7,"sense_key":"take%2:38:10::","synset_id":"bring.v.01"},{"index":8,"sense_key":"take%2:40:00::","synset_id":"take.v.08"},{"index":9,"sense_key":"take%2:38:02::","synset_id":"take.v.09"},{"index":10,"sense_key":"take%2:31:01::","synset_id":"choose.v.01"},{"index":11,"sense_key":"take%2:40:05::","synset_id":"accept.v.02"},{"index":12,"sense_key":"take%2:41:00::","synset_id":"fill.v.04"},{"index":13,"sense_key":"take%2:31:03::","synset_id":"consider.v.03"},{"index":14,"sense_key":"take%2:42:00::","synset_id":"necessitate.v.01"},{"index":15,"sense_key":"take%2:39:00::","synset_id":"take.v.15"},{"index":16,"sense_key":"take%2:32:02::","synset_id":"film.v.01"},{"index":17,"sense_key":"take%2:30:00::","synset_id":"remove.v.01"},{"index":18,"sense_key":"take%2:34:00::","synset_id":"consume.v.02"},{"index":19,"sense_key":"take%2:31:09::","synset_id":"take.v.19"},{"index":20,"sense_key":"take%2:40:15::","synset_id":"take.v.20"},{"index":21,"sense_key":"take%2:40:01::","synset_id":"take.v.21"},{"index":22,"sense_key":"take%2:38:00::","synset_id":"assume.v.05"},{"index":23,"sense_key":"take%2:40:07::","synset_id":"accept.v.05"},{"index":24,"sense_key":"take%2:30:08::","synset_id":"take.v.24"},{"index":25,"sense_key":"take%2:31:02::","synset_id":"learn.v.04"},{"index":26,"sense_key":"take%2:32:00::","synset_id":"claim.v.05"},{"index":27,"sense_key":"take%2:38:05::","synset_id":"take.v.27"},{"index":28,"sense_key":"take%2:33:09::","synset_id":"aim.v.01"},{"index":29,"sense_key":"take%2:30:09::","synset_id":"take.v.29"},{"index":30,"sense_key":"take%2:42:10::","synset_id":"carry.v.02"},{"index":31,"sense_key":"take%2:40:03::","synset_id":"lease.v.04"},{"index":32,"sense_key":"take%2:40:04::","synset_id":"subscribe.v.05"},{"index":33,"sense_key":"take%2:40:02::","synset_id":"take.v.33"},{"index":34,"sense_key":"take%2:40:08::","synset_id":"take.v.34"},{"index":35,"sense_key":"take%2:35:14::","synset_id":"take.v.35"},{"index":36,"sense_key":"take%2:32:09::","synset_id":"claim.v.04"},{"index":37,"sense_key":"take%2:42:03::","synset_id":"accept.v.08"},{"index":38,"sense_key":"take%2:42:15::","synset_id":"contain.v.05"},{"index":39,"sense_key":"take%2:41:13::","synset_id":"take.v.39"},{"index":40,"sense_key":"take%2:38:11::","synset_id":"drive.v.16"},{"index":41,"sense_key":"take%2:33:08::","synset_id":"take.v.41"},{"index":42,"sense_key":"take%2:29:08::","synset_id":"contract.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"take%2:31:07::","call_ids":["senseval3.d000.s030.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"take%2:31:07::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":3063,"cached_input_tokens":0,"output_tokens":90,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.007025999999999999,"input_uncached_usd":0.0061259999999999995,"input_cached_usd":0.0,"output_usd":0.0009000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8606370409252122}
{"item_id":"senseval3.d000.s030.t001","gold_sense_keys":["offer%1:10:00::"],"candidates":[{"index":1,"sense_key":"offer%1:10:01::","synset_id":"offer.n.01"},{"index":2,"sense_key":"offer%1:10:00::","synset_id":"offer.n.02"},{"index":3,"sense_key":"offer%1:04:01::","synset_id":"crack.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"offer%1:10:01::","call_ids":["senseval3.d000.s030.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"offer%1:10:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":648,"cached_input_tokens":0,"output_tokens":229,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003586,"input_uncached_usd":0.0012959999999999998,"input_cached_usd":0.0,"output_usd":0.0022900000000000004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.6771963339997455}
{"item_id":"senseval3.d000.s030.t003","gold_sense_keys":["feel%2:29:00::","feel%2:31:01::","feel%2:37:00::"],"candidates":[{"index":1,"sense_key":"feel%2:37:00::","synset_id":"feel.v.01"},{"index":2,"sense_key":"feel%2:31:00::","synset_id":"find.v.05"},{"index":3,"sense_key":"feel%2:39:00::","synset_id":"feel.v.03"},{"index":4,"sense_key":"feel%2:29:00::","synset_id":"feel.v.04"},{"index":5,"sense_key":"feel%2:31:01::","synset_id":"feel.v.05"},{"index":6,"sense_key":"feel%2:39:01::","synset_id":"feel.v.06"},{"index":7,"sense_key":"feel%2:42:00::","synset_id":"feel.v.07"},{"index":8,"sense_key":"feel%2:39:10::","synset_id":"feel.v.08"},{"index":9,"sense_key":"feel%2:35:00::","synset_id":"feel.v.09"},{"index":10,"sense_key":"feel%2:35:01::","synset_id":"palpate.v.01"},{"index":11,"sense_key":"feel%2:40:00::","synset_id":"feel.v.11"},{"index":12,"sense_key":"feel%2:39:09::","synset_id":"feel.v.12"},{"index":13,"sense_key":"feel%2:35:11::","synset_id":"feel.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"feel%2:37:00::","call_ids":["senseval3.d000.s030.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"feel%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1253,"cached_input_tokens":0,"output_tokens":132,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0038260000000000004,"input_uncached_usd":0.002506,"input_cached_usd":0.0,"output_usd":0.0013200000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.634353292058222}
{"item_id":"senseval3.d000.s030.t004","gold_sense_keys":["uneasy%3:00:00::"],"candidates":[{"index":1,"sense_key":"uneasy%3:00:00::","synset_id":"uneasy.a.01"},{"index":2,"sense_key":"uneasy%3:00:05::","synset_id":"restless.a.03"},{"index":3,"sense_key":"uneasy%5:00:00:troubled:00","synset_id":"anxious.s.02"},{"index":4,"sense_key":"uneasy%5:00:00:uncomfortable:01","synset_id":"awkward.s.05"},{"index":5,"sense_key":"uneasy%5:00:00:uncomfortable:00","synset_id":"uneasy.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"uneasy%3:00:00::","call_ids":["senseval3.d000.s030.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"uneasy%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":841,"cached_input_tokens":0,"output_tokens":289,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004572,"input_uncached_usd":0.001682,"input_cached_usd":0.0,"output_usd":0.00289,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.3888222500681877}
{"item_id":"senseval3.d000.s031.t000","gold_sense_keys":["get%2:30:02::"],"candidates":[{"index":1,"sense_key":"get%2:40:00::","synset_id":"get.v.01"},{"index":2,"sense_key":"get%2:30:00::","synset_id":"become.v.01"},{"index":3,"sense_key":"get%2:30:02::","synset_id":"get.v.03"},{"index":4,"sense_key":"get%2:30:03::","synset_id":"receive.v.02"},{"index":5,"sense_key":"get%2:38:00::","synset_id":"arrive.v.01"},{"index":6,"sense_key":"get%2:35:03::","synset_id":"bring.v.04"},{"index":7,"sense_key":"get%2:39:00::","synset_id":"experience.v.03"},{"index":8,"sense_key":"get%2:33:00::","synset_id":"pay_back.v.02"},{"index":9,"sense_key":"get%2:40:01::","synset_id":"have.v.17"},{"index":10,"sense_key":"get%2:32:00::","synset_id":"induce.v.02"},{"index":11,"sense_key":"get%2:35:09::","synset_id":"get.v.11"},{"index":12,"sense_key":"get%2:29:01::","synset_id":"grow.v.08"},{"index":13,"sense_key":"get%2:29:00::","synset_id":"contract.v.04"},{"index":14,"sense_key":"get%2:32:03::","synset_id":"get.v.14"},{"index":15,"sense_key":"get%2:30:01::","synset_id":"make.v.02"},{"index":16,"sense_key":"get%2:32:08::","synset_id":"drive.v.11"},{"index":17,"sense_key":"get%2:31:09::","synset_id":"catch.v.18"},{"index":18,"sense_key":"get%2:35:00::","synset_id":"catch.v.07"},{"index":19,"sense_key":"get%2:35:10::","synset_id":"get.v.19"},{"index":20,"sense_key":"get%2:31:03::","synset_id":"get.v.20"},{"index":21,"sense_key":"get%2:40:14::","synset_id":"get.v.21"},{"index":22,"sense_key":"get%2:40:13::","synset_id":"get.v.22"},{"index":23,"sense_key":"get%2:39:13::","synset_id":"catch.v.21"},{"index":24,"sense_key":"get%2:39:09::","synset_id":"catch.v.22"},{"index":25,"sense_key":"get%2:39:14::","synset_id":"get.v.25"},{"index":26,"sense_key":"get%2:38:15::","synset_id":"scram.v.01"},{"index":27,"sense_key":"get%2:38:12::","synset_id":"get.v.27"},{"index":28,"sense_key":"get%2:37:12::","synset_id":"get.v.28"},{"index":29,"sense_key":"get%2:37:11::","synset_id":"get.v.29"},{"index":30,"sense_key":"get%2:36:14::","synset_id":"catch.v.24"},{"index":31,"sense_key":"get%2:36:00::","synset_id":"draw.v.15"},{"index":32,"sense_key":"get%2:35:12::","synset_id":"get.v.32"},{"index":33,"sense_key":"get%2:31:10::","synset_id":"perplex.v.01"},{"index":34,"sense_key":"get%2:30:12::","synset_id":"get_down.v.07"},{"index":35,"sense_key":"get%2:29:11::","synset_id":"suffer.v.02"},{"index":36,"sense_key":"get%2:29:08::","synset_id":"beget.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"get%2:30:02::","call_ids":["senseval3.d000.s031.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"get%2:30:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2772,"cached_input_tokens":0,"output_tokens":144,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.006984,"input_uncached_usd":0.0055439999999999994,"input_cached_usd":0.0,"output_usd":0.00144,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.333116916939616}
{"item_id":"senseval3.d000.s031.t003","gold_sense_keys":["give%2:40:07::"],"candidates":[{"index":1,"sense_key":"give%2:40:03::","synset_id":"give.v.01"},{"index":2,"sense_key":"give%2:40:05::","synset_id":"yield.v.01"},{"index":3,"sense_key":"give%2:40:00::","synset_id":"give.v.03"},{"index":4,"sense_key":"give%2:40:07::","synset_id":"give.v.04"},{"index":5,"sense_key":"give%2:32:01::","synset_id":"give.v.05"},{"index":6,"sense_key":"give%2:36:02::","synset_id":"hold.v.03"},{"index":7,"sense_key":"give%2:32:00::","synset_id":"give.v.07"},{"index":8,"sense_key":"give%2:40:01::","synset_id":"give.v.08"},{"index":9,"sense_key":"give%2:36:03::","synset_id":"give.v.09"},{"index":10,"sense_key":"give%2:31:02::","synset_id":"give.v.10"},{"index":11,"sense_key":"give%2:36:00::","synset_id":"render.v.04"},{"index":12,"sense_key":"give%2:40:09::","synset_id":"impart.v.01"},{"index":13,"sense_key":"give%2:36:01::","synset_id":"establish.v.05"},{"index":14,"sense_key":"give%2:40:02::","synset_id":"give.v.14"},{"index":15,"sense_key":"give%2:41:00::","synset_id":"give.v.15"},{"index":16,"sense_key":"give%2:40:10::","synset_id":"sacrifice.v.01"},{"index":17,"sense_key":"give%2:40:06::","synset_id":"pass.v.05"},{"index":18,"sense_key":"give%2:32:03::","synset_id":"give.v.18"},{"index":19,"sense_key":"give%2:40:13::","synset_id":"give.v.19"},{"index":20,"sense_key":"give%2:40:14::","synset_id":"give.v.20"},{"index":21,"sense_key":"give%2:32:02::","synset_id":"give.v.21"},{"index":22,"sense_key":"give%2:40:11::","synset_id":"grant.v.05"},{"index":23,"sense_key":"give%2:38:00::","synset_id":"move_over.v.01"},{"index":24,"sense_key":"give%2:34:00::","synset_id":"feed.v.02"},{"index":25,"sense_key":"give%2:40:08::","synset_id":"contribute.v.02"},{"index":26,"sense_key":"give%2:38:13::","synset_id":"collapse.v.01"},{"index":27,"sense_key":"give%2:31:00::","synset_id":"give.v.27"},{"index":28,"sense_key":"give%2:41:10::","synset_id":"give.v.28"},{"index":29,"sense_key":"give%2:40:12::","synset_id":"give.v.29"},{"index":30,"sense_key":"give%2:40:15::","synset_id":"afford.v.04"},{"index":31,"sense_key":"give%2:36:13::","synset_id":"give.v.31"},{"index":32,"sense_key":"give%2:36:12::","synset_id":"give.v.32"},{"index":33,"sense_key":"give%2:35:00::","synset_id":"give.v.33"},{"index":34,"sense_key":"give%2:34:10::","synset_id":"give.v.34"},{"index":35,"sense_key":"give%2:32:11::","synset_id":"give.v.35"},{"index":36,"sense_key":"give%2:32:08::","synset_id":"give.v.36"},{"index":37,"sense_key":"give%2:32:15::","synset_id":"give.v.37"},{"index":38,"sense_key":"give%2:32:14::","synset_id":"give.v.38"},{"index":39,"sense_key":"give%2:32:09::","synset_id":"give.v.39"},{"index":40,"sense_key":"give%2:32:07::","synset_id":"give.v.40"},{"index":41,"sense_key":"give%2:32:10::","synset_id":"give.v.41"},{"index":42,"sense_key":"give%2:30:13::","synset_id":"give.v.42"},{"index":43,"sense_key":"give%2:29:11::","synset_id":"give.v.43"},{"index":44,"sense_key":"give%2:29:10::","synset_id":"give.v.44"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"give%2:40:07::","call_ids":["senseval3.d000.s031.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"give%2:40:07::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2830,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00627,"input_uncached_usd":0.00566,"input_cached_usd":0.0,"output_usd":0.0006100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5750025829765946}
{"item_id":"senseval3.d000.s031.t005","gold_sense_keys":["wait%2:42:00::"],"candidates":[{"index":1,"sense_key":"wait%2:42:00::","synset_id":"wait.v.01"},{"index":2,"sense_key":"wait%2:42:01::","synset_id":"wait.v.02"},{"index":3,"sense_key":"wait%2:31:00::","synset_id":"expect.v.03"},{"index":4,"sense_key":"wait%2:41:00::","synset_id":"wait.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"wait%2:42:00::","call_ids":["senseval3.d000.s031.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"wait%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":693,"cached_input_tokens":0,"output_tokens":136,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002746,"input_uncached_usd":0.0013859999999999999,"input_cached_usd":0.0,"output_usd":0.00136,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.4471337909344584}
{"item_id":"senseval3.d000.s031.t006","gold_sense_keys":["answer%1:10:01::"],"candidates":[{"index":1,"sense_key":"answer%1:10:01::","synset_id":"answer.n.01"},{"index":2,"sense_key":"answer%1:10:02::","synset_id":"solution.n.02"},{"index":3,"sense_key":"answer%1:10:00::","synset_id":"answer.n.03"},{"index":4,"sense_key":"answer%1:10:03::","synset_id":"answer.n.04"},{"index":5,"sense_key":"answer%1:04:00::","synset_id":"answer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"answer%1:10:01::","call_ids":["senseval3.d000.s031.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"answer%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":835,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0023499999999999997,"input_uncached_usd":0.0016699999999999998,"input_cached_usd":0.0,"output_usd":0.00068,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8863885420141742}
{"item_id":"senseval3.d000.s031.t007","gold_sense_keys":["seem%2:39:00::"],"candidates":[{"index":1,"sense_key":"seem%2:39:00::","synset_id":"look.v.02"},{"index":2,"sense_key":"seem%2:39:01::","synset_id":"appear.v.04"},{"index":3,"sense_key":"seem%2:42:00::","synset_id":"seem.v.03"},{"index":4,"sense_key":"seem%2:39:02::","synset_id":"seem.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"seem%2:39:00::","call_ids":["senseval3.d000.s031.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"seem%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":690,"cached_input_tokens":0,"output_tokens":177,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00315,"input_uncached_usd":0.00138,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.147794708958827}
{"item_id":"senseval3.d000.s031.t008","gold_sense_keys":["wink%2:29:00::"],"candidates":[{"index":1,"sense_key":"wink%2:29:02::","synset_id":"wink.v.01"},{"index":2,"sense_key":"wink%2:39:00::","synset_id":"flash.v.01"},{"index":3,"sense_key":"wink%2:29:00::","synset_id":"blink.v.01"},{"index":4,"sense_key":"wink%2:29:01::","synset_id":"wink.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"wink%2:29:00::","call_ids":["senseval3.d000.s031.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"wink%2:29:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":645,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00201,"input_uncached_usd":0.00129,"input_cached_usd":0.0,"output_usd":0.00072,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":5.335801166947931}
{"item_id":"senseval3.d000.s031.t009","gold_sense_keys":["continue%2:42:01::"],"candidates":[{"index":1,"sense_key":"continue%2:42:01::","synset_id":"continue.v.01"},{"index":2,"sense_key":"continue%2:32:00::","synset_id":"continue.v.02"},{"index":3,"sense_key":"continue%2:42:00::","synset_id":"continue.v.03"},{"index":4,"sense_key":"continue%2:38:00::","synset_id":"proceed.v.02"},{"index":5,"sense_key":"continue%2:41:00::","synset_id":"retain.v.02"},{"index":6,"sense_key":"continue%2:42:12::","synset_id":"continue.v.06"},{"index":7,"sense_key":"continue%2:42:11::","synset_id":"continue.v.07"},{"index":8,"sense_key":"continue%2:42:10::","synset_id":"stay.v.04"},{"index":9,"sense_key":"continue%2:42:06::","synset_id":"cover.v.03"},{"index":10,"sense_key":"continue%2:30:10::","synset_id":"continue.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"continue%2:42:01::","call_ids":["senseval3.d000.s031.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"continue%2:42:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1131,"cached_input_tokens":0,"output_tokens":88,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003142,"input_uncached_usd":0.0022619999999999997,"input_cached_usd":0.0,"output_usd":0.00088,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2556432919809595}
{"item_id":"senseval3.d000.s031.t010","gold_sense_keys":["stare%2:39:00::"],"candidates":[{"index":1,"sense_key":"stare%2:39:00::","synset_id":"gaze.v.01"},{"index":2,"sense_key":"stare%2:29:00::","synset_id":"stare.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"stare%2:39:00::","call_ids":["senseval3.d000.s031.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"stare%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":539,"cached_input_tokens":0,"output_tokens":180,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0028780000000000003,"input_uncached_usd":0.001078,"input_cached_usd":0.0,"output_usd":0.0018000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.339017916005105}
{"item_id":"senseval3.d000.s032.t001","gold_sense_keys":["weak%5:00:00:perceptible:00"],"candidates":[{"index":1,"sense_key":"weak%3:00:00::","synset_id":"weak.a.01"},{"index":2,"sense_key":"weak%5:00:00:diluted:00","synset_id":"watery.s.04"},{"index":3,"sense_key":"weak%5:00:00:unstressed:00","synset_id":"unaccented.s.01"},{"index":4,"sense_key":"weak%5:00:00:human:00","synset_id":"fallible.s.01"},{"index":5,"sense_key":"weak%5:00:00:down:00","synset_id":"weak.s.04"},{"index":6,"sense_key":"weak%5:00:00:unskilled:00","synset_id":"weak.s.05"},{"index":7,"sense_key":"weak%5:00:00:frail:00","synset_id":"decrepit.s.02"},{"index":8,"sense_key":"weak%5:00:00:regular:00","synset_id":"weak.s.07"},{"index":9,"sense_key":"weak%5:00:00:powerless:00","synset_id":"weak.s.08"},{"index":10,"sense_key":"weak%5:00:00:perceptible:00","synset_id":"faint.s.01"},{"index":11,"sense_key":"weak%5:00:00:fallible:00","synset_id":"weak.s.10"},{"index":12,"sense_key":"weak%5:00:00:stupid:00","synset_id":"weak.s.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"weak%5:00:00:perceptible:00","call_ids":["senseval3.d000.s032.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"weak%5:00:00:perceptible:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1172,"cached_input_tokens":0,"output_tokens":137,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003714,"input_uncached_usd":0.0023439999999999997,"input_cached_usd":0.0,"output_usd":0.0013700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9899091669358313}
{"item_id":"senseval3.d000.s032.t002","gold_sense_keys":["laugh%1:10:00::"],"candidates":[{"index":1,"sense_key":"laugh%1:10:00::","synset_id":"laugh.n.01"},{"index":2,"sense_key":"laugh%1:10:02::","synset_id":"laugh.n.02"},{"index":3,"sense_key":"laugh%1:10:01::","synset_id":"joke.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"laugh%1:10:00::","call_ids":["senseval3.d000.s032.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"laugh%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":589,"cached_input_tokens":0,"output_tokens":75,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001928,"input_uncached_usd":0.001178,"input_cached_usd":0.0,"output_usd":0.00075,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3756374580552801}
{"item_id":"senseval3.d000.s034.t000","gold_sense_keys":["thanks%1:10:00::"],"candidates":[{"index":1,"sense_key":"thanks%1:10:00::","synset_id":"thanks.n.01"},{"index":2,"sense_key":"thanks%1:04:00::","synset_id":"thanks.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"thanks%1:10:00::","call_ids":["senseval3.d000.s034.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"thanks%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":483,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001086,"input_uncached_usd":0.000966,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0669262500014156}
{"item_id":"senseval3.d000.s035.t000","gold_sense_keys":["faint%5:00:00:perceptible:00"],"candidates":[{"index":1,"sense_key":"faint%5:00:00:perceptible:00","synset_id":"faint.s.01"},{"index":2,"sense_key":"faint%5:00:00:indistinct:00","synset_id":"dim.s.02"},{"index":3,"sense_key":"faint%5:00:00:weak:00","synset_id":"faint.s.03"},{"index":4,"sense_key":"faint%5:00:00:ill:01","synset_id":"faint.s.04"},{"index":5,"sense_key":"faint%5:00:02:indistinct:00","synset_id":"faint.s.05"},{"index":6,"sense_key":"faint%5:00:00:cowardly:00","synset_id":"faint.s.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"faint%5:00:00:perceptible:00","call_ids":["senseval3.d000.s035.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"faint%5:00:00:perceptible:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":849,"cached_input_tokens":0,"output_tokens":103,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002728,"input_uncached_usd":0.0016979999999999999,"input_cached_usd":0.0,"output_usd":0.00103,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4773268339922652}
{"item_id":"senseval3.d000.s035.t001","gold_sense_keys":["crease%1:25:01::"],"candidates":[{"index":1,"sense_key":"crease%1:25:02::","synset_id":"fold.n.01"},{"index":2,"sense_key":"crease%1:25:01::","synset_id":"wrinkle.n.01"},{"index":3,"sense_key":"crease%1:06:00::","synset_id":"kris.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"crease%1:25:01::","call_ids":["senseval3.d000.s035.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"crease%1:25:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":608,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001986,"input_uncached_usd":0.0012159999999999999,"input_cached_usd":0.0,"output_usd":0.0007700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1893989999080077}
{"item_id":"senseval3.d000.s035.t002","gold_sense_keys":["appear%2:30:00::"],"candidates":[{"index":1,"sense_key":"appear%2:39:00::","synset_id":"look.v.02"},{"index":2,"sense_key":"appear%2:30:00::","synset_id":"appear.v.02"},{"index":3,"sense_key":"appear%2:30:01::","synset_id":"appear.v.03"},{"index":4,"sense_key":"appear%2:39:01::","synset_id":"appear.v.04"},{"index":5,"sense_key":"appear%2:30:02::","synset_id":"appear.v.05"},{"index":6,"sense_key":"appear%2:36:00::","synset_id":"appear.v.06"},{"index":7,"sense_key":"appear%2:41:03::","synset_id":"appear.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"appear%2:30:00::","call_ids":["senseval3.d000.s035.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"appear%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":884,"cached_input_tokens":0,"output_tokens":45,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002218,"input_uncached_usd":0.0017679999999999998,"input_cached_usd":0.0,"output_usd":0.00045000000000000004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1381089999340475}
{"item_id":"senseval3.d000.s035.t003","gold_sense_keys":["man%1:18:00::"],"candidates":[{"index":1,"sense_key":"man%1:18:00::","synset_id":"man.n.01"},{"index":2,"sense_key":"man%1:18:03::","synset_id":"serviceman.n.01"},{"index":3,"sense_key":"man%1:18:04::","synset_id":"man.n.03"},{"index":4,"sense_key":"man%1:05:01::","synset_id":"homo.n.02"},{"index":5,"sense_key":"man%1:18:08::","synset_id":"man.n.05"},{"index":6,"sense_key":"man%1:18:06::","synset_id":"man.n.06"},{"index":7,"sense_key":"man%1:18:05::","synset_id":"valet.n.01"},{"index":8,"sense_key":"man%1:18:07::","synset_id":"man.n.08"},{"index":9,"sense_key":"man%1:15:00::","synset_id":"man.n.09"},{"index":10,"sense_key":"man%1:06:00::","synset_id":"man.n.10"},{"index":11,"sense_key":"man%1:05:00::","synset_id":"world.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"man%1:18:00::","call_ids":["senseval3.d000.s035.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"man%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1135,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0023899999999999998,"input_uncached_usd":0.00227,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1415292919846252}
{"item_id":"senseval3.d000.s036.t000","gold_sense_keys":["think%2:31:03::"],"candidates":[{"index":1,"sense_key":"think%2:31:01::","synset_id":"think.v.01"},{"index":2,"sense_key":"think%2:31:03::","synset_id":"think.v.02"},{"index":3,"sense_key":"think%2:31:00::","synset_id":"think.v.03"},{"index":4,"sense_key":"think%2:31:02::","synset_id":"remember.v.01"},{"index":5,"sense_key":"think%2:36:00::","synset_id":"think.v.05"},{"index":6,"sense_key":"think%2:31:09::","synset_id":"think.v.06"},{"index":7,"sense_key":"think%2:31:06::","synset_id":"intend.v.01"},{"index":8,"sense_key":"think%2:31:12::","synset_id":"think.v.08"},{"index":9,"sense_key":"think%2:31:13::","synset_id":"think.v.09"},{"index":10,"sense_key":"think%2:31:15::","synset_id":"think.v.10"},{"index":11,"sense_key":"think%2:31:14::","synset_id":"think.v.11"},{"index":12,"sense_key":"think%2:31:08::","synset_id":"think.v.12"},{"index":13,"sense_key":"think%2:30:00::","synset_id":"think.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"think%2:31:03::","call_ids":["senseval3.d000.s036.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"think%2:31:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1164,"cached_input_tokens":0,"output_tokens":160,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003928,"input_uncached_usd":0.0023279999999999998,"input_cached_usd":0.0,"output_usd":0.0016,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.4013394579524174}
{"item_id":"senseval3.d000.s036.t001","gold_sense_keys":["take%2:31:07::"],"candidates":[{"index":1,"sense_key":"take%2:41:04::","synset_id":"take.v.01"},{"index":2,"sense_key":"take%2:40:06::","synset_id":"take.v.02"},{"index":3,"sense_key":"take%2:38:09::","synset_id":"lead.v.01"},{"index":4,"sense_key":"take%2:35:00::","synset_id":"take.v.04"},{"index":5,"sense_key":"take%2:30:01::","synset_id":"assume.v.03"},{"index":6,"sense_key":"take%2:31:07::","synset_id":"take.v.06"},{"index":7,"sense_key":"take%2:38:10::","synset_id":"bring.v.01"},{"index":8,"sense_key":"take%2:40:00::","synset_id":"take.v.08"},{"index":9,"sense_key":"take%2:38:02::","synset_id":"take.v.09"},{"index":10,"sense_key":"take%2:31:01::","synset_id":"choose.v.01"},{"index":11,"sense_key":"take%2:40:05::","synset_id":"accept.v.02"},{"index":12,"sense_key":"take%2:41:00::","synset_id":"fill.v.04"},{"index":13,"sense_key":"take%2:31:03::","synset_id":"consider.v.03"},{"index":14,"sense_key":"take%2:42:00::","synset_id":"necessitate.v.01"},{"index":15,"sense_key":"take%2:39:00::","synset_id":"take.v.15"},{"index":16,"sense_key":"take%2:32:02::","synset_id":"film.v.01"},{"index":17,"sense_key":"take%2:30:00::","synset_id":"remove.v.01"},{"index":18,"sense_key":"take%2:34:00::","synset_id":"consume.v.02"},{"index":19,"sense_key":"take%2:31:09::","synset_id":"take.v.19"},{"index":20,"sense_key":"take%2:40:15::","synset_id":"take.v.20"},{"index":21,"sense_key":"take%2:40:01::","synset_id":"take.v.21"},{"index":22,"sense_key":"take%2:38:00::","synset_id":"assume.v.05"},{"index":23,"sense_key":"take%2:40:07::","synset_id":"accept.v.05"},{"index":24,"sense_key":"take%2:30:08::","synset_id":"take.v.24"},{"index":25,"sense_key":"take%2:31:02::","synset_id":"learn.v.04"},{"index":26,"sense_key":"take%2:32:00::","synset_id":"claim.v.05"},{"index":27,"sense_key":"take%2:38:05::","synset_id":"take.v.27"},{"index":28,"sense_key":"take%2:33:09::","synset_id":"aim.v.01"},{"index":29,"sense_key":"take%2:30:09::","synset_id":"take.v.29"},{"index":30,"sense_key":"take%2:42:10::","synset_id":"carry.v.02"},{"index":31,"sense_key":"take%2:40:03::","synset_id":"lease.v.04"},{"index":32,"sense_key":"take%2:40:04::","synset_id":"subscribe.v.05"},{"index":33,"sense_key":"take%2:40:02::","synset_id":"take.v.33"},{"index":34,"sense_key":"take%2:40:08::","synset_id":"take.v.34"},{"index":35,"sense_key":"take%2:35:14::","synset_id":"take.v.35"},{"index":36,"sense_key":"take%2:32:09::","synset_id":"claim.v.04"},{"index":37,"sense_key":"take%2:42:03::","synset_id":"accept.v.08"},{"index":38,"sense_key":"take%2:42:15::","synset_id":"contain.v.05"},{"index":39,"sense_key":"take%2:41:13::","synset_id":"take.v.39"},{"index":40,"sense_key":"take%2:38:11::","synset_id":"drive.v.16"},{"index":41,"sense_key":"take%2:33:08::","synset_id":"take.v.41"},{"index":42,"sense_key":"take%2:29:08::","synset_id":"contract.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"take%2:31:07::","call_ids":["senseval3.d000.s036.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"take%2:31:07::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2977,"cached_input_tokens":0,"output_tokens":115,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.007104,"input_uncached_usd":0.005954,"input_cached_usd":0.0,"output_usd":0.0011500000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.722835374996066}
{"item_id":"senseval3.d000.s037.t000","gold_sense_keys":["mean%2:32:01::"],"candidates":[{"index":1,"sense_key":"mean%2:32:01::","synset_id":"mean.v.01"},{"index":2,"sense_key":"mean%2:42:00::","synset_id":"entail.v.01"},{"index":3,"sense_key":"mean%2:32:03::","synset_id":"mean.v.03"},{"index":4,"sense_key":"mean%2:31:00::","synset_id":"intend.v.01"},{"index":5,"sense_key":"mean%2:42:03::","synset_id":"mean.v.05"},{"index":6,"sense_key":"mean%2:31:01::","synset_id":"think_of.v.04"},{"index":7,"sense_key":"mean%2:31:05::","synset_id":"mean.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"mean%2:32:01::","call_ids":["senseval3.d000.s037.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"mean%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":755,"cached_input_tokens":0,"output_tokens":120,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0027099999999999997,"input_uncached_usd":0.0015099999999999998,"input_cached_usd":0.0,"output_usd":0.0012000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7421637079678476}
{"item_id":"senseval3.d000.s038.t000","gold_sense_keys":["have%2:40:00::"],"candidates":[{"index":1,"sense_key":"have%2:40:00::","synset_id":"have.v.01"},{"index":2,"sense_key":"have%2:42:00::","synset_id":"have.v.02"},{"index":3,"sense_key":"have%2:39:06::","synset_id":"experience.v.03"},{"index":4,"sense_key":"have%2:40:04::","synset_id":"own.v.01"},{"index":5,"sense_key":"have%2:30:00::","synset_id":"get.v.03"},{"index":6,"sense_key":"have%2:34:00::","synset_id":"consume.v.02"},{"index":7,"sense_key":"have%2:41:00::","synset_id":"have.v.07"},{"index":8,"sense_key":"have%2:36:00::","synset_id":"hold.v.03"},{"index":9,"sense_key":"have%2:40:01::","synset_id":"have.v.09"},{"index":10,"sense_key":"have%2:42:12::","synset_id":"have.v.10"},{"index":11,"sense_key":"have%2:30:01::","synset_id":"have.v.11"},{"index":12,"sense_key":"have%2:29:02::","synset_id":"have.v.12"},{"index":13,"sense_key":"have%2:32:00::","synset_id":"induce.v.02"},{"index":14,"sense_key":"have%2:40:05::","synset_id":"accept.v.02"},{"index":15,"sense_key":"have%2:40:03::","synset_id":"receive.v.01"},{"index":16,"sense_key":"have%2:29:05::","synset_id":"suffer.v.02"},{"index":17,"sense_key":"have%2:40:02::","synset_id":"have.v.17"},{"index":18,"sense_key":"have%2:29:00::","synset_id":"give_birth.v.01"},{"index":19,"sense_key":"have%2:35:00::","synset_id":"take.v.35"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"have%2:40:00::","call_ids":["senseval3.d000.s038.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"have%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1497,"cached_input_tokens":0,"output_tokens":75,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003744,"input_uncached_usd":0.0029939999999999997,"input_cached_usd":0.0,"output_usd":0.00075,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8109469170449302}
{"item_id":"senseval3.d000.s039.t000","gold_sense_keys":["hit%2:35:01::"],"candidates":[{"index":1,"sense_key":"hit%2:35:03::","synset_id":"hit.v.01"},{"index":2,"sense_key":"hit%2:35:01::","synset_id":"hit.v.02"},{"index":3,"sense_key":"hit%2:35:00::","synset_id":"hit.v.03"},{"index":4,"sense_key":"hit%2:38:00::","synset_id":"reach.v.01"},{"index":5,"sense_key":"hit%2:35:05::","synset_id":"hit.v.05"},{"index":6,"sense_key":"hit%2:33:01::","synset_id":"shoot.v.01"},{"index":7,"sense_key":"hit%2:40:05::","synset_id":"stumble.v.03"},{"index":8,"sense_key":"hit%2:33:02::","synset_id":"score.v.01"},{"index":9,"sense_key":"hit%2:39:00::","synset_id":"hit.v.09"},{"index":10,"sense_key":"hit%2:33:00::","synset_id":"strike.v.04"},{"index":11,"sense_key":"hit%2:41:00::","synset_id":"murder.v.01"},{"index":12,"sense_key":"hit%2:38:02::","synset_id":"hit.v.12"},{"index":13,"sense_key":"hit%2:38:07::","synset_id":"reach.v.02"},{"index":14,"sense_key":"hit%2:35:02::","synset_id":"strike.v.10"},{"index":15,"sense_key":"hit%2:34:00::","synset_id":"hit.v.15"},{"index":16,"sense_key":"hit%2:33:03::","synset_id":"hit.v.16"},{"index":17,"sense_key":"hit%2:32:12::","synset_id":"hit.v.17"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"hit%2:35:01::","call_ids":["senseval3.d000.s039.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"hit%2:35:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1402,"cached_input_tokens":0,"output_tokens":158,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004384,"input_uncached_usd":0.002804,"input_cached_usd":0.0,"output_usd":0.00158,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.110064291046001}
{"item_id":"senseval3.d000.s039.t001","gold_sense_keys":["truck%1:06:00::"],"candidates":[{"index":1,"sense_key":"truck%1:06:00::","synset_id":"truck.n.01"},{"index":2,"sense_key":"truck%1:06:01::","synset_id":"hand_truck.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"truck%1:06:00::","call_ids":["senseval3.d000.s039.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"truck%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":445,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00101,"input_uncached_usd":0.00089,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.47889666701667}
{"item_id":"senseval3.d000.s040.t000","gold_sense_keys":["return%2:40:03::"],"candidates":[{"index":1,"sense_key":"return%2:38:00::","synset_id":"return.v.01"},{"index":2,"sense_key":"return%2:40:08::","synset_id":"render.v.07"},{"index":3,"sense_key":"return%2:30:00::","synset_id":"revert.v.01"},{"index":4,"sense_key":"return%2:32:00::","synset_id":"hark_back.v.01"},{"index":5,"sense_key":"return%2:38:12::","synset_id":"return.v.05"},{"index":6,"sense_key":"return%2:40:03::","synset_id":"return.v.06"},{"index":7,"sense_key":"return%2:35:01::","synset_id":"return.v.07"},{"index":8,"sense_key":"return%2:32:01::","synset_id":"retort.v.01"},{"index":9,"sense_key":"return%2:30:01::","synset_id":"come_back.v.01"},{"index":10,"sense_key":"return%2:40:00::","synset_id":"refund.v.01"},{"index":11,"sense_key":"return%2:32:03::","synset_id":"render.v.05"},{"index":12,"sense_key":"return%2:41:00::","synset_id":"reelect.v.01"},{"index":13,"sense_key":"return%2:40:13::","synset_id":"fall.v.21"},{"index":14,"sense_key":"return%2:38:01::","synset_id":"return.v.14"},{"index":15,"sense_key":"return%2:36:00::","synset_id":"render.v.04"},{"index":16,"sense_key":"return%2:32:02::","synset_id":"return.v.16"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"return%2:40:03::","call_ids":["senseval3.d000.s040.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"return%2:40:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1127,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002894,"input_uncached_usd":0.002254,"input_cached_usd":0.0,"output_usd":0.00064,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9358960420358926}
{"item_id":"senseval3.d000.s040.t001","gold_sense_keys":["favor%1:04:00::"],"candidates":[{"index":1,"sense_key":"favor%1:04:00::","synset_id":"favor.n.01"},{"index":2,"sense_key":"favor%1:07:00::","synset_id":"favor.n.02"},{"index":3,"sense_key":"favor%1:09:00::","synset_id":"favor.n.03"},{"index":4,"sense_key":"favor%1:12:00::","synset_id":"favor.n.04"},{"index":5,"sense_key":"favor%1:06:00::","synset_id":"party_favor.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"favor%1:04:00::","call_ids":["senseval3.d000.s040.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"favor%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":569,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001258,"input_uncached_usd":0.001138,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0991224169265479}
{"item_id":"senseval3.d000.s041.t000","gold_sense_keys":["do%2:36:01::","do%2:41:01::","do%2:41:04::"],"candidates":[{"index":1,"sense_key":"do%2:41:01::","synset_id":"make.v.01"},{"index":2,"sense_key":"do%2:36:01::","synset_id":"perform.v.01"},{"index":3,"sense_key":"do%2:41:04::","synset_id":"do.v.03"},{"index":4,"sense_key":"do%2:42:00::","synset_id":"do.v.04"},{"index":5,"sense_key":"do%2:36:02::","synset_id":"cause.v.01"},{"index":6,"sense_key":"do%2:41:02::","synset_id":"practice.v.01"},{"index":7,"sense_key":"do%2:42:01::","synset_id":"suffice.v.01"},{"index":8,"sense_key":"do%2:36:00::","synset_id":"do.v.08"},{"index":9,"sense_key":"do%2:29:09::","synset_id":"act.v.02"},{"index":10,"sense_key":"do%2:42:02::","synset_id":"serve.v.09"},{"index":11,"sense_key":"do%2:41:03::","synset_id":"do.v.11"},{"index":12,"sense_key":"do%2:29:04::","synset_id":"dress.v.16"},{"index":13,"sense_key":"do%2:38:00::","synset_id":"do.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"do%2:36:01::","call_ids":["senseval3.d000.s041.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"do%2:36:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1131,"cached_input_tokens":0,"output_tokens":88,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003142,"input_uncached_usd":0.0022619999999999997,"input_cached_usd":0.0,"output_usd":0.00088,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9971263749757782}
{"item_id":"senseval3.d000.s041.t002","gold_sense_keys":["like%2:37:04::"],"candidates":[{"index":1,"sense_key":"like%2:37:01::","synset_id":"wish.v.02"},{"index":2,"sense_key":"like%2:37:05::","synset_id":"like.v.02"},{"index":3,"sense_key":"like%2:37:04::","synset_id":"like.v.03"},{"index":4,"sense_key":"like%2:31:00::","synset_id":"like.v.04"},{"index":5,"sense_key":"like%2:37:06::","synset_id":"like.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"like%2:37:04::","call_ids":["senseval3.d000.s041.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"like%2:37:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":566,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001712,"input_uncached_usd":0.001132,"input_cached_usd":0.0,"output_usd":0.00058,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1795764589915052}
{"item_id":"senseval3.d000.s043.t001","gold_sense_keys":["try%2:37:00::","try%2:41:01::"],"candidates":[{"index":1,"sense_key":"try%2:41:00::","synset_id":"try.v.01"},{"index":2,"sense_key":"try%2:41:01::","synset_id":"test.v.01"},{"index":3,"sense_key":"try%2:41:06::","synset_id":"judge.v.05"},{"index":4,"sense_key":"try%2:34:00::","synset_id":"sample.v.01"},{"index":5,"sense_key":"try%2:41:05::","synset_id":"hear.v.03"},{"index":6,"sense_key":"try%2:37:01::","synset_id":"try.v.06"},{"index":7,"sense_key":"try%2:37:00::","synset_id":"try.v.07"},{"index":8,"sense_key":"try%2:30:00::","synset_id":"try.v.08"},{"index":9,"sense_key":"try%2:29:00::","synset_id":"try_on.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"try%2:41:01::","call_ids":["senseval3.d000.s043.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"try%2:41:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":923,"cached_input_tokens":0,"output_tokens":81,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002656,"input_uncached_usd":0.001846,"input_cached_usd":0.0,"output_usd":0.0008100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0829530409537256}
{"item_id":"senseval3.d000.s044.t000","gold_sense_keys":["rub%2:35:00::"],"candidates":[{"index":1,"sense_key":"rub%2:35:00::","synset_id":"rub.v.01"},{"index":2,"sense_key":"rub%2:35:01::","synset_id":"rub.v.02"},{"index":3,"sense_key":"rub%2:39:00::","synset_id":"rub.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"rub%2:35:00::","call_ids":["senseval3.d000.s044.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"rub%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":493,"cached_input_tokens":0,"output_tokens":107,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0020559999999999997,"input_uncached_usd":0.000986,"input_cached_usd":0.0,"output_usd":0.00107,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.600735375075601}
{"item_id":"senseval3.d000.s044.t001","gold_sense_keys":["forehead%1:08:00::"],"candidates":[{"index":1,"sense_key":"forehead%1:08:00::","synset_id":"brow.n.01"},{"index":2,"sense_key":"forehead%1:08:01::","synset_id":"frontal_bone.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"forehead%1:08:00::","call_ids":["senseval3.d000.s044.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"forehead%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":435,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00099,"input_uncached_usd":0.00087,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.026752999983728}
{"item_id":"senseval3.d000.s045.t001","gold_sense_keys":["feel%2:29:00::","feel%2:31:01::","feel%2:37:00::"],"candidates":[{"index":1,"sense_key":"feel%2:37:00::","synset_id":"feel.v.01"},{"index":2,"sense_key":"feel%2:31:00::","synset_id":"find.v.05"},{"index":3,"sense_key":"feel%2:39:00::","synset_id":"feel.v.03"},{"index":4,"sense_key":"feel%2:29:00::","synset_id":"feel.v.04"},{"index":5,"sense_key":"feel%2:31:01::","synset_id":"feel.v.05"},{"index":6,"sense_key":"feel%2:39:01::","synset_id":"feel.v.06"},{"index":7,"sense_key":"feel%2:42:00::","synset_id":"feel.v.07"},{"index":8,"sense_key":"feel%2:39:10::","synset_id":"feel.v.08"},{"index":9,"sense_key":"feel%2:35:00::","synset_id":"feel.v.09"},{"index":10,"sense_key":"feel%2:35:01::","synset_id":"palpate.v.01"},{"index":11,"sense_key":"feel%2:40:00::","synset_id":"feel.v.11"},{"index":12,"sense_key":"feel%2:39:09::","synset_id":"feel.v.12"},{"index":13,"sense_key":"feel%2:35:11::","synset_id":"feel.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"feel%2:29:00::","call_ids":["senseval3.d000.s045.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"feel%2:29:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1116,"cached_input_tokens":0,"output_tokens":145,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0036820000000000004,"input_uncached_usd":0.002232,"input_cached_usd":0.0,"output_usd":0.0014500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9221173749538139}
{"item_id":"senseval3.d000.s045.t002","gold_sense_keys":["woolly%5:00:00:confused:00"],"candidates":[{"index":1,"sense_key":"woolly%5:00:00:soft:01","synset_id":"flocculent.s.01"},{"index":2,"sense_key":"woolly%5:00:00:confused:00","synset_id":"addled.s.02"},{"index":3,"sense_key":"woolly%5:00:00:hairy:00","synset_id":"wooly.s.03"},{"index":4,"sense_key":"woolly%5:00:02:hairy:00","synset_id":"lanate.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"woolly%5:00:00:confused:00","call_ids":["senseval3.d000.s045.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"woolly%5:00:00:confused:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":628,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001376,"input_uncached_usd":0.001256,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5188137500081211}
{"item_id":"senseval3.d000.s046.t001","gold_sense_keys":["guy%1:18:00::"],"candidates":[{"index":1,"sense_key":"guy%1:18:00::","synset_id":"guy.n.01"},{"index":2,"sense_key":"guy%1:06:01::","synset_id":"guy.n.02"},{"index":3,"sense_key":"guy%1:06:00::","synset_id":"guy.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"guy%1:18:00::","call_ids":["senseval3.d000.s046.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"guy%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":529,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001178,"input_uncached_usd":0.001058,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4341217910405248}
{"item_id":"senseval3.d000.s046.t002","gold_sense_keys":["make%2:36:12::"],"candidates":[{"index":1,"sense_key":"make%2:41:00::","synset_id":"make.v.01"},{"index":2,"sense_key":"make%2:30:00::","synset_id":"make.v.02"},{"index":3,"sense_key":"make%2:36:00::","synset_id":"make.v.03"},{"index":4,"sense_key":"make%2:32:00::","synset_id":"induce.v.02"},{"index":5,"sense_key":"make%2:36:08::","synset_id":"cause.v.01"},{"index":6,"sense_key":"make%2:36:01::","synset_id":"produce.v.02"},{"index":7,"sense_key":"make%2:31:00::","synset_id":"draw.v.04"},{"index":8,"sense_key":"make%2:36:15::","synset_id":"make.v.08"},{"index":9,"sense_key":"make%2:36:09::","synset_id":"create.v.05"},{"index":10,"sense_key":"make%2:40:01::","synset_id":"gain.v.08"},{"index":11,"sense_key":"make%2:36:05::","synset_id":"do.v.08"},{"index":12,"sense_key":"make%2:42:05::","synset_id":"form.v.02"},{"index":13,"sense_key":"make%2:38:02::","synset_id":"reach.v.07"},{"index":14,"sense_key":"make%2:42:00::","synset_id":"make.v.14"},{"index":15,"sense_key":"make%2:36:11::","synset_id":"make.v.15"},{"index":16,"sense_key":"make%2:36:12::","synset_id":"make.v.16"},{"index":17,"sense_key":"make%2:36:13::","synset_id":"construct.v.01"},{"index":18,"sense_key":"make%2:30:15::","synset_id":"make.v.18"},{"index":19,"sense_key":"make%2:29:08::","synset_id":"make.v.19"},{"index":20,"sense_key":"make%2:41:03::","synset_id":"name.v.03"},{"index":21,"sense_key":"make%2:40:02::","synset_id":"have.v.17"},{"index":22,"sense_key":"make%2:38:00::","synset_id":"reach.v.01"},{"index":23,"sense_key":"make%2:31:13::","synset_id":"lay_down.v.01"},{"index":24,"sense_key":"make%2:41:13::","synset_id":"make.v.24"},{"index":25,"sense_key":"make%2:41:01::","synset_id":"make.v.25"},{"index":26,"sense_key":"make%2:36:04::","synset_id":"hold.v.03"},{"index":27,"sense_key":"make%2:30:02::","synset_id":"make.v.27"},{"index":28,"sense_key":"make%2:38:05::","synset_id":"take.v.27"},{"index":29,"sense_key":"make%2:29:00::","synset_id":"stool.v.04"},{"index":30,"sense_key":"make%2:42:09::","synset_id":"make.v.30"},{"index":31,"sense_key":"make%2:42:13::","synset_id":"make.v.31"},{"index":32,"sense_key":"make%2:42:12::","synset_id":"make.v.32"},{"index":33,"sense_key":"make%2:42:11::","synset_id":"make.v.33"},{"index":34,"sense_key":"make%2:42:15::","synset_id":"make.v.34"},{"index":35,"sense_key":"make%2:39:12::","synset_id":"make.v.35"},{"index":36,"sense_key":"make%2:38:10::","synset_id":"make.v.36"},{"index":37,"sense_key":"make%2:38:15::","synset_id":"make.v.37"},{"index":38,"sense_key":"make%2:36:10::","synset_id":"make.v.38"},{"index":39,"sense_key":"make%2:36:07::","synset_id":"cook.v.02"},{"index":40,"sense_key":"make%2:35:13::","synset_id":"seduce.v.01"},{"index":41,"sense_key":"make%2:32:14::","synset_id":"make.v.41"},{"index":42,"sense_key":"make%2:32:09::","synset_id":"make.v.42"},{"index":43,"sense_key":"make%2:31:10::","synset_id":"make.v.43"},{"index":44,"sense_key":"make%2:31:12::","synset_id":"make.v.44"},{"index":45,"sense_key":"make%2:30:10::","synset_id":"make.v.45"},{"index":46,"sense_key":"make%2:30:14::","synset_id":"make.v.46"},{"index":47,"sense_key":"make%2:30:12::","synset_id":"make.v.47"},{"index":48,"sense_key":"make%2:29:06::","synset_id":"make.v.48"},{"index":49,"sense_key":"make%2:29:01::","synset_id":"make.v.49"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":16,"chosen_sense_key":"make%2:36:12::","call_ids":["senseval3.d000.s046.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":16,"predicted_sense_key":"make%2:36:12::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":3002,"cached_input_tokens":0,"output_tokens":82,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.006824,"input_uncached_usd":0.006004,"input_cached_usd":0.0,"output_usd":0.0008200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4304877090035006}
{"item_id":"senseval3.d000.s046.t003","gold_sense_keys":["exit%1:04:00::"],"candidates":[{"index":1,"sense_key":"exit%1:06:00::","synset_id":"exit.n.01"},{"index":2,"sense_key":"exit%1:11:00::","synset_id":"passing.n.02"},{"index":3,"sense_key":"exit%1:04:00::","synset_id":"exit.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"exit%1:04:00::","call_ids":["senseval3.d000.s046.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"exit%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":499,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001118,"input_uncached_usd":0.000998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4033021250506863}
{"item_id":"senseval3.d000.s047.t000","gold_sense_keys":["expect%2:31:00::"],"candidates":[{"index":1,"sense_key":"expect%2:31:00::","synset_id":"expect.v.01"},{"index":2,"sense_key":"expect%2:32:00::","synset_id":"ask.v.04"},{"index":3,"sense_key":"expect%2:31:01::","synset_id":"expect.v.03"},{"index":4,"sense_key":"expect%2:31:05::","synset_id":"expect.v.04"},{"index":5,"sense_key":"expect%2:29:02::","synset_id":"expect.v.05"},{"index":6,"sense_key":"expect%2:29:00::","synset_id":"have_a_bun_in_the_oven.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"expect%2:31:00::","call_ids":["senseval3.d000.s047.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"expect%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":731,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002072,"input_uncached_usd":0.001462,"input_cached_usd":0.0,"output_usd":0.0006100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2111022080061957}
{"item_id":"senseval3.d000.s047.t001","gold_sense_keys":["come%2:30:13::"],"candidates":[{"index":1,"sense_key":"come%2:38:00::","synset_id":"come.v.01"},{"index":2,"sense_key":"come%2:38:04::","synset_id":"arrive.v.01"},{"index":3,"sense_key":"come%2:30:01::","synset_id":"come.v.03"},{"index":4,"sense_key":"come%2:30:03::","synset_id":"come.v.04"},{"index":5,"sense_key":"come%2:42:04::","synset_id":"come.v.05"},{"index":6,"sense_key":"come%2:30:02::","synset_id":"come.v.06"},{"index":7,"sense_key":"come%2:42:13::","synset_id":"issue_forth.v.01"},{"index":8,"sense_key":"come%2:42:09::","synset_id":"hail.v.02"},{"index":9,"sense_key":"come%2:42:03::","synset_id":"come.v.09"},{"index":10,"sense_key":"come%2:42:05::","synset_id":"come.v.10"},{"index":11,"sense_key":"come%2:38:03::","synset_id":"come.v.11"},{"index":12,"sense_key":"come%2:42:00::","synset_id":"fall.v.04"},{"index":13,"sense_key":"come%2:30:13::","synset_id":"come.v.13"},{"index":14,"sense_key":"come%2:42:12::","synset_id":"total.v.01"},{"index":15,"sense_key":"come%2:42:11::","synset_id":"come.v.15"},{"index":16,"sense_key":"come%2:38:02::","synset_id":"come.v.16"},{"index":17,"sense_key":"come%2:31:14::","synset_id":"occur.v.02"},{"index":18,"sense_key":"come%2:42:15::","synset_id":"derive.v.05"},{"index":19,"sense_key":"come%2:42:02::","synset_id":"do.v.04"},{"index":20,"sense_key":"come%2:39:00::","synset_id":"come.v.20"},{"index":21,"sense_key":"come%2:31:13::","synset_id":"come.v.21"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":13,"chosen_sense_key":"come%2:30:13::","call_ids":["senseval3.d000.s047.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":13,"predicted_sense_key":"come%2:30:13::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1526,"cached_input_tokens":0,"output_tokens":102,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004072,"input_uncached_usd":0.003052,"input_cached_usd":0.0,"output_usd":0.00102,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.971449250006117}
{"item_id":"senseval3.d000.s047.t003","gold_sense_keys":["people%1:14:00::"],"candidates":[{"index":1,"sense_key":"people%1:14:00::","synset_id":"people.n.01"},{"index":2,"sense_key":"people%1:14:01::","synset_id":"citizenry.n.01"},{"index":3,"sense_key":"people%1:14:02::","synset_id":"people.n.03"},{"index":4,"sense_key":"people%1:14:03::","synset_id":"multitude.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"people%1:14:00::","call_ids":["senseval3.d000.s047.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"people%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":585,"cached_input_tokens":0,"output_tokens":51,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00168,"input_uncached_usd":0.00117,"input_cached_usd":0.0,"output_usd":0.00051,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.563996292068623}
{"item_id":"senseval3.d000.s048.t000","gold_sense_keys":["concede%2:32:01::"],"candidates":[{"index":1,"sense_key":"concede%2:32:00::","synset_id":"concede.v.01"},{"index":2,"sense_key":"concede%2:32:01::","synset_id":"concede.v.02"},{"index":3,"sense_key":"concede%2:40:00::","synset_id":"concede.v.03"},{"index":4,"sense_key":"concede%2:33:00::","synset_id":"concede.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"concede%2:32:01::","call_ids":["senseval3.d000.s048.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"concede%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":589,"cached_input_tokens":0,"output_tokens":225,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0034280000000000005,"input_uncached_usd":0.001178,"input_cached_usd":0.0,"output_usd":0.0022500000000000003,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.693260125000961}
{"item_id":"senseval3.d000.s048.t001","gold_sense_keys":["know%2:31:01::"],"candidates":[{"index":1,"sense_key":"know%2:31:01::","synset_id":"know.v.01"},{"index":2,"sense_key":"know%2:31:03::","synset_id":"know.v.02"},{"index":3,"sense_key":"know%2:31:02::","synset_id":"know.v.03"},{"index":4,"sense_key":"know%2:31:00::","synset_id":"know.v.04"},{"index":5,"sense_key":"know%2:31:04::","synset_id":"know.v.05"},{"index":6,"sense_key":"know%2:31:06::","synset_id":"acknowledge.v.06"},{"index":7,"sense_key":"know%2:31:15::","synset_id":"know.v.07"},{"index":8,"sense_key":"know%2:35:00::","synset_id":"sleep_together.v.01"},{"index":9,"sense_key":"know%2:31:11::","synset_id":"know.v.09"},{"index":10,"sense_key":"know%2:31:12::","synset_id":"know.v.10"},{"index":11,"sense_key":"know%2:31:14::","synset_id":"know.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"know%2:31:01::","call_ids":["senseval3.d000.s048.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"know%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1089,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002728,"input_uncached_usd":0.002178,"input_cached_usd":0.0,"output_usd":0.00055,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.477206208044663}
{"item_id":"senseval3.d000.s049.t000","gold_sense_keys":["keep%2:42:07::"],"candidates":[{"index":1,"sense_key":"keep%2:42:00::","synset_id":"keep.v.01"},{"index":2,"sense_key":"keep%2:42:07::","synset_id":"continue.v.01"},{"index":3,"sense_key":"keep%2:40:00::","synset_id":"keep.v.03"},{"index":4,"sense_key":"keep%2:41:01::","synset_id":"prevent.v.02"},{"index":5,"sense_key":"keep%2:41:03::","synset_id":"observe.v.09"},{"index":6,"sense_key":"keep%2:31:00::","synset_id":"observe.v.08"},{"index":7,"sense_key":"keep%2:40:01::","synset_id":"keep.v.07"},{"index":8,"sense_key":"keep%2:32:00::","synset_id":"keep.v.08"},{"index":9,"sense_key":"keep%2:42:01::","synset_id":"keep.v.09"},{"index":10,"sense_key":"keep%2:41:02::","synset_id":"retain.v.02"},{"index":11,"sense_key":"keep%2:34:00::","synset_id":"sustain.v.04"},{"index":12,"sense_key":"keep%2:42:03::","synset_id":"keep.v.12"},{"index":13,"sense_key":"keep%2:41:05::","synset_id":"observe.v.06"},{"index":14,"sense_key":"keep%2:41:00::","synset_id":"restrain.v.01"},{"index":15,"sense_key":"keep%2:42:02::","synset_id":"keep.v.15"},{"index":16,"sense_key":"keep%2:42:12::","synset_id":"keep.v.16"},{"index":17,"sense_key":"keep%2:40:09::","synset_id":"keep_open.v.01"},{"index":18,"sense_key":"keep%2:40:13::","synset_id":"keep.v.18"},{"index":19,"sense_key":"keep%2:40:12::","synset_id":"keep.v.19"},{"index":20,"sense_key":"keep%2:40:10::","synset_id":"keep.v.20"},{"index":21,"sense_key":"keep%2:35:10::","synset_id":"keep.v.21"},{"index":22,"sense_key":"keep%2:30:10::","synset_id":"preserve.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"keep%2:42:07::","call_ids":["senseval3.d000.s049.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"keep%2:42:07::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1722,"cached_input_tokens":0,"output_tokens":129,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004734,"input_uncached_usd":0.003444,"input_cached_usd":0.0,"output_usd":0.0012900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6469454589532688}
{"item_id":"senseval3.d000.s049.t001","gold_sense_keys":["ride%2:32:00::"],"candidates":[{"index":1,"sense_key":"ride%2:38:01::","synset_id":"ride.v.01"},{"index":2,"sense_key":"ride%2:38:00::","synset_id":"ride.v.02"},{"index":3,"sense_key":"ride%2:42:00::","synset_id":"ride.v.03"},{"index":4,"sense_key":"ride%2:38:02::","synset_id":"ride.v.04"},{"index":5,"sense_key":"ride%2:32:00::","synset_id":"tease.v.02"},{"index":6,"sense_key":"ride%2:42:04::","synset_id":"ride.v.06"},{"index":7,"sense_key":"ride%2:42:03::","synset_id":"drive.v.12"},{"index":8,"sense_key":"ride%2:42:07::","synset_id":"depend_on.v.01"},{"index":9,"sense_key":"ride%2:42:02::","synset_id":"ride.v.09"},{"index":10,"sense_key":"ride%2:38:04::","synset_id":"ride.v.10"},{"index":11,"sense_key":"ride%2:38:03::","synset_id":"ride.v.11"},{"index":12,"sense_key":"ride%2:38:07::","synset_id":"ride.v.12"},{"index":13,"sense_key":"ride%2:35:00::","synset_id":"ride.v.13"},{"index":14,"sense_key":"ride%2:35:03::","synset_id":"ride.v.14"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"ride%2:32:00::","call_ids":["senseval3.d000.s049.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"ride%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1201,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002522,"input_uncached_usd":0.002402,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2430035000434145}
{"item_id":"senseval3.d000.s049.t002","gold_sense_keys":["like%2:37:05::"],"candidates":[{"index":1,"sense_key":"like%2:37:01::","synset_id":"wish.v.02"},{"index":2,"sense_key":"like%2:37:05::","synset_id":"like.v.02"},{"index":3,"sense_key":"like%2:37:04::","synset_id":"like.v.03"},{"index":4,"sense_key":"like%2:31:00::","synset_id":"like.v.04"},{"index":5,"sense_key":"like%2:37:06::","synset_id":"like.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"like%2:37:05::","call_ids":["senseval3.d000.s049.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"like%2:37:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":620,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00197,"input_uncached_usd":0.00124,"input_cached_usd":0.0,"output_usd":0.0007300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.34423320798669}
{"item_id":"senseval3.d000.s049.t003","gold_sense_keys":["listen%2:39:00::"],"candidates":[{"index":1,"sense_key":"listen%2:39:00::","synset_id":"listen.v.01"},{"index":2,"sense_key":"listen%2:39:01::","synset_id":"listen.v.02"},{"index":3,"sense_key":"listen%2:41:11::","synset_id":"heed.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"listen%2:39:00::","call_ids":["senseval3.d000.s049.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"listen%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":541,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001852,"input_uncached_usd":0.0010819999999999998,"input_cached_usd":0.0,"output_usd":0.0007700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4319578328868374}
{"item_id":"senseval3.d000.s049.t004","gold_sense_keys":["radio%1:06:01::"],"candidates":[{"index":1,"sense_key":"radio%1:10:00::","synset_id":"radio.n.01"},{"index":2,"sense_key":"radio%1:06:01::","synset_id":"radio_receiver.n.01"},{"index":3,"sense_key":"radio%1:06:00::","synset_id":"radio.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"radio%1:06:01::","call_ids":["senseval3.d000.s049.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"radio%1:06:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":545,"cached_input_tokens":0,"output_tokens":205,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0031400000000000004,"input_uncached_usd":0.00109,"input_cached_usd":0.0,"output_usd":0.00205,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.7971400000387803}
{"item_id":"senseval3.d000.s049.t005","gold_sense_keys":["sing%2:36:00::"],"candidates":[{"index":1,"sense_key":"sing%2:36:01::","synset_id":"sing.v.01"},{"index":2,"sense_key":"sing%2:36:00::","synset_id":"sing.v.02"},{"index":3,"sense_key":"sing%2:32:08::","synset_id":"sing.v.03"},{"index":4,"sense_key":"sing%2:32:00::","synset_id":"whistle.v.05"},{"index":5,"sense_key":"sing%2:32:01::","synset_id":"spill_the_beans.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"sing%2:36:00::","call_ids":["senseval3.d000.s049.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"sing%2:36:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":661,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0014420000000000001,"input_uncached_usd":0.001322,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2011852080468088}
{"item_id":"senseval3.d000.s049.t007","gold_sense_keys":["bath%1:04:00::"],"candidates":[{"index":1,"sense_key":"bath%1:06:02::","synset_id":"bath.n.01"},{"index":2,"sense_key":"bath%1:04:00::","synset_id":"bath.n.02"},{"index":3,"sense_key":"bath%1:06:00::","synset_id":"bathtub.n.01"},{"index":4,"sense_key":"bath%1:23:00::","synset_id":"bath.n.04"},{"index":5,"sense_key":"bath%1:15:00::","synset_id":"bath.n.05"},{"index":6,"sense_key":"bath%1:06:01::","synset_id":"bathroom.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bath%1:04:00::","call_ids":["senseval3.d000.s049.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bath%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":727,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001574,"input_uncached_usd":0.001454,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.384673000080511}
{"item_id":"senseval3.d000.s050.t000","gold_sense_keys":["say%2:32:01::","say%2:32:13::"],"candidates":[{"index":1,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":6,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":9,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":10,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":11,"sense_key":"say%2:32:09::","synset_id":"say.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"say%2:32:01::","call_ids":["senseval3.d000.s050.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"say%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1037,"cached_input_tokens":0,"output_tokens":82,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002894,"input_uncached_usd":0.002074,"input_cached_usd":0.0,"output_usd":0.0008200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.137610250036232}
{"item_id":"senseval3.d000.s050.t001","gold_sense_keys":["neighbor%1:18:00::"],"candidates":[{"index":1,"sense_key":"neighbor%1:18:00::","synset_id":"neighbor.n.01"},{"index":2,"sense_key":"neighbor%1:17:00::","synset_id":"neighbor.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"neighbor%1:18:00::","call_ids":["senseval3.d000.s050.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"neighbor%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":511,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001142,"input_uncached_usd":0.001022,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8180183749645948}
{"item_id":"senseval3.d000.s050.t002","gold_sense_keys":["complain%2:32:00::"],"candidates":[{"index":1,"sense_key":"complain%2:32:00::","synset_id":"complain.v.01"},{"index":2,"sense_key":"complain%2:32:01::","synset_id":"complain.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"complain%2:32:00::","call_ids":["senseval3.d000.s050.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"complain%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":533,"cached_input_tokens":0,"output_tokens":50,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001566,"input_uncached_usd":0.0010659999999999999,"input_cached_usd":0.0,"output_usd":0.0005,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3576354159740731}
{"item_id":"senseval3.d000.s050.t003","gold_sense_keys":["believe%2:31:00::"],"candidates":[{"index":1,"sense_key":"believe%2:31:00::","synset_id":"believe.v.01"},{"index":2,"sense_key":"believe%2:31:04::","synset_id":"think.v.01"},{"index":3,"sense_key":"believe%2:31:03::","synset_id":"believe.v.03"},{"index":4,"sense_key":"believe%2:31:01::","synset_id":"believe.v.04"},{"index":5,"sense_key":"believe%2:31:02::","synset_id":"believe.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"believe%2:31:00::","call_ids":["senseval3.d000.s050.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"believe%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":669,"cached_input_tokens":0,"output_tokens":134,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002678,"input_uncached_usd":0.001338,"input_cached_usd":0.0,"output_usd":0.00134,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1425748330075294}
{"item_id":"senseval3.d000.s051.t000","gold_sense_keys":["tell%2:32:00::"],"candidates":[{"index":1,"sense_key":"tell%2:32:04::","synset_id":"state.v.01"},{"index":2,"sense_key":"tell%2:32:00::","synset_id":"tell.v.02"},{"index":3,"sense_key":"tell%2:32:01::","synset_id":"tell.v.03"},{"index":4,"sense_key":"tell%2:32:02::","synset_id":"order.v.01"},{"index":5,"sense_key":"tell%2:31:00::","synset_id":"tell.v.05"},{"index":6,"sense_key":"tell%2:32:08::","synset_id":"assure.v.02"},{"index":7,"sense_key":"tell%2:32:03::","synset_id":"tell.v.07"},{"index":8,"sense_key":"tell%2:31:01::","synset_id":"distinguish.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"tell%2:32:00::","call_ids":["senseval3.d000.s051.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"tell%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":845,"cached_input_tokens":0,"output_tokens":138,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00307,"input_uncached_usd":0.0016899999999999999,"input_cached_usd":0.0,"output_usd":0.0013800000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.4388897500466555}
{"item_id":"senseval3.d000.s051.t001","gold_sense_keys":["bother%2:37:00::"],"candidates":[{"index":1,"sense_key":"bother%2:41:00::","synset_id":"trouble_oneself.v.01"},{"index":2,"sense_key":"bother%2:37:00::","synset_id":"annoy.v.01"},{"index":3,"sense_key":"bother%2:41:01::","synset_id":"trouble.v.02"},{"index":4,"sense_key":"bother%2:38:00::","synset_id":"bother.v.04"},{"index":5,"sense_key":"bother%2:37:03::","synset_id":"bother.v.05"},{"index":6,"sense_key":"bother%2:37:02::","synset_id":"bother.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bother%2:37:00::","call_ids":["senseval3.d000.s051.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bother%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":804,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002338,"input_uncached_usd":0.0016079999999999998,"input_cached_usd":0.0,"output_usd":0.0007300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4939541249768808}
{"item_id":"senseval3.d000.s052.t000","gold_sense_keys":["man%1:18:00::"],"candidates":[{"index":1,"sense_key":"man%1:18:00::","synset_id":"man.n.01"},{"index":2,"sense_key":"man%1:18:03::","synset_id":"serviceman.n.01"},{"index":3,"sense_key":"man%1:18:04::","synset_id":"man.n.03"},{"index":4,"sense_key":"man%1:05:01::","synset_id":"homo.n.02"},{"index":5,"sense_key":"man%1:18:08::","synset_id":"man.n.05"},{"index":6,"sense_key":"man%1:18:06::","synset_id":"man.n.06"},{"index":7,"sense_key":"man%1:18:05::","synset_id":"valet.n.01"},{"index":8,"sense_key":"man%1:18:07::","synset_id":"man.n.08"},{"index":9,"sense_key":"man%1:15:00::","synset_id":"man.n.09"},{"index":10,"sense_key":"man%1:06:00::","synset_id":"man.n.10"},{"index":11,"sense_key":"man%1:05:00::","synset_id":"world.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"man%1:18:00::","call_ids":["senseval3.d000.s052.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"man%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1115,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0023499999999999997,"input_uncached_usd":0.0022299999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1807976249838248}
{"item_id":"senseval3.d000.s052.t001","gold_sense_keys":["close%2:35:00::"],"candidates":[{"index":1,"sense_key":"close%2:35:00::","synset_id":"close.v.01"},{"index":2,"sense_key":"close%2:35:06::","synset_id":"close.v.02"},{"index":3,"sense_key":"close%2:41:00::","synset_id":"close_up.v.01"},{"index":4,"sense_key":"close%2:41:01::","synset_id":"close.v.04"},{"index":5,"sense_key":"close%2:42:00::","synset_id":"conclude.v.04"},{"index":6,"sense_key":"close%2:32:05::","synset_id":"close.v.06"},{"index":7,"sense_key":"close%2:42:01::","synset_id":"close.v.07"},{"index":8,"sense_key":"close%2:41:15::","synset_id":"close.v.08"},{"index":9,"sense_key":"close%2:39:10::","synset_id":"close.v.09"},{"index":10,"sense_key":"close%2:38:08::","synset_id":"close.v.10"},{"index":11,"sense_key":"close%2:38:10::","synset_id":"close.v.11"},{"index":12,"sense_key":"close%2:38:15::","synset_id":"close.v.12"},{"index":13,"sense_key":"close%2:35:14::","synset_id":"close.v.13"},{"index":14,"sense_key":"close%2:35:07::","synset_id":"close.v.14"},{"index":15,"sense_key":"close%2:35:11::","synset_id":"close.v.15"},{"index":16,"sense_key":"close%2:35:15::","synset_id":"close_up.v.03"},{"index":17,"sense_key":"close%2:30:08::","synset_id":"close.v.17"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"close%2:35:00::","call_ids":["senseval3.d000.s052.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"close%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1286,"cached_input_tokens":0,"output_tokens":109,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0036620000000000003,"input_uncached_usd":0.002572,"input_cached_usd":0.0,"output_usd":0.00109,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5850192920770496}
{"item_id":"senseval3.d000.s052.t002","gold_sense_keys":["eye%1:08:00::"],"candidates":[{"index":1,"sense_key":"eye%1:08:00::","synset_id":"eye.n.01"},{"index":2,"sense_key":"eye%1:09:00::","synset_id":"eye.n.02"},{"index":3,"sense_key":"eye%1:09:01::","synset_id":"eye.n.03"},{"index":4,"sense_key":"eye%1:15:00::","synset_id":"center.n.01"},{"index":5,"sense_key":"eye%1:06:00::","synset_id":"eye.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"eye%1:08:00::","call_ids":["senseval3.d000.s052.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"eye%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":649,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001418,"input_uncached_usd":0.001298,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0620267080375925}
{"item_id":"senseval3.d000.s052.t003","gold_sense_keys":["nod%2:32:00::"],"candidates":[{"index":1,"sense_key":"nod%2:32:01::","synset_id":"nod.v.01"},{"index":2,"sense_key":"nod%2:32:00::","synset_id":"nod.v.02"},{"index":3,"sense_key":"nod%2:29:03::","synset_id":"nod.v.03"},{"index":4,"sense_key":"nod%2:38:00::","synset_id":"nod.v.04"},{"index":5,"sense_key":"nod%2:29:02::","synset_id":"nod.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"nod%2:32:00::","call_ids":["senseval3.d000.s052.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"nod%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":636,"cached_input_tokens":0,"output_tokens":153,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002802,"input_uncached_usd":0.001272,"input_cached_usd":0.0,"output_usd":0.0015300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.970769083010964}
{"item_id":"senseval3.d000.s053.t000","gold_sense_keys":["look%2:39:00::"],"candidates":[{"index":1,"sense_key":"look%2:39:00::","synset_id":"look.v.01"},{"index":2,"sense_key":"look%2:39:01::","synset_id":"look.v.02"},{"index":3,"sense_key":"look%2:29:00::","synset_id":"look.v.03"},{"index":4,"sense_key":"look%2:39:02::","synset_id":"search.v.02"},{"index":5,"sense_key":"look%2:42:00::","synset_id":"front.v.01"},{"index":6,"sense_key":"look%2:41:13::","synset_id":"attend.v.02"},{"index":7,"sense_key":"look%2:32:00::","synset_id":"look.v.07"},{"index":8,"sense_key":"look%2:31:00::","synset_id":"expect.v.03"},{"index":9,"sense_key":"look%2:42:02::","synset_id":"look.v.09"},{"index":10,"sense_key":"look%2:31:02::","synset_id":"count.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"look%2:39:00::","call_ids":["senseval3.d000.s053.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"look%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":975,"cached_input_tokens":0,"output_tokens":49,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00244,"input_uncached_usd":0.00195,"input_cached_usd":0.0,"output_usd":0.0004900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7607357919914648}
{"item_id":"senseval3.d000.s053.t001","gold_sense_keys":["seem%2:39:00::"],"candidates":[{"index":1,"sense_key":"seem%2:39:00::","synset_id":"look.v.02"},{"index":2,"sense_key":"seem%2:39:01::","synset_id":"appear.v.04"},{"index":3,"sense_key":"seem%2:42:00::","synset_id":"seem.v.03"},{"index":4,"sense_key":"seem%2:39:02::","synset_id":"seem.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"seem%2:39:00::","call_ids":["senseval3.d000.s053.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"seem%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":621,"cached_input_tokens":0,"output_tokens":52,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0017620000000000001,"input_uncached_usd":0.001242,"input_cached_usd":0.0,"output_usd":0.0005200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.403411583043635}
{"item_id":"senseval3.d000.s054.t000","gold_sense_keys":["fine%4:02:00::"],"candidates":[{"index":1,"sense_key":"fine%4:02:00::","synset_id":"very_well.r.02"},{"index":2,"sense_key":"fine%4:02:02::","synset_id":"finely.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"fine%4:02:00::","call_ids":["senseval3.d000.s054.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"fine%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":465,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00105,"input_uncached_usd":0.0009299999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0115934999193996}
{"item_id":"senseval3.d000.s055.t000","gold_sense_keys":["give%2:40:07::"],"candidates":[{"index":1,"sense_key":"give%2:40:03::","synset_id":"give.v.01"},{"index":2,"sense_key":"give%2:40:05::","synset_id":"yield.v.01"},{"index":3,"sense_key":"give%2:40:00::","synset_id":"give.v.03"},{"index":4,"sense_key":"give%2:40:07::","synset_id":"give.v.04"},{"index":5,"sense_key":"give%2:32:01::","synset_id":"give.v.05"},{"index":6,"sense_key":"give%2:36:02::","synset_id":"hold.v.03"},{"index":7,"sense_key":"give%2:32:00::","synset_id":"give.v.07"},{"index":8,"sense_key":"give%2:40:01::","synset_id":"give.v.08"},{"index":9,"sense_key":"give%2:36:03::","synset_id":"give.v.09"},{"index":10,"sense_key":"give%2:31:02::","synset_id":"give.v.10"},{"index":11,"sense_key":"give%2:36:00::","synset_id":"render.v.04"},{"index":12,"sense_key":"give%2:40:09::","synset_id":"impart.v.01"},{"index":13,"sense_key":"give%2:36:01::","synset_id":"establish.v.05"},{"index":14,"sense_key":"give%2:40:02::","synset_id":"give.v.14"},{"index":15,"sense_key":"give%2:41:00::","synset_id":"give.v.15"},{"index":16,"sense_key":"give%2:40:10::","synset_id":"sacrifice.v.01"},{"index":17,"sense_key":"give%2:40:06::","synset_id":"pass.v.05"},{"index":18,"sense_key":"give%2:32:03::","synset_id":"give.v.18"},{"index":19,"sense_key":"give%2:40:13::","synset_id":"give.v.19"},{"index":20,"sense_key":"give%2:40:14::","synset_id":"give.v.20"},{"index":21,"sense_key":"give%2:32:02::","synset_id":"give.v.21"},{"index":22,"sense_key":"give%2:40:11::","synset_id":"grant.v.05"},{"index":23,"sense_key":"give%2:38:00::","synset_id":"move_over.v.01"},{"index":24,"sense_key":"give%2:34:00::","synset_id":"feed.v.02"},{"index":25,"sense_key":"give%2:40:08::","synset_id":"contribute.v.02"},{"index":26,"sense_key":"give%2:38:13::","synset_id":"collapse.v.01"},{"index":27,"sense_key":"give%2:31:00::","synset_id":"give.v.27"},{"index":28,"sense_key":"give%2:41:10::","synset_id":"give.v.28"},{"index":29,"sense_key":"give%2:40:12::","synset_id":"give.v.29"},{"index":30,"sense_key":"give%2:40:15::","synset_id":"afford.v.04"},{"index":31,"sense_key":"give%2:36:13::","synset_id":"give.v.31"},{"index":32,"sense_key":"give%2:36:12::","synset_id":"give.v.32"},{"index":33,"sense_key":"give%2:35:00::","synset_id":"give.v.33"},{"index":34,"sense_key":"give%2:34:10::","synset_id":"give.v.34"},{"index":35,"sense_key":"give%2:32:11::","synset_id":"give.v.35"},{"index":36,"sense_key":"give%2:32:08::","synset_id":"give.v.36"},{"index":37,"sense_key":"give%2:32:15::","synset_id":"give.v.37"},{"index":38,"sense_key":"give%2:32:14::","synset_id":"give.v.38"},{"index":39,"sense_key":"give%2:32:09::","synset_id":"give.v.39"},{"index":40,"sense_key":"give%2:32:07::","synset_id":"give.v.40"},{"index":41,"sense_key":"give%2:32:10::","synset_id":"give.v.41"},{"index":42,"sense_key":"give%2:30:13::","synset_id":"give.v.42"},{"index":43,"sense_key":"give%2:29:11::","synset_id":"give.v.43"},{"index":44,"sense_key":"give%2:29:10::","synset_id":"give.v.44"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"give%2:40:07::","call_ids":["senseval3.d000.s055.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"give%2:40:07::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2714,"cached_input_tokens":0,"output_tokens":108,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.006508,"input_uncached_usd":0.005428,"input_cached_usd":0.0,"output_usd":0.00108,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4950577089330181}
{"item_id":"senseval3.d000.s055.t001","gold_sense_keys":["address%1:15:00::"],"candidates":[{"index":1,"sense_key":"address%1:10:04::","synset_id":"address.n.01"},{"index":2,"sense_key":"address%1:15:00::","synset_id":"address.n.02"},{"index":3,"sense_key":"address%1:10:00::","synset_id":"address.n.03"},{"index":4,"sense_key":"address%1:10:03::","synset_id":"address.n.04"},{"index":5,"sense_key":"address%1:10:01::","synset_id":"address.n.05"},{"index":6,"sense_key":"address%1:10:02::","synset_id":"address.n.06"},{"index":7,"sense_key":"address%1:07:01::","synset_id":"address.n.07"},{"index":8,"sense_key":"address%1:07:00::","synset_id":"savoir-faire.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"address%1:15:00::","call_ids":["senseval3.d000.s055.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"address%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":702,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001524,"input_uncached_usd":0.0014039999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8503695420222357}
{"item_id":"senseval3.d000.s056.t000","gold_sense_keys":["take%2:40:06::"],"candidates":[{"index":1,"sense_key":"take%2:41:04::","synset_id":"take.v.01"},{"index":2,"sense_key":"take%2:40:06::","synset_id":"take.v.02"},{"index":3,"sense_key":"take%2:38:09::","synset_id":"lead.v.01"},{"index":4,"sense_key":"take%2:35:00::","synset_id":"take.v.04"},{"index":5,"sense_key":"take%2:30:01::","synset_id":"assume.v.03"},{"index":6,"sense_key":"take%2:31:07::","synset_id":"take.v.06"},{"index":7,"sense_key":"take%2:38:10::","synset_id":"bring.v.01"},{"index":8,"sense_key":"take%2:40:00::","synset_id":"take.v.08"},{"index":9,"sense_key":"take%2:38:02::","synset_id":"take.v.09"},{"index":10,"sense_key":"take%2:31:01::","synset_id":"choose.v.01"},{"index":11,"sense_key":"take%2:40:05::","synset_id":"accept.v.02"},{"index":12,"sense_key":"take%2:41:00::","synset_id":"fill.v.04"},{"index":13,"sense_key":"take%2:31:03::","synset_id":"consider.v.03"},{"index":14,"sense_key":"take%2:42:00::","synset_id":"necessitate.v.01"},{"index":15,"sense_key":"take%2:39:00::","synset_id":"take.v.15"},{"index":16,"sense_key":"take%2:32:02::","synset_id":"film.v.01"},{"index":17,"sense_key":"take%2:30:00::","synset_id":"remove.v.01"},{"index":18,"sense_key":"take%2:34:00::","synset_id":"consume.v.02"},{"index":19,"sense_key":"take%2:31:09::","synset_id":"take.v.19"},{"index":20,"sense_key":"take%2:40:15::","synset_id":"take.v.20"},{"index":21,"sense_key":"take%2:40:01::","synset_id":"take.v.21"},{"index":22,"sense_key":"take%2:38:00::","synset_id":"assume.v.05"},{"index":23,"sense_key":"take%2:40:07::","synset_id":"accept.v.05"},{"index":24,"sense_key":"take%2:30:08::","synset_id":"take.v.24"},{"index":25,"sense_key":"take%2:31:02::","synset_id":"learn.v.04"},{"index":26,"sense_key":"take%2:32:00::","synset_id":"claim.v.05"},{"index":27,"sense_key":"take%2:38:05::","synset_id":"take.v.27"},{"index":28,"sense_key":"take%2:33:09::","synset_id":"aim.v.01"},{"index":29,"sense_key":"take%2:30:09::","synset_id":"take.v.29"},{"index":30,"sense_key":"take%2:42:10::","synset_id":"carry.v.02"},{"index":31,"sense_key":"take%2:40:03::","synset_id":"lease.v.04"},{"index":32,"sense_key":"take%2:40:04::","synset_id":"subscribe.v.05"},{"index":33,"sense_key":"take%2:40:02::","synset_id":"take.v.33"},{"index":34,"sense_key":"take%2:40:08::","synset_id":"take.v.34"},{"index":35,"sense_key":"take%2:35:14::","synset_id":"take.v.35"},{"index":36,"sense_key":"take%2:32:09::","synset_id":"claim.v.04"},{"index":37,"sense_key":"take%2:42:03::","synset_id":"accept.v.08"},{"index":38,"sense_key":"take%2:42:15::","synset_id":"contain.v.05"},{"index":39,"sense_key":"take%2:41:13::","synset_id":"take.v.39"},{"index":40,"sense_key":"take%2:38:11::","synset_id":"drive.v.16"},{"index":41,"sense_key":"take%2:33:08::","synset_id":"take.v.41"},{"index":42,"sense_key":"take%2:29:08::","synset_id":"contract.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"take%2:40:06::","call_ids":["senseval3.d000.s056.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"take%2:40:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2913,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.006425999999999999,"input_uncached_usd":0.005826,"input_cached_usd":0.0,"output_usd":0.0006000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0087777079315856}
{"item_id":"senseval3.d000.s056.t001","gold_sense_keys":["time%1:03:00::","time%1:28:05::"],"candidates":[{"index":1,"sense_key":"time%1:11:00::","synset_id":"time.n.01"},{"index":2,"sense_key":"time%1:28:05::","synset_id":"time.n.02"},{"index":3,"sense_key":"time%1:28:00::","synset_id":"time.n.03"},{"index":4,"sense_key":"time%1:28:06::","synset_id":"time.n.04"},{"index":5,"sense_key":"time%1:03:00::","synset_id":"time.n.05"},{"index":6,"sense_key":"time%1:11:01::","synset_id":"time.n.06"},{"index":7,"sense_key":"time%1:28:03::","synset_id":"clock_time.n.01"},{"index":8,"sense_key":"time%1:28:04::","synset_id":"fourth_dimension.n.01"},{"index":9,"sense_key":"time%1:07:01::","synset_id":"meter.n.04"},{"index":10,"sense_key":"time%1:28:01::","synset_id":"prison_term.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"time%1:28:05::","call_ids":["senseval3.d000.s056.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"time%1:28:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":882,"cached_input_tokens":0,"output_tokens":116,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002924,"input_uncached_usd":0.001764,"input_cached_usd":0.0,"output_usd":0.00116,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8744943749625236}
{"item_id":"senseval3.d000.s057.t000","gold_sense_keys":["want%2:37:00::"],"candidates":[{"index":1,"sense_key":"want%2:37:00::","synset_id":"desire.v.01"},{"index":2,"sense_key":"want%2:34:00::","synset_id":"want.v.02"},{"index":3,"sense_key":"want%2:35:00::","synset_id":"want.v.03"},{"index":4,"sense_key":"want%2:31:01::","synset_id":"want.v.04"},{"index":5,"sense_key":"want%2:42:01::","synset_id":"want.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"want%2:37:00::","call_ids":["senseval3.d000.s057.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"want%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":573,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001266,"input_uncached_usd":0.0011459999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2424087499966845}
{"item_id":"senseval3.d000.s057.t001","gold_sense_keys":["study%2:31:02::"],"candidates":[{"index":1,"sense_key":"study%2:31:02::","synset_id":"analyze.v.01"},{"index":2,"sense_key":"study%2:31:03::","synset_id":"study.v.02"},{"index":3,"sense_key":"study%2:39:00::","synset_id":"study.v.03"},{"index":4,"sense_key":"study%2:31:00::","synset_id":"learn.v.04"},{"index":5,"sense_key":"study%2:31:01::","synset_id":"study.v.05"},{"index":6,"sense_key":"study%2:31:04::","synset_id":"study.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"study%2:31:02::","call_ids":["senseval3.d000.s057.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"study%2:31:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":687,"cached_input_tokens":0,"output_tokens":114,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002514,"input_uncached_usd":0.001374,"input_cached_usd":0.0,"output_usd":0.0011400000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7406056249747053}
{"item_id":"senseval3.d000.s057.t003","gold_sense_keys":["habit%1:09:00::"],"candidates":[{"index":1,"sense_key":"habit%1:09:00::","synset_id":"habit.n.01"},{"index":2,"sense_key":"habit%1:04:02::","synset_id":"habit.n.02"},{"index":3,"sense_key":"habit%1:06:00::","synset_id":"habit.n.03"},{"index":4,"sense_key":"habit%1:22:00::","synset_id":"habit.n.04"},{"index":5,"sense_key":"habit%1:06:01::","synset_id":"habit.n.05"},{"index":6,"sense_key":"habit%1:04:01::","synset_id":"substance_abuse.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"habit%1:04:02::","call_ids":["senseval3.d000.s057.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"habit%1:04:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":662,"cached_input_tokens":0,"output_tokens":219,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0035139999999999998,"input_uncached_usd":0.0013239999999999999,"input_cached_usd":0.0,"output_usd":0.00219,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.5036420830292627}
{"item_id":"senseval3.d000.s057.t004","gold_sense_keys":["movement%1:04:04::"],"candidates":[{"index":1,"sense_key":"movement%1:04:00::","synset_id":"motion.n.03"},{"index":2,"sense_key":"movement%1:04:04::","synset_id":"motion.n.06"},{"index":3,"sense_key":"movement%1:11:00::","synset_id":"movement.n.03"},{"index":4,"sense_key":"movement%1:14:00::","synset_id":"movement.n.04"},{"index":5,"sense_key":"movement%1:10:00::","synset_id":"movement.n.05"},{"index":6,"sense_key":"movement%1:04:03::","synset_id":"campaign.n.02"},{"index":7,"sense_key":"movement%1:19:00::","synset_id":"apparent_motion.n.01"},{"index":8,"sense_key":"movement%1:22:00::","synset_id":"bowel_movement.n.01"},{"index":9,"sense_key":"movement%1:09:00::","synset_id":"drift.n.05"},{"index":10,"sense_key":"movement%1:06:00::","synset_id":"movement.n.10"},{"index":11,"sense_key":"movement%1:04:02::","synset_id":"movement.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"movement%1:04:04::","call_ids":["senseval3.d000.s057.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"movement%1:04:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1090,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00281,"input_uncached_usd":0.00218,"input_cached_usd":0.0,"output_usd":0.00063,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.530575415934436}
{"item_id":"senseval3.d000.s058.t000","gold_sense_keys":["see%2:31:01::"],"candidates":[{"index":1,"sense_key":"see%2:39:00::","synset_id":"see.v.01"},{"index":2,"sense_key":"see%2:31:01::","synset_id":"understand.v.02"},{"index":3,"sense_key":"see%2:39:02::","synset_id":"witness.v.02"},{"index":4,"sense_key":"see%2:36:00::","synset_id":"visualize.v.01"},{"index":5,"sense_key":"see%2:31:00::","synset_id":"see.v.05"},{"index":6,"sense_key":"see%2:31:02::","synset_id":"learn.v.02"},{"index":7,"sense_key":"see%2:39:03::","synset_id":"watch.v.03"},{"index":8,"sense_key":"see%2:38:00::","synset_id":"meet.v.01"},{"index":9,"sense_key":"see%2:32:00::","synset_id":"determine.v.08"},{"index":10,"sense_key":"see%2:31:03::","synset_id":"see.v.10"},{"index":11,"sense_key":"see%2:41:11::","synset_id":"see.v.11"},{"index":12,"sense_key":"see%2:41:12::","synset_id":"see.v.12"},{"index":13,"sense_key":"see%2:41:10::","synset_id":"visit.v.01"},{"index":14,"sense_key":"see%2:41:14::","synset_id":"attend.v.02"},{"index":15,"sense_key":"see%2:35:09::","synset_id":"see.v.15"},{"index":16,"sense_key":"see%2:41:00::","synset_id":"go_steady.v.01"},{"index":17,"sense_key":"see%2:39:05::","synset_id":"see.v.17"},{"index":18,"sense_key":"see%2:31:08::","synset_id":"see.v.18"},{"index":19,"sense_key":"see%2:39:11::","synset_id":"see.v.19"},{"index":20,"sense_key":"see%2:39:13::","synset_id":"examine.v.02"},{"index":21,"sense_key":"see%2:39:12::","synset_id":"experience.v.01"},{"index":22,"sense_key":"see%2:38:03::","synset_id":"see.v.22"},{"index":23,"sense_key":"see%2:33:00::","synset_id":"see.v.23"},{"index":24,"sense_key":"see%2:31:13::","synset_id":"interpret.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"see%2:31:01::","call_ids":["senseval3.d000.s058.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"see%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1864,"cached_input_tokens":0,"output_tokens":79,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004518,"input_uncached_usd":0.003728,"input_cached_usd":0.0,"output_usd":0.00079,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3697994169779122}
{"item_id":"senseval3.d000.s058.t001","gold_sense_keys":["make%2:30:00::"],"candidates":[{"index":1,"sense_key":"make%2:41:00::","synset_id":"make.v.01"},{"index":2,"sense_key":"make%2:30:00::","synset_id":"make.v.02"},{"index":3,"sense_key":"make%2:36:00::","synset_id":"make.v.03"},{"index":4,"sense_key":"make%2:32:00::","synset_id":"induce.v.02"},{"index":5,"sense_key":"make%2:36:08::","synset_id":"cause.v.01"},{"index":6,"sense_key":"make%2:36:01::","synset_id":"produce.v.02"},{"index":7,"sense_key":"make%2:31:00::","synset_id":"draw.v.04"},{"index":8,"sense_key":"make%2:36:15::","synset_id":"make.v.08"},{"index":9,"sense_key":"make%2:36:09::","synset_id":"create.v.05"},{"index":10,"sense_key":"make%2:40:01::","synset_id":"gain.v.08"},{"index":11,"sense_key":"make%2:36:05::","synset_id":"do.v.08"},{"index":12,"sense_key":"make%2:42:05::","synset_id":"form.v.02"},{"index":13,"sense_key":"make%2:38:02::","synset_id":"reach.v.07"},{"index":14,"sense_key":"make%2:42:00::","synset_id":"make.v.14"},{"index":15,"sense_key":"make%2:36:11::","synset_id":"make.v.15"},{"index":16,"sense_key":"make%2:36:12::","synset_id":"make.v.16"},{"index":17,"sense_key":"make%2:36:13::","synset_id":"construct.v.01"},{"index":18,"sense_key":"make%2:30:15::","synset_id":"make.v.18"},{"index":19,"sense_key":"make%2:29:08::","synset_id":"make.v.19"},{"index":20,"sense_key":"make%2:41:03::","synset_id":"name.v.03"},{"index":21,"sense_key":"make%2:40:02::","synset_id":"have.v.17"},{"index":22,"sense_key":"make%2:38:00::","synset_id":"reach.v.01"},{"index":23,"sense_key":"make%2:31:13::","synset_id":"lay_down.v.01"},{"index":24,"sense_key":"make%2:41:13::","synset_id":"make.v.24"},{"index":25,"sense_key":"make%2:41:01::","synset_id":"make.v.25"},{"index":26,"sense_key":"make%2:36:04::","synset_id":"hold.v.03"},{"index":27,"sense_key":"make%2:30:02::","synset_id":"make.v.27"},{"index":28,"sense_key":"make%2:38:05::","synset_id":"take.v.27"},{"index":29,"sense_key":"make%2:29:00::","synset_id":"stool.v.04"},{"index":30,"sense_key":"make%2:42:09::","synset_id":"make.v.30"},{"index":31,"sense_key":"make%2:42:13::","synset_id":"make.v.31"},{"index":32,"sense_key":"make%2:42:12::","synset_id":"make.v.32"},{"index":33,"sense_key":"make%2:42:11::","synset_id":"make.v.33"},{"index":34,"sense_key":"make%2:42:15::","synset_id":"make.v.34"},{"index":35,"sense_key":"make%2:39:12::","synset_id":"make.v.35"},{"index":36,"sense_key":"make%2:38:10::","synset_id":"make.v.36"},{"index":37,"sense_key":"make%2:38:15::","synset_id":"make.v.37"},{"index":38,"sense_key":"make%2:36:10::","synset_id":"make.v.38"},{"index":39,"sense_key":"make%2:36:07::","synset_id":"cook.v.02"},{"index":40,"sense_key":"make%2:35:13::","synset_id":"seduce.v.01"},{"index":41,"sense_key":"make%2:32:14::","synset_id":"make.v.41"},{"index":42,"sense_key":"make%2:32:09::","synset_id":"make.v.42"},{"index":43,"sense_key":"make%2:31:10::","synset_id":"make.v.43"},{"index":44,"sense_key":"make%2:31:12::","synset_id":"make.v.44"},{"index":45,"sense_key":"make%2:30:10::","synset_id":"make.v.45"},{"index":46,"sense_key":"make%2:30:14::","synset_id":"make.v.46"},{"index":47,"sense_key":"make%2:30:12::","synset_id":"make.v.47"},{"index":48,"sense_key":"make%2:29:06::","synset_id":"make.v.48"},{"index":49,"sense_key":"make%2:29:01::","synset_id":"make.v.49"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"make%2:30:00::","call_ids":["senseval3.d000.s058.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"make%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2976,"cached_input_tokens":0,"output_tokens":239,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.008342,"input_uncached_usd":0.005952,"input_cached_usd":0.0,"output_usd":0.00239,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.5636454169871286}
{"item_id":"senseval3.d000.s059.t000","gold_sense_keys":["suicide%1:04:00::"],"candidates":[{"index":1,"sense_key":"suicide%1:04:00::","synset_id":"suicide.n.01"},{"index":2,"sense_key":"suicide%1:18:00::","synset_id":"suicide.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"suicide%1:04:00::","call_ids":["senseval3.d000.s059.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"suicide%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":422,"cached_input_tokens":0,"output_tokens":83,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0016740000000000001,"input_uncached_usd":0.0008439999999999999,"input_cached_usd":0.0,"output_usd":0.0008300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5364242090145126}
{"item_id":"senseval3.d000.s059.t001","gold_sense_keys":["look%2:39:01::"],"candidates":[{"index":1,"sense_key":"look%2:39:00::","synset_id":"look.v.01"},{"index":2,"sense_key":"look%2:39:01::","synset_id":"look.v.02"},{"index":3,"sense_key":"look%2:29:00::","synset_id":"look.v.03"},{"index":4,"sense_key":"look%2:39:02::","synset_id":"search.v.02"},{"index":5,"sense_key":"look%2:42:00::","synset_id":"front.v.01"},{"index":6,"sense_key":"look%2:41:13::","synset_id":"attend.v.02"},{"index":7,"sense_key":"look%2:32:00::","synset_id":"look.v.07"},{"index":8,"sense_key":"look%2:31:00::","synset_id":"expect.v.03"},{"index":9,"sense_key":"look%2:42:02::","synset_id":"look.v.09"},{"index":10,"sense_key":"look%2:31:02::","synset_id":"count.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"look%2:39:01::","call_ids":["senseval3.d000.s059.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"look%2:39:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":924,"cached_input_tokens":0,"output_tokens":82,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002668,"input_uncached_usd":0.0018479999999999998,"input_cached_usd":0.0,"output_usd":0.0008200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.921237042057328}
{"item_id":"senseval3.d000.s060.t000","gold_sense_keys":["trouble%1:04:02::","trouble%1:09:00::","trouble%1:11:00::"],"candidates":[{"index":1,"sense_key":"trouble%1:09:00::","synset_id":"trouble.n.01"},{"index":2,"sense_key":"trouble%1:11:01::","synset_id":"fuss.n.02"},{"index":3,"sense_key":"trouble%1:11:00::","synset_id":"trouble.n.03"},{"index":4,"sense_key":"trouble%1:04:02::","synset_id":"trouble.n.04"},{"index":5,"sense_key":"trouble%1:12:03::","synset_id":"worry.n.02"},{"index":6,"sense_key":"trouble%1:26:02::","synset_id":"trouble.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"trouble%1:09:00::","call_ids":["senseval3.d000.s060.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"trouble%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":678,"cached_input_tokens":0,"output_tokens":249,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003846,"input_uncached_usd":0.001356,"input_cached_usd":0.0,"output_usd":0.00249,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.60349212505389}
{"item_id":"senseval3.d000.s061.t001","gold_sense_keys":["fly%2:38:02::"],"candidates":[{"index":1,"sense_key":"fly%2:38:00::","synset_id":"fly.v.01"},{"index":2,"sense_key":"fly%2:38:02::","synset_id":"fly.v.02"},{"index":3,"sense_key":"fly%2:38:01::","synset_id":"fly.v.03"},{"index":4,"sense_key":"fly%2:35:00::","synset_id":"fly.v.04"},{"index":5,"sense_key":"fly%2:38:07::","synset_id":"fly.v.05"},{"index":6,"sense_key":"fly%2:38:08::","synset_id":"fly.v.06"},{"index":7,"sense_key":"fly%2:30:01::","synset_id":"fly.v.07"},{"index":8,"sense_key":"fly%2:38:06::","synset_id":"fly.v.08"},{"index":9,"sense_key":"fly%2:38:05::","synset_id":"fly.v.09"},{"index":10,"sense_key":"fly%2:39:00::","synset_id":"fly.v.10"},{"index":11,"sense_key":"fly%2:38:03::","synset_id":"flee.v.01"},{"index":12,"sense_key":"fly%2:38:04::","synset_id":"fly.v.12"},{"index":13,"sense_key":"fly%2:35:01::","synset_id":"fly.v.13"},{"index":14,"sense_key":"fly%2:30:00::","synset_id":"vanish.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"fly%2:38:02::","call_ids":["senseval3.d000.s061.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"fly%2:38:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1040,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0026699999999999996,"input_uncached_usd":0.00208,"input_cached_usd":0.0,"output_usd":0.00059,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.187330957967788}
{"item_id":"senseval3.d000.s063.t000","gold_sense_keys":["smile%2:29:00::"],"candidates":[{"index":1,"sense_key":"smile%2:29:00::","synset_id":"smile.v.01"},{"index":2,"sense_key":"smile%2:32:00::","synset_id":"smile.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"smile%2:29:00::","call_ids":["senseval3.d000.s063.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"smile%2:29:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":401,"cached_input_tokens":0,"output_tokens":76,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001562,"input_uncached_usd":0.000802,"input_cached_usd":0.0,"output_usd":0.00076,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4434232920175418}
{"item_id":"senseval3.d000.s064.t001","gold_sense_keys":["say%2:32:15::"],"candidates":[{"index":1,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":6,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":9,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":10,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":11,"sense_key":"say%2:32:09::","synset_id":"say.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"say%2:32:00::","call_ids":["senseval3.d000.s064.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"say%2:32:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":991,"cached_input_tokens":0,"output_tokens":131,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003292,"input_uncached_usd":0.001982,"input_cached_usd":0.0,"output_usd":0.0013100000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.4474602499976754}
{"item_id":"senseval3.d000.s065.t000","gold_sense_keys":["minute%1:28:00::"],"candidates":[{"index":1,"sense_key":"minute%1:28:00::","synset_id":"minute.n.01"},{"index":2,"sense_key":"minute%1:28:01::","synset_id":"moment.n.02"},{"index":3,"sense_key":"minute%1:28:02::","synset_id":"moment.n.01"},{"index":4,"sense_key":"minute%1:23:00::","synset_id":"minute.n.04"},{"index":5,"sense_key":"minute%1:10:00::","synset_id":"minute.n.05"},{"index":6,"sense_key":"minute%1:07:00::","synset_id":"hour.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"minute%1:28:00::","call_ids":["senseval3.d000.s065.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"minute%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":717,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002094,"input_uncached_usd":0.001434,"input_cached_usd":0.0,"output_usd":0.0006600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.770316957961768}
{"item_id":"senseval3.d000.s065.t001","gold_sense_keys":["make%2:36:12::"],"candidates":[{"index":1,"sense_key":"make%2:41:00::","synset_id":"make.v.01"},{"index":2,"sense_key":"make%2:30:00::","synset_id":"make.v.02"},{"index":3,"sense_key":"make%2:36:00::","synset_id":"make.v.03"},{"index":4,"sense_key":"make%2:32:00::","synset_id":"induce.v.02"},{"index":5,"sense_key":"make%2:36:08::","synset_id":"cause.v.01"},{"index":6,"sense_key":"make%2:36:01::","synset_id":"produce.v.02"},{"index":7,"sense_key":"make%2:31:00::","synset_id":"draw.v.04"},{"index":8,"sense_key":"make%2:36:15::","synset_id":"make.v.08"},{"index":9,"sense_key":"make%2:36:09::","synset_id":"create.v.05"},{"index":10,"sense_key":"make%2:40:01::","synset_id":"gain.v.08"},{"index":11,"sense_key":"make%2:36:05::","synset_id":"do.v.08"},{"index":12,"sense_key":"make%2:42:05::","synset_id":"form.v.02"},{"index":13,"sense_key":"make%2:38:02::","synset_id":"reach.v.07"},{"index":14,"sense_key":"make%2:42:00::","synset_id":"make.v.14"},{"index":15,"sense_key":"make%2:36:11::","synset_id":"make.v.15"},{"index":16,"sense_key":"make%2:36:12::","synset_id":"make.v.16"},{"index":17,"sense_key":"make%2:36:13::","synset_id":"construct.v.01"},{"index":18,"sense_key":"make%2:30:15::","synset_id":"make.v.18"},{"index":19,"sense_key":"make%2:29:08::","synset_id":"make.v.19"},{"index":20,"sense_key":"make%2:41:03::","synset_id":"name.v.03"},{"index":21,"sense_key":"make%2:40:02::","synset_id":"have.v.17"},{"index":22,"sense_key":"make%2:38:00::","synset_id":"reach.v.01"},{"index":23,"sense_key":"make%2:31:13::","synset_id":"lay_down.v.01"},{"index":24,"sense_key":"make%2:41:13::","synset_id":"make.v.24"},{"index":25,"sense_key":"make%2:41:01::","synset_id":"make.v.25"},{"index":26,"sense_key":"make%2:36:04::","synset_id":"hold.v.03"},{"index":27,"sense_key":"make%2:30:02::","synset_id":"make.v.27"},{"index":28,"sense_key":"make%2:38:05::","synset_id":"take.v.27"},{"index":29,"sense_key":"make%2:29:00::","synset_id":"stool.v.04"},{"index":30,"sense_key":"make%2:42:09::","synset_id":"make.v.30"},{"index":31,"sense_key":"make%2:42:13::","synset_id":"make.v.31"},{"index":32,"sense_key":"make%2:42:12::","synset_id":"make.v.32"},{"index":33,"sense_key":"make%2:42:11::","synset_id":"make.v.33"},{"index":34,"sense_key":"make%2:42:15::","synset_id":"make.v.34"},{"index":35,"sense_key":"make%2:39:12::","synset_id":"make.v.35"},{"index":36,"sense_key":"make%2:38:10::","synset_id":"make.v.36"},{"index":37,"sense_key":"make%2:38:15::","synset_id":"make.v.37"},{"index":38,"sense_key":"make%2:36:10::","synset_id":"make.v.38"},{"index":39,"sense_key":"make%2:36:07::","synset_id":"cook.v.02"},{"index":40,"sense_key":"make%2:35:13::","synset_id":"seduce.v.01"},{"index":41,"sense_key":"make%2:32:14::","synset_id":"make.v.41"},{"index":42,"sense_key":"make%2:32:09::","synset_id":"make.v.42"},{"index":43,"sense_key":"make%2:31:10::","synset_id":"make.v.43"},{"index":44,"sense_key":"make%2:31:12::","synset_id":"make.v.44"},{"index":45,"sense_key":"make%2:30:10::","synset_id":"make.v.45"},{"index":46,"sense_key":"make%2:30:14::","synset_id":"make.v.46"},{"index":47,"sense_key":"make%2:30:12::","synset_id":"make.v.47"},{"index":48,"sense_key":"make%2:29:06::","synset_id":"make.v.48"},{"index":49,"sense_key":"make%2:29:01::","synset_id":"make.v.49"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":16,"chosen_sense_key":"make%2:36:12::","call_ids":["senseval3.d000.s065.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":16,"predicted_sense_key":"make%2:36:12::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":3029,"cached_input_tokens":0,"output_tokens":69,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0067480000000000005,"input_uncached_usd":0.006058,"input_cached_usd":0.0,"output_usd":0.0006900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.434662500047125}
{"item_id":"senseval3.d000.s065.t003","gold_sense_keys":["stranger%1:18:01::"],"candidates":[{"index":1,"sense_key":"stranger%1:18:00::","synset_id":"stranger.n.01"},{"index":2,"sense_key":"stranger%1:18:01::","synset_id":"stranger.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"stranger%1:18:01::","call_ids":["senseval3.d000.s065.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"stranger%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":453,"cached_input_tokens":0,"output_tokens":89,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001796,"input_uncached_usd":0.000906,"input_cached_usd":0.0,"output_usd":0.0008900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.409871666925028}
{"item_id":"senseval3.d000.s065.t004","gold_sense_keys":["stand_up%2:38:00::"],"candidates":[{"index":1,"sense_key":"stand_up%2:38:00::","synset_id":"arise.v.03"},{"index":2,"sense_key":"stand_up%2:33:00::","synset_id":"stand_up.v.02"},{"index":3,"sense_key":"stand_up%2:35:03::","synset_id":"stand.v.10"},{"index":4,"sense_key":"stand_up%2:35:00::","synset_id":"stand.v.01"},{"index":5,"sense_key":"stand_up%2:32:00::","synset_id":"stand_up.v.05"},{"index":6,"sense_key":"stand_up%2:42:00::","synset_id":"stand_up.v.06"},{"index":7,"sense_key":"stand_up%2:38:06::","synset_id":"bristle.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"stand_up%2:38:00::","call_ids":["senseval3.d000.s065.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"stand_up%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":809,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001738,"input_uncached_usd":0.0016179999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9961189168971032}
{"item_id":"senseval3.d000.s065.t005","gold_sense_keys":["nod%2:32:01::"],"candidates":[{"index":1,"sense_key":"nod%2:32:01::","synset_id":"nod.v.01"},{"index":2,"sense_key":"nod%2:32:00::","synset_id":"nod.v.02"},{"index":3,"sense_key":"nod%2:29:03::","synset_id":"nod.v.03"},{"index":4,"sense_key":"nod%2:38:00::","synset_id":"nod.v.04"},{"index":5,"sense_key":"nod%2:29:02::","synset_id":"nod.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"nod%2:32:01::","call_ids":["senseval3.d000.s065.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"nod%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":624,"cached_input_tokens":0,"output_tokens":84,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002088,"input_uncached_usd":0.001248,"input_cached_usd":0.0,"output_usd":0.00084,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2708554579876363}
{"item_id":"senseval3.d000.s065.t006","gold_sense_keys":["farewell%1:10:00::"],"candidates":[{"index":1,"sense_key":"farewell%1:10:00::","synset_id":"farewell.n.01"},{"index":2,"sense_key":"farewell%1:04:00::","synset_id":"farewell.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"farewell%1:10:00::","call_ids":["senseval3.d000.s065.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"farewell%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":483,"cached_input_tokens":0,"output_tokens":104,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002006,"input_uncached_usd":0.000966,"input_cached_usd":0.0,"output_usd":0.0010400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1403370409971103}
{"item_id":"senseval3.d000.s066.t000","gold_sense_keys":["watch%2:39:00::","watch%2:39:02::","watch%2:41:00::"],"candidates":[{"index":1,"sense_key":"watch%2:39:00::","synset_id":"watch.v.01"},{"index":2,"sense_key":"watch%2:41:00::","synset_id":"watch.v.02"},{"index":3,"sense_key":"watch%2:39:05::","synset_id":"watch.v.03"},{"index":4,"sense_key":"watch%2:39:02::","synset_id":"watch.v.04"},{"index":5,"sense_key":"watch%2:39:01::","synset_id":"watch.v.05"},{"index":6,"sense_key":"watch%2:39:03::","synset_id":"watch.v.06"},{"index":7,"sense_key":"watch%2:32:09::","synset_id":"determine.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"watch%2:41:00::","call_ids":["senseval3.d000.s066.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"watch%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":830,"cached_input_tokens":0,"output_tokens":192,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0035800000000000003,"input_uncached_usd":0.00166,"input_cached_usd":0.0,"output_usd":0.0019200000000000003,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0899118330562487}
{"item_id":"senseval3.d000.s066.t001","gold_sense_keys":["small%3:00:00::"],"candidates":[{"index":1,"sense_key":"small%3:00:00::","synset_id":"small.a.01"},{"index":2,"sense_key":"small%5:00:00:limited:00","synset_id":"minor.s.03"},{"index":3,"sense_key":"small%5:00:00:young:00","synset_id":"little.s.01"},{"index":4,"sense_key":"small%5:00:00:little:03","synset_id":"small.s.03"},{"index":5,"sense_key":"small%5:00:00:inferior:01","synset_id":"humble.s.01"},{"index":6,"sense_key":"small%5:00:00:lowercase:00","synset_id":"little.s.04"},{"index":7,"sense_key":"small%5:00:00:soft:04","synset_id":"little.s.03"},{"index":8,"sense_key":"small%5:00:00:fine:00","synset_id":"small.s.07"},{"index":9,"sense_key":"small%5:00:00:moderate:00","synset_id":"modest.s.01"},{"index":10,"sense_key":"small%5:00:00:decreased:00","synset_id":"belittled.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"small%3:00:00::","call_ids":["senseval3.d000.s066.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"small%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":999,"cached_input_tokens":0,"output_tokens":110,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003098,"input_uncached_usd":0.0019979999999999998,"input_cached_usd":0.0,"output_usd":0.0011,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.17081041703932}
{"item_id":"senseval3.d000.s066.t002","gold_sense_keys":["wiry%5:00:00:thin:03"],"candidates":[{"index":1,"sense_key":"wiry%5:00:00:thin:03","synset_id":"stringy.s.01"},{"index":2,"sense_key":"wiry%3:01:00::","synset_id":"wiry.a.02"},{"index":3,"sense_key":"wiry%5:00:00:hairy:00","synset_id":"wiry.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"wiry%5:00:00:thin:03","call_ids":["senseval3.d000.s066.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"wiry%5:00:00:thin:03","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":520,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00116,"input_uncached_usd":0.00104,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8923305409261957}
{"item_id":"senseval3.d000.s066.t003","gold_sense_keys":["man%1:18:00::"],"candidates":[{"index":1,"sense_key":"man%1:18:00::","synset_id":"man.n.01"},{"index":2,"sense_key":"man%1:18:03::","synset_id":"serviceman.n.01"},{"index":3,"sense_key":"man%1:18:04::","synset_id":"man.n.03"},{"index":4,"sense_key":"man%1:05:01::","synset_id":"homo.n.02"},{"index":5,"sense_key":"man%1:18:08::","synset_id":"man.n.05"},{"index":6,"sense_key":"man%1:18:06::","synset_id":"man.n.06"},{"index":7,"sense_key":"man%1:18:05::","synset_id":"valet.n.01"},{"index":8,"sense_key":"man%1:18:07::","synset_id":"man.n.08"},{"index":9,"sense_key":"man%1:15:00::","synset_id":"man.n.09"},{"index":10,"sense_key":"man%1:06:00::","synset_id":"man.n.10"},{"index":11,"sense_key":"man%1:05:00::","synset_id":"world.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"man%1:18:00::","call_ids":["senseval3.d000.s066.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"man%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1132,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002384,"input_uncached_usd":0.002264,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.7301782499998808}
{"item_id":"senseval3.d000.s066.t004","gold_sense_keys":["slip%2:38:01::"],"candidates":[{"index":1,"sense_key":"slip%2:38:01::","synset_id":"steal.v.02"},{"index":2,"sense_key":"slip%2:30:04::","synset_id":"slip.v.02"},{"index":3,"sense_key":"slip%2:38:00::","synset_id":"skid.v.04"},{"index":4,"sense_key":"slip%2:30:00::","synset_id":"slip.v.04"},{"index":5,"sense_key":"slip%2:38:05::","synset_id":"slip.v.05"},{"index":6,"sense_key":"slip%2:31:00::","synset_id":"err.v.01"},{"index":7,"sense_key":"slip%2:40:10::","synset_id":"slip.v.07"},{"index":8,"sense_key":"slip%2:38:02::","synset_id":"slip.v.08"},{"index":9,"sense_key":"slip%2:38:12::","synset_id":"slip.v.09"},{"index":10,"sense_key":"slip%2:31:01::","synset_id":"slip.v.10"},{"index":11,"sense_key":"slip%2:30:03::","synset_id":"dislocate.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"slip%2:38:01::","call_ids":["senseval3.d000.s066.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"slip%2:38:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":991,"cached_input_tokens":0,"output_tokens":76,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0027419999999999996,"input_uncached_usd":0.001982,"input_cached_usd":0.0,"output_usd":0.00076,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8152965829940513}
{"item_id":"senseval3.d000.s066.t005","gold_sense_keys":["door%1:06:01::"],"candidates":[{"index":1,"sense_key":"door%1:06:00::","synset_id":"door.n.01"},{"index":2,"sense_key":"door%1:06:01::","synset_id":"doorway.n.01"},{"index":3,"sense_key":"door%1:07:00::","synset_id":"door.n.03"},{"index":4,"sense_key":"door%1:06:04::","synset_id":"door.n.04"},{"index":5,"sense_key":"door%1:06:03::","synset_id":"door.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"door%1:06:01::","call_ids":["senseval3.d000.s066.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"door%1:06:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":711,"cached_input_tokens":0,"output_tokens":99,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002412,"input_uncached_usd":0.0014219999999999999,"input_cached_usd":0.0,"output_usd":0.00099,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6464340410893783}
{"item_id":"senseval3.d000.s066.t006","gold_sense_keys":["feel%2:29:00::","feel%2:31:01::","feel%2:37:00::"],"candidates":[{"index":1,"sense_key":"feel%2:37:00::","synset_id":"feel.v.01"},{"index":2,"sense_key":"feel%2:31:00::","synset_id":"find.v.05"},{"index":3,"sense_key":"feel%2:39:00::","synset_id":"feel.v.03"},{"index":4,"sense_key":"feel%2:29:00::","synset_id":"feel.v.04"},{"index":5,"sense_key":"feel%2:31:01::","synset_id":"feel.v.05"},{"index":6,"sense_key":"feel%2:39:01::","synset_id":"feel.v.06"},{"index":7,"sense_key":"feel%2:42:00::","synset_id":"feel.v.07"},{"index":8,"sense_key":"feel%2:39:10::","synset_id":"feel.v.08"},{"index":9,"sense_key":"feel%2:35:00::","synset_id":"feel.v.09"},{"index":10,"sense_key":"feel%2:35:01::","synset_id":"palpate.v.01"},{"index":11,"sense_key":"feel%2:40:00::","synset_id":"feel.v.11"},{"index":12,"sense_key":"feel%2:39:09::","synset_id":"feel.v.12"},{"index":13,"sense_key":"feel%2:35:11::","synset_id":"feel.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"feel%2:37:00::","call_ids":["senseval3.d000.s066.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"feel%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1183,"cached_input_tokens":0,"output_tokens":124,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003606,"input_uncached_usd":0.002366,"input_cached_usd":0.0,"output_usd":0.00124,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.720872416975908}
{"item_id":"senseval3.d000.s066.t008","gold_sense_keys":["see%2:31:02::","see%2:32:00::","see%2:39:00::"],"candidates":[{"index":1,"sense_key":"see%2:39:00::","synset_id":"see.v.01"},{"index":2,"sense_key":"see%2:31:01::","synset_id":"understand.v.02"},{"index":3,"sense_key":"see%2:39:02::","synset_id":"witness.v.02"},{"index":4,"sense_key":"see%2:36:00::","synset_id":"visualize.v.01"},{"index":5,"sense_key":"see%2:31:00::","synset_id":"see.v.05"},{"index":6,"sense_key":"see%2:31:02::","synset_id":"learn.v.02"},{"index":7,"sense_key":"see%2:39:03::","synset_id":"watch.v.03"},{"index":8,"sense_key":"see%2:38:00::","synset_id":"meet.v.01"},{"index":9,"sense_key":"see%2:32:00::","synset_id":"determine.v.08"},{"index":10,"sense_key":"see%2:31:03::","synset_id":"see.v.10"},{"index":11,"sense_key":"see%2:41:11::","synset_id":"see.v.11"},{"index":12,"sense_key":"see%2:41:12::","synset_id":"see.v.12"},{"index":13,"sense_key":"see%2:41:10::","synset_id":"visit.v.01"},{"index":14,"sense_key":"see%2:41:14::","synset_id":"attend.v.02"},{"index":15,"sense_key":"see%2:35:09::","synset_id":"see.v.15"},{"index":16,"sense_key":"see%2:41:00::","synset_id":"go_steady.v.01"},{"index":17,"sense_key":"see%2:39:05::","synset_id":"see.v.17"},{"index":18,"sense_key":"see%2:31:08::","synset_id":"see.v.18"},{"index":19,"sense_key":"see%2:39:11::","synset_id":"see.v.19"},{"index":20,"sense_key":"see%2:39:13::","synset_id":"examine.v.02"},{"index":21,"sense_key":"see%2:39:12::","synset_id":"experience.v.01"},{"index":22,"sense_key":"see%2:38:03::","synset_id":"see.v.22"},{"index":23,"sense_key":"see%2:33:00::","synset_id":"see.v.23"},{"index":24,"sense_key":"see%2:31:13::","synset_id":"interpret.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"see%2:31:02::","call_ids":["senseval3.d000.s066.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"see%2:31:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1946,"cached_input_tokens":0,"output_tokens":138,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.005272,"input_uncached_usd":0.0038919999999999996,"input_cached_usd":0.0,"output_usd":0.0013800000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.908730374998413}
{"item_id":"senseval3.d000.s066.t009","gold_sense_keys":["seem%2:39:00::"],"candidates":[{"index":1,"sense_key":"seem%2:39:00::","synset_id":"look.v.02"},{"index":2,"sense_key":"seem%2:39:01::","synset_id":"appear.v.04"},{"index":3,"sense_key":"seem%2:42:00::","synset_id":"seem.v.03"},{"index":4,"sense_key":"seem%2:39:02::","synset_id":"seem.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"seem%2:39:00::","call_ids":["senseval3.d000.s066.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"seem%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":653,"cached_input_tokens":0,"output_tokens":186,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003166,"input_uncached_usd":0.0013059999999999999,"input_cached_usd":0.0,"output_usd":0.00186,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2251025839941576}
{"item_id":"senseval3.d000.s066.t010","gold_sense_keys":["notice%2:39:00::"],"candidates":[{"index":1,"sense_key":"notice%2:39:04::","synset_id":"detect.v.01"},{"index":2,"sense_key":"notice%2:39:00::","synset_id":"notice.v.02"},{"index":3,"sense_key":"notice%2:32:00::","synset_id":"comment.v.01"},{"index":4,"sense_key":"notice%2:32:01::","synset_id":"notice.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"notice%2:39:00::","call_ids":["senseval3.d000.s066.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"notice%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":685,"cached_input_tokens":0,"output_tokens":119,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0025599999999999998,"input_uncached_usd":0.00137,"input_cached_usd":0.0,"output_usd":0.00119,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.565494375070557}
{"item_id":"senseval3.d000.s066.t011","gold_sense_keys":["departure%1:04:00::"],"candidates":[{"index":1,"sense_key":"departure%1:04:00::","synset_id":"departure.n.01"},{"index":2,"sense_key":"departure%1:11:01::","synset_id":"deviation.n.01"},{"index":3,"sense_key":"departure%1:11:00::","synset_id":"passing.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"departure%1:04:00::","call_ids":["senseval3.d000.s066.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"departure%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":565,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00125,"input_uncached_usd":0.00113,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.6885676248930395}
{"item_id":"senseval3.d000.s067.t000","gold_sense_keys":["decide%2:31:00::"],"candidates":[{"index":1,"sense_key":"decide%2:31:00::","synset_id":"decide.v.01"},{"index":2,"sense_key":"decide%2:31:01::","synset_id":"decide.v.02"},{"index":3,"sense_key":"decide%2:31:02::","synset_id":"decide.v.03"},{"index":4,"sense_key":"decide%2:31:06::","synset_id":"decide.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"decide%2:31:00::","call_ids":["senseval3.d000.s067.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"decide%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":663,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001446,"input_uncached_usd":0.001326,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2188167079584673}
{"item_id":"senseval3.d000.s067.t001","gold_sense_keys":["stay%2:38:00::","stay%2:42:01::","stay%2:42:02::"],"candidates":[{"index":1,"sense_key":"stay%2:30:00::","synset_id":"stay.v.01"},{"index":2,"sense_key":"stay%2:38:00::","synset_id":"stay.v.02"},{"index":3,"sense_key":"stay%2:42:01::","synset_id":"bide.v.01"},{"index":4,"sense_key":"stay%2:42:02::","synset_id":"stay.v.04"},{"index":5,"sense_key":"stay%2:38:01::","synset_id":"stay.v.05"},{"index":6,"sense_key":"stay%2:30:01::","synset_id":"stay.v.06"},{"index":7,"sense_key":"stay%2:42:04::","synset_id":"persist.v.03"},{"index":8,"sense_key":"stay%2:42:03::","synset_id":"last_out.v.01"},{"index":9,"sense_key":"stay%2:41:00::","synset_id":"stay.v.09"},{"index":10,"sense_key":"stay%2:35:00::","synset_id":"stay.v.10"},{"index":11,"sense_key":"stay%2:34:00::","synset_id":"quell.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"stay%2:42:01::","call_ids":["senseval3.d000.s067.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"stay%2:42:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1060,"cached_input_tokens":0,"output_tokens":90,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00302,"input_uncached_usd":0.00212,"input_cached_usd":0.0,"output_usd":0.0009000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4053677920019254}
{"item_id":"senseval3.d000.s067.t002","gold_sense_keys":["time%1:03:00::"],"candidates":[{"index":1,"sense_key":"time%1:11:00::","synset_id":"time.n.01"},{"index":2,"sense_key":"time%1:28:05::","synset_id":"time.n.02"},{"index":3,"sense_key":"time%1:28:00::","synset_id":"time.n.03"},{"index":4,"sense_key":"time%1:28:06::","synset_id":"time.n.04"},{"index":5,"sense_key":"time%1:03:00::","synset_id":"time.n.05"},{"index":6,"sense_key":"time%1:11:01::","synset_id":"time.n.06"},{"index":7,"sense_key":"time%1:28:03::","synset_id":"clock_time.n.01"},{"index":8,"sense_key":"time%1:28:04::","synset_id":"fourth_dimension.n.01"},{"index":9,"sense_key":"time%1:07:01::","synset_id":"meter.n.04"},{"index":10,"sense_key":"time%1:28:01::","synset_id":"prison_term.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"time%1:03:00::","call_ids":["senseval3.d000.s067.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"time%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":990,"cached_input_tokens":0,"output_tokens":46,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00244,"input_uncached_usd":0.00198,"input_cached_usd":0.0,"output_usd":0.00046,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8749061670387164}
{"item_id":"senseval3.d000.s067.t003","gold_sense_keys":["pass%2:38:03::"],"candidates":[{"index":1,"sense_key":"pass%2:38:00::","synset_id":"pass.v.01"},{"index":2,"sense_key":"pass%2:38:05::","synset_id":"travel_by.v.01"},{"index":3,"sense_key":"pass%2:41:02::","synset_id":"legislate.v.01"},{"index":4,"sense_key":"pass%2:38:03::","synset_id":"elapse.v.01"},{"index":5,"sense_key":"pass%2:40:00::","synset_id":"pass.v.05"},{"index":6,"sense_key":"pass%2:42:08::","synset_id":"run.v.03"},{"index":7,"sense_key":"pass%2:38:01::","synset_id":"pass.v.07"},{"index":8,"sense_key":"pass%2:30:00::","synset_id":"happen.v.01"},{"index":9,"sense_key":"pass%2:41:04::","synset_id":"pass.v.09"},{"index":10,"sense_key":"pass%2:42:00::","synset_id":"spend.v.01"},{"index":11,"sense_key":"pass%2:35:00::","synset_id":"guide.v.05"},{"index":12,"sense_key":"pass%2:32:01::","synset_id":"communicate.v.01"},{"index":13,"sense_key":"pass%2:30:01::","synset_id":"evanesce.v.01"},{"index":14,"sense_key":"pass%2:41:08::","synset_id":"pass.v.14"},{"index":15,"sense_key":"pass%2:42:04::","synset_id":"exceed.v.02"},{"index":16,"sense_key":"pass%2:41:07::","synset_id":"pass.v.16"},{"index":17,"sense_key":"pass%2:41:11::","synset_id":"pass.v.17"},{"index":18,"sense_key":"pass%2:40:02::","synset_id":"pass.v.18"},{"index":19,"sense_key":"pass%2:38:02::","synset_id":"sink.v.03"},{"index":20,"sense_key":"pass%2:40:01::","synset_id":"pass.v.20"},{"index":21,"sense_key":"pass%2:40:13::","synset_id":"fall.v.21"},{"index":22,"sense_key":"pass%2:38:04::","synset_id":"pass.v.22"},{"index":23,"sense_key":"pass%2:32:00::","synset_id":"authorize.v.01"},{"index":24,"sense_key":"pass%2:30:11::","synset_id":"die.v.01"},{"index":25,"sense_key":"pass%2:29:00::","synset_id":"excrete.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"pass%2:38:03::","call_ids":["senseval3.d000.s067.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"pass%2:38:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2037,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004194,"input_uncached_usd":0.0040739999999999995,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1568279169732705}
{"item_id":"senseval3.d000.s067.t004","gold_sense_keys":["seem%2:39:01::"],"candidates":[{"index":1,"sense_key":"seem%2:39:00::","synset_id":"look.v.02"},{"index":2,"sense_key":"seem%2:39:01::","synset_id":"appear.v.04"},{"index":3,"sense_key":"seem%2:42:00::","synset_id":"seem.v.03"},{"index":4,"sense_key":"seem%2:39:02::","synset_id":"seem.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"seem%2:39:01::","call_ids":["senseval3.d000.s067.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"seem%2:39:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":681,"cached_input_tokens":0,"output_tokens":76,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0021219999999999998,"input_uncached_usd":0.001362,"input_cached_usd":0.0,"output_usd":0.00076,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9720262920018286}
{"item_id":"senseval3.d000.s067.t005","gold_sense_keys":["strange%3:00:00::"],"candidates":[{"index":1,"sense_key":"strange%3:00:00::","synset_id":"strange.a.01"},{"index":2,"sense_key":"strange%5:00:01:unfamiliar:00","synset_id":"strange.s.01"},{"index":3,"sense_key":"strange%3:00:01::","synset_id":"foreign.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"strange%5:00:01:unfamiliar:00","call_ids":["senseval3.d000.s067.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"strange%5:00:01:unfamiliar:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":639,"cached_input_tokens":0,"output_tokens":126,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002538,"input_uncached_usd":0.0012779999999999998,"input_cached_usd":0.0,"output_usd":0.00126,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.413104624953121}
{"item_id":"senseval3.d000.s067.t006","gold_sense_keys":["little%3:00:00::","little%3:00:01::"],"candidates":[{"index":1,"sense_key":"little%3:00:01::","synset_id":"small.a.01"},{"index":2,"sense_key":"little%3:00:03::","synset_id":"little.a.02"},{"index":3,"sense_key":"little%5:00:00:young:00","synset_id":"little.s.01"},{"index":4,"sense_key":"little%5:00:00:unimportant:00","synset_id":"fiddling.s.01"},{"index":5,"sense_key":"little%5:00:00:soft:04","synset_id":"little.s.03"},{"index":6,"sense_key":"little%3:00:00::","synset_id":"short.a.03"},{"index":7,"sense_key":"little%5:00:00:lowercase:00","synset_id":"little.s.04"},{"index":8,"sense_key":"little%5:00:00:emotional:00","synset_id":"little.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"little%3:00:01::","call_ids":["senseval3.d000.s067.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"little%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":936,"cached_input_tokens":0,"output_tokens":196,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0038320000000000003,"input_uncached_usd":0.001872,"input_cached_usd":0.0,"output_usd":0.0019600000000000004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.7867049579508603}
{"item_id":"senseval3.d000.s067.t007","gold_sense_keys":["man%1:18:00::"],"candidates":[{"index":1,"sense_key":"man%1:18:00::","synset_id":"man.n.01"},{"index":2,"sense_key":"man%1:18:03::","synset_id":"serviceman.n.01"},{"index":3,"sense_key":"man%1:18:04::","synset_id":"man.n.03"},{"index":4,"sense_key":"man%1:05:01::","synset_id":"homo.n.02"},{"index":5,"sense_key":"man%1:18:08::","synset_id":"man.n.05"},{"index":6,"sense_key":"man%1:18:06::","synset_id":"man.n.06"},{"index":7,"sense_key":"man%1:18:05::","synset_id":"valet.n.01"},{"index":8,"sense_key":"man%1:18:07::","synset_id":"man.n.08"},{"index":9,"sense_key":"man%1:15:00::","synset_id":"man.n.09"},{"index":10,"sense_key":"man%1:06:00::","synset_id":"man.n.10"},{"index":11,"sense_key":"man%1:05:00::","synset_id":"world.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"man%1:18:00::","call_ids":["senseval3.d000.s067.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"man%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1160,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00244,"input_uncached_usd":0.00232,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.54599779099226}
{"item_id":"senseval3.d000.s067.t008","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":2,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":3,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"other%3:00:00::","call_ids":["senseval3.d000.s067.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":654,"cached_input_tokens":0,"output_tokens":51,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001818,"input_uncached_usd":0.001308,"input_cached_usd":0.0,"output_usd":0.00051,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.327917749993503}
{"item_id":"senseval3.d000.s067.t009","gold_sense_keys":["glass%1:06:00::"],"candidates":[{"index":1,"sense_key":"glass%1:27:00::","synset_id":"glass.n.01"},{"index":2,"sense_key":"glass%1:06:00::","synset_id":"glass.n.02"},{"index":3,"sense_key":"glass%1:23:00::","synset_id":"glass.n.03"},{"index":4,"sense_key":"glass%1:06:03::","synset_id":"field_glass.n.01"},{"index":5,"sense_key":"glass%1:06:01::","synset_id":"methamphetamine.n.01"},{"index":6,"sense_key":"glass%1:06:02::","synset_id":"looking_glass.n.01"},{"index":7,"sense_key":"glass%1:06:04::","synset_id":"glass.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"glass%1:06:00::","call_ids":["senseval3.d000.s067.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"glass%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":807,"cached_input_tokens":0,"output_tokens":49,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002104,"input_uncached_usd":0.001614,"input_cached_usd":0.0,"output_usd":0.0004900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.735122665995732}
{"item_id":"senseval3.d000.s067.t010","gold_sense_keys":["table%1:06:01::"],"candidates":[{"index":1,"sense_key":"table%1:14:00::","synset_id":"table.n.01"},{"index":2,"sense_key":"table%1:06:01::","synset_id":"table.n.02"},{"index":3,"sense_key":"table%1:06:02::","synset_id":"table.n.03"},{"index":4,"sense_key":"table%1:17:00::","synset_id":"mesa.n.01"},{"index":5,"sense_key":"table%1:14:01::","synset_id":"table.n.05"},{"index":6,"sense_key":"table%1:13:01::","synset_id":"board.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"table%1:06:01::","call_ids":["senseval3.d000.s067.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"table%1:06:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":770,"cached_input_tokens":0,"output_tokens":95,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00249,"input_uncached_usd":0.00154,"input_cached_usd":0.0,"output_usd":0.0009500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4719678750261664}
{"item_id":"senseval3.d000.s068.t002","gold_sense_keys":["return%2:38:00::"],"candidates":[{"index":1,"sense_key":"return%2:38:00::","synset_id":"return.v.01"},{"index":2,"sense_key":"return%2:40:08::","synset_id":"render.v.07"},{"index":3,"sense_key":"return%2:30:00::","synset_id":"revert.v.01"},{"index":4,"sense_key":"return%2:32:00::","synset_id":"hark_back.v.01"},{"index":5,"sense_key":"return%2:38:12::","synset_id":"return.v.05"},{"index":6,"sense_key":"return%2:40:03::","synset_id":"return.v.06"},{"index":7,"sense_key":"return%2:35:01::","synset_id":"return.v.07"},{"index":8,"sense_key":"return%2:32:01::","synset_id":"retort.v.01"},{"index":9,"sense_key":"return%2:30:01::","synset_id":"come_back.v.01"},{"index":10,"sense_key":"return%2:40:00::","synset_id":"refund.v.01"},{"index":11,"sense_key":"return%2:32:03::","synset_id":"render.v.05"},{"index":12,"sense_key":"return%2:41:00::","synset_id":"reelect.v.01"},{"index":13,"sense_key":"return%2:40:13::","synset_id":"fall.v.21"},{"index":14,"sense_key":"return%2:38:01::","synset_id":"return.v.14"},{"index":15,"sense_key":"return%2:36:00::","synset_id":"render.v.04"},{"index":16,"sense_key":"return%2:32:02::","synset_id":"return.v.16"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"return%2:38:00::","call_ids":["senseval3.d000.s068.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"return%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1243,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002606,"input_uncached_usd":0.002486,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1484019580530003}
{"item_id":"senseval3.d000.s068.t005","gold_sense_keys":["put%2:35:00::"],"candidates":[{"index":1,"sense_key":"put%2:35:00::","synset_id":"put.v.01"},{"index":2,"sense_key":"put%2:35:05::","synset_id":"put.v.02"},{"index":3,"sense_key":"put%2:32:00::","synset_id":"frame.v.04"},{"index":4,"sense_key":"put%2:34:00::","synset_id":"put.v.04"},{"index":5,"sense_key":"put%2:40:00::","synset_id":"invest.v.01"},{"index":6,"sense_key":"put%2:31:13::","synset_id":"place.v.11"},{"index":7,"sense_key":"put%2:39:13::","synset_id":"put.v.07"},{"index":8,"sense_key":"put%2:36:13::","synset_id":"put.v.08"},{"index":9,"sense_key":"put%2:31:00::","synset_id":"arrange.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"put%2:35:05::","call_ids":["senseval3.d000.s068.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"put%2:35:05::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":952,"cached_input_tokens":0,"output_tokens":127,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0031739999999999997,"input_uncached_usd":0.0019039999999999999,"input_cached_usd":0.0,"output_usd":0.00127,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.823521292069927}
{"item_id":"senseval3.d000.s068.t006","gold_sense_keys":["whole%3:00:00::"],"candidates":[{"index":1,"sense_key":"whole%3:00:00::","synset_id":"whole.a.01"},{"index":2,"sense_key":"whole%3:00:02::","synset_id":"whole.a.02"},{"index":3,"sense_key":"whole%5:00:00:uninjured:00","synset_id":"unharmed.s.01"},{"index":4,"sense_key":"whole%5:00:00:healthy:00","synset_id":"hale.s.01"},{"index":5,"sense_key":"whole%5:00:00:undiversified:00","synset_id":"solid.s.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"whole%3:00:00::","call_ids":["senseval3.d000.s068.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"whole%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":747,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001614,"input_uncached_usd":0.0014939999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0306226250249892}
{"item_id":"senseval3.d000.s068.t007","gold_sense_keys":["incident%1:11:00::"],"candidates":[{"index":1,"sense_key":"incident%1:11:00::","synset_id":"incident.n.01"},{"index":2,"sense_key":"incident%1:26:00::","synset_id":"incident.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"incident%1:11:00::","call_ids":["senseval3.d000.s068.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"incident%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":515,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00115,"input_uncached_usd":0.0010299999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3845079579623416}
{"item_id":"senseval3.d000.s068.t008","gold_sense_keys":["mind%1:09:00::","mind%1:09:01::","mind%1:09:03::"],"candidates":[{"index":1,"sense_key":"mind%1:09:00::","synset_id":"mind.n.01"},{"index":2,"sense_key":"mind%1:09:01::","synset_id":"mind.n.02"},{"index":3,"sense_key":"mind%1:09:04::","synset_id":"judgment.n.01"},{"index":4,"sense_key":"mind%1:18:00::","synset_id":"thinker.n.01"},{"index":5,"sense_key":"mind%1:09:03::","synset_id":"mind.n.05"},{"index":6,"sense_key":"mind%1:09:06::","synset_id":"mind.n.06"},{"index":7,"sense_key":"mind%1:09:05::","synset_id":"mind.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"mind%1:09:01::","call_ids":["senseval3.d000.s068.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"mind%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":869,"cached_input_tokens":0,"output_tokens":174,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003478,"input_uncached_usd":0.001738,"input_cached_usd":0.0,"output_usd":0.0017400000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.084629625082016}
{"item_id":"senseval3.d000.s069.t000","gold_sense_keys":["next%5:00:00:succeeding:00"],"candidates":[{"index":1,"sense_key":"next%5:00:00:succeeding:00","synset_id":"following.s.02"},{"index":2,"sense_key":"next%5:00:00:close:01","synset_id":"adjacent.s.01"},{"index":3,"sense_key":"next%5:00:00:incoming:00","synset_id":"future.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"next%5:00:00:succeeding:00","call_ids":["senseval3.d000.s069.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"next%5:00:00:succeeding:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":647,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0014140000000000001,"input_uncached_usd":0.001294,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1103337090462446}
{"item_id":"senseval3.d000.s069.t001","gold_sense_keys":["day%1:28:00::"],"candidates":[{"index":1,"sense_key":"day%1:28:00::","synset_id":"day.n.01"},{"index":2,"sense_key":"day%1:28:03::","synset_id":"day.n.02"},{"index":3,"sense_key":"day%1:28:01::","synset_id":"day.n.03"},{"index":4,"sense_key":"day%1:28:02::","synset_id":"day.n.04"},{"index":5,"sense_key":"day%1:28:04::","synset_id":"day.n.05"},{"index":6,"sense_key":"day%1:28:05::","synset_id":"day.n.06"},{"index":7,"sense_key":"day%1:28:07::","synset_id":"day.n.07"},{"index":8,"sense_key":"day%1:28:06::","synset_id":"sidereal_day.n.01"},{"index":9,"sense_key":"day%1:26:00::","synset_id":"day.n.09"},{"index":10,"sense_key":"day%1:18:00::","synset_id":"day.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"day%1:28:00::","call_ids":["senseval3.d000.s069.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"day%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1113,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0027860000000000003,"input_uncached_usd":0.002226,"input_cached_usd":0.0,"output_usd":0.0005600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2010404169559479}
{"item_id":"senseval3.d000.s069.t002","gold_sense_keys":["hangover%1:26:00::"],"candidates":[{"index":1,"sense_key":"hangover%1:26:00::","synset_id":"hangover.n.01"},{"index":2,"sense_key":"hangover%1:18:00::","synset_id":"holdover.n.01"},{"index":3,"sense_key":"hangover%1:04:00::","synset_id":"hangover.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"hangover%1:26:00::","call_ids":["senseval3.d000.s069.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"hangover%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":629,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001378,"input_uncached_usd":0.001258,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0247058330569416}
{"item_id":"senseval3.d000.s069.t003","gold_sense_keys":["remind%2:31:00::"],"candidates":[{"index":1,"sense_key":"remind%2:31:00::","synset_id":"remind.v.01"},{"index":2,"sense_key":"remind%2:32:00::","synset_id":"prompt.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"remind%2:31:00::","call_ids":["senseval3.d000.s069.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"remind%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":574,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001268,"input_uncached_usd":0.001148,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4808399999747053}
{"item_id":"senseval3.d000.s069.t004","gold_sense_keys":["night%1:28:03::"],"candidates":[{"index":1,"sense_key":"night%1:28:00::","synset_id":"night.n.01"},{"index":2,"sense_key":"night%1:28:01::","synset_id":"night.n.02"},{"index":3,"sense_key":"night%1:28:04::","synset_id":"night.n.03"},{"index":4,"sense_key":"night%1:28:02::","synset_id":"night.n.04"},{"index":5,"sense_key":"night%1:26:00::","synset_id":"night.n.05"},{"index":6,"sense_key":"night%1:28:05::","synset_id":"night.n.06"},{"index":7,"sense_key":"night%1:28:03::","synset_id":"night.n.07"},{"index":8,"sense_key":"night%1:18:00::","synset_id":"nox.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"night%1:28:02::","call_ids":["senseval3.d000.s069.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"night%1:28:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":819,"cached_input_tokens":0,"output_tokens":170,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003338,"input_uncached_usd":0.001638,"input_cached_usd":0.0,"output_usd":0.0017000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.588140540989116}
{"item_id":"senseval3.d000.s070.t000","gold_sense_keys":["hangover%1:26:00::"],"candidates":[{"index":1,"sense_key":"hangover%1:26:00::","synset_id":"hangover.n.01"},{"index":2,"sense_key":"hangover%1:18:00::","synset_id":"holdover.n.01"},{"index":3,"sense_key":"hangover%1:04:00::","synset_id":"hangover.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"hangover%1:26:00::","call_ids":["senseval3.d000.s070.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"hangover%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":671,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001462,"input_uncached_usd":0.0013419999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4369389999192208}
{"item_id":"senseval3.d000.s070.t001","gold_sense_keys":["remind%2:31:00::"],"candidates":[{"index":1,"sense_key":"remind%2:31:00::","synset_id":"remind.v.01"},{"index":2,"sense_key":"remind%2:32:00::","synset_id":"prompt.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"remind%2:31:00::","call_ids":["senseval3.d000.s070.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"remind%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":616,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0013520000000000001,"input_uncached_usd":0.001232,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9705504169687629}
{"item_id":"senseval3.d000.s071.t000","gold_sense_keys":["go%2:38:00::"],"candidates":[{"index":1,"sense_key":"go%2:38:00::","synset_id":"travel.v.01"},{"index":2,"sense_key":"go%2:41:00::","synset_id":"go.v.02"},{"index":3,"sense_key":"go%2:38:01::","synset_id":"go.v.03"},{"index":4,"sense_key":"go%2:30:04::","synset_id":"become.v.01"},{"index":5,"sense_key":"go%2:42:06::","synset_id":"go.v.05"},{"index":6,"sense_key":"go%2:42:02::","synset_id":"run.v.05"},{"index":7,"sense_key":"go%2:42:00::","synset_id":"run.v.03"},{"index":8,"sense_key":"go%2:30:01::","synset_id":"proceed.v.04"},{"index":9,"sense_key":"go%2:30:08::","synset_id":"go.v.09"},{"index":10,"sense_key":"go%2:42:07::","synset_id":"go.v.10"},{"index":11,"sense_key":"go%2:39:00::","synset_id":"sound.v.02"},{"index":12,"sense_key":"go%2:35:00::","synset_id":"function.v.01"},{"index":13,"sense_key":"go%2:34:00::","synset_id":"run_low.v.01"},{"index":14,"sense_key":"go%2:30:02::","synset_id":"move.v.13"},{"index":15,"sense_key":"go%2:42:14::","synset_id":"survive.v.01"},{"index":16,"sense_key":"go%2:42:03::","synset_id":"go.v.16"},{"index":17,"sense_key":"go%2:30:00::","synset_id":"die.v.01"},{"index":18,"sense_key":"go%2:42:04::","synset_id":"belong.v.03"},{"index":19,"sense_key":"go%2:42:10::","synset_id":"go.v.19"},{"index":20,"sense_key":"go%2:38:09::","synset_id":"start.v.09"},{"index":21,"sense_key":"go%2:33:00::","synset_id":"move.v.15"},{"index":22,"sense_key":"go%2:42:13::","synset_id":"go.v.22"},{"index":23,"sense_key":"go%2:42:12::","synset_id":"go.v.23"},{"index":24,"sense_key":"go%2:42:11::","synset_id":"blend.v.02"},{"index":25,"sense_key":"go%2:42:08::","synset_id":"go.v.25"},{"index":26,"sense_key":"go%2:42:05::","synset_id":"fit.v.02"},{"index":27,"sense_key":"go%2:35:13::","synset_id":"rifle.v.02"},{"index":28,"sense_key":"go%2:34:09::","synset_id":"go.v.28"},{"index":29,"sense_key":"go%2:31:12::","synset_id":"plump.v.04"},{"index":30,"sense_key":"go%2:30:03::","synset_id":"fail.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"go%2:38:00::","call_ids":["senseval3.d000.s071.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"go%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2386,"cached_input_tokens":0,"output_tokens":141,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.006182,"input_uncached_usd":0.004772,"input_cached_usd":0.0,"output_usd":0.00141,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8390182920265943}
{"item_id":"senseval3.d000.s071.t002","gold_sense_keys":["day%1:28:04::"],"candidates":[{"index":1,"sense_key":"day%1:28:00::","synset_id":"day.n.01"},{"index":2,"sense_key":"day%1:28:03::","synset_id":"day.n.02"},{"index":3,"sense_key":"day%1:28:01::","synset_id":"day.n.03"},{"index":4,"sense_key":"day%1:28:02::","synset_id":"day.n.04"},{"index":5,"sense_key":"day%1:28:04::","synset_id":"day.n.05"},{"index":6,"sense_key":"day%1:28:05::","synset_id":"day.n.06"},{"index":7,"sense_key":"day%1:28:07::","synset_id":"day.n.07"},{"index":8,"sense_key":"day%1:28:06::","synset_id":"sidereal_day.n.01"},{"index":9,"sense_key":"day%1:26:00::","synset_id":"day.n.09"},{"index":10,"sense_key":"day%1:18:00::","synset_id":"day.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"day%1:28:02::","call_ids":["senseval3.d000.s071.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"day%1:28:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1137,"cached_input_tokens":0,"output_tokens":269,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004964,"input_uncached_usd":0.002274,"input_cached_usd":0.0,"output_usd":0.00269,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.790217334055342}
{"item_id":"senseval3.d000.s071.t003","gold_sense_keys":["pass%2:38:05::"],"candidates":[{"index":1,"sense_key":"pass%2:38:00::","synset_id":"pass.v.01"},{"index":2,"sense_key":"pass%2:38:05::","synset_id":"travel_by.v.01"},{"index":3,"sense_key":"pass%2:41:02::","synset_id":"legislate.v.01"},{"index":4,"sense_key":"pass%2:38:03::","synset_id":"elapse.v.01"},{"index":5,"sense_key":"pass%2:40:00::","synset_id":"pass.v.05"},{"index":6,"sense_key":"pass%2:42:08::","synset_id":"run.v.03"},{"index":7,"sense_key":"pass%2:38:01::","synset_id":"pass.v.07"},{"index":8,"sense_key":"pass%2:30:00::","synset_id":"happen.v.01"},{"index":9,"sense_key":"pass%2:41:04::","synset_id":"pass.v.09"},{"index":10,"sense_key":"pass%2:42:00::","synset_id":"spend.v.01"},{"index":11,"sense_key":"pass%2:35:00::","synset_id":"guide.v.05"},{"index":12,"sense_key":"pass%2:32:01::","synset_id":"communicate.v.01"},{"index":13,"sense_key":"pass%2:30:01::","synset_id":"evanesce.v.01"},{"index":14,"sense_key":"pass%2:41:08::","synset_id":"pass.v.14"},{"index":15,"sense_key":"pass%2:42:04::","synset_id":"exceed.v.02"},{"index":16,"sense_key":"pass%2:41:07::","synset_id":"pass.v.16"},{"index":17,"sense_key":"pass%2:41:11::","synset_id":"pass.v.17"},{"index":18,"sense_key":"pass%2:40:02::","synset_id":"pass.v.18"},{"index":19,"sense_key":"pass%2:38:02::","synset_id":"sink.v.03"},{"index":20,"sense_key":"pass%2:40:01::","synset_id":"pass.v.20"},{"index":21,"sense_key":"pass%2:40:13::","synset_id":"fall.v.21"},{"index":22,"sense_key":"pass%2:38:04::","synset_id":"pass.v.22"},{"index":23,"sense_key":"pass%2:32:00::","synset_id":"authorize.v.01"},{"index":24,"sense_key":"pass%2:30:11::","synset_id":"die.v.01"},{"index":25,"sense_key":"pass%2:29:00::","synset_id":"excrete.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"pass%2:38:05::","call_ids":["senseval3.d000.s071.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"pass%2:38:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2101,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004852,"input_uncached_usd":0.004202,"input_cached_usd":0.0,"output_usd":0.0006500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.303064625011757}
{"item_id":"senseval3.d000.s071.t007","gold_sense_keys":["alive%3:00:01::"],"candidates":[{"index":1,"sense_key":"alive%3:00:01::","synset_id":"alive.a.01"},{"index":2,"sense_key":"alive%5:00:00:lively:00","synset_id":"alive.s.01"},{"index":3,"sense_key":"alive%3:00:04::","synset_id":"animated.a.01"},{"index":4,"sense_key":"alive%5:00:00:sensitive:02","synset_id":"alive.s.02"},{"index":5,"sense_key":"alive%5:00:00:existent:00","synset_id":"active.s.03"},{"index":6,"sense_key":"alive%5:00:00:aware:00","synset_id":"alert.s.02"},{"index":7,"sense_key":"alive%5:00:00:active:05","synset_id":"alive.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"alive%3:00:01::","call_ids":["senseval3.d000.s071.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"alive%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":892,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002314,"input_uncached_usd":0.001784,"input_cached_usd":0.0,"output_usd":0.0005300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.318361165933311}
{"item_id":"senseval3.d000.s071.t009","gold_sense_keys":["tenant%1:18:00::"],"candidates":[{"index":1,"sense_key":"tenant%1:18:00::","synset_id":"tenant.n.01"},{"index":2,"sense_key":"tenant%1:18:02::","synset_id":"tenant.n.02"},{"index":3,"sense_key":"tenant%1:18:01::","synset_id":"tenant.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"tenant%1:18:00::","call_ids":["senseval3.d000.s071.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"tenant%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":649,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001418,"input_uncached_usd":0.001298,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4542157920077443}
{"item_id":"senseval3.d000.s071.t010","gold_sense_keys":["burden%1:09:01::"],"candidates":[{"index":1,"sense_key":"burden%1:09:01::","synset_id":"burden.n.01"},{"index":2,"sense_key":"burden%1:06:01::","synset_id":"load.n.01"},{"index":3,"sense_key":"burden%1:10:00::","synset_id":"effect.n.04"},{"index":4,"sense_key":"burden%1:09:00::","synset_id":"burden.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"burden%1:09:01::","call_ids":["senseval3.d000.s071.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"burden%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":695,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00151,"input_uncached_usd":0.00139,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2055113749811426}
{"item_id":"senseval3.d000.s071.t011","gold_sense_keys":["life%1:26:02::"],"candidates":[{"index":1,"sense_key":"life%1:26:01::","synset_id":"life.n.01"},{"index":2,"sense_key":"life%1:09:00::","synset_id":"life.n.02"},{"index":3,"sense_key":"life%1:26:02::","synset_id":"life.n.03"},{"index":4,"sense_key":"life%1:26:00::","synset_id":"animation.n.01"},{"index":5,"sense_key":"life%1:28:00::","synset_id":"life.n.05"},{"index":6,"sense_key":"life%1:28:01::","synset_id":"life.n.06"},{"index":7,"sense_key":"life%1:28:02::","synset_id":"life.n.07"},{"index":8,"sense_key":"life%1:18:00::","synset_id":"life.n.08"},{"index":9,"sense_key":"life%1:07:01::","synset_id":"liveliness.n.02"},{"index":10,"sense_key":"life%1:03:00::","synset_id":"life.n.10"},{"index":11,"sense_key":"life%1:19:00::","synset_id":"life.n.11"},{"index":12,"sense_key":"life%1:10:00::","synset_id":"biography.n.01"},{"index":13,"sense_key":"life%1:16:00::","synset_id":"life.n.13"},{"index":14,"sense_key":"life%1:28:03::","synset_id":"life_sentence.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"life%1:26:02::","call_ids":["senseval3.d000.s071.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"life%1:26:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1244,"cached_input_tokens":0,"output_tokens":105,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003538,"input_uncached_usd":0.0024879999999999998,"input_cached_usd":0.0,"output_usd":0.0010500000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.891180209000595}
{"item_id":"senseval3.d000.s072.t001","gold_sense_keys":["ignore%2:32:02::"],"candidates":[{"index":1,"sense_key":"ignore%2:32:02::","synset_id":"ignore.v.01"},{"index":2,"sense_key":"ignore%2:32:00::","synset_id":"dismiss.v.01"},{"index":3,"sense_key":"ignore%2:39:00::","synset_id":"ignore.v.03"},{"index":4,"sense_key":"ignore%2:31:03::","synset_id":"neglect.v.04"},{"index":5,"sense_key":"ignore%2:31:01::","synset_id":"ignore.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ignore%2:32:02::","call_ids":["senseval3.d000.s072.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ignore%2:32:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":702,"cached_input_tokens":0,"output_tokens":142,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002824,"input_uncached_usd":0.0014039999999999999,"input_cached_usd":0.0,"output_usd":0.00142,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0538872909964994}
{"item_id":"senseval3.d000.s074.t001","gold_sense_keys":["alive%3:00:01::"],"candidates":[{"index":1,"sense_key":"alive%3:00:01::","synset_id":"alive.a.01"},{"index":2,"sense_key":"alive%5:00:00:lively:00","synset_id":"alive.s.01"},{"index":3,"sense_key":"alive%3:00:04::","synset_id":"animated.a.01"},{"index":4,"sense_key":"alive%5:00:00:sensitive:02","synset_id":"alive.s.02"},{"index":5,"sense_key":"alive%5:00:00:existent:00","synset_id":"active.s.03"},{"index":6,"sense_key":"alive%5:00:00:aware:00","synset_id":"alert.s.02"},{"index":7,"sense_key":"alive%5:00:00:active:05","synset_id":"alive.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"alive%3:00:01::","call_ids":["senseval3.d000.s074.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"alive%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":832,"cached_input_tokens":0,"output_tokens":40,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002064,"input_uncached_usd":0.001664,"input_cached_usd":0.0,"output_usd":0.0004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3528775409795344}
{"item_id":"senseval3.d000.s075.t001","gold_sense_keys":["mail%2:35:00::"],"candidates":[{"index":1,"sense_key":"mail%2:35:00::","synset_id":"mail.v.01"},{"index":2,"sense_key":"mail%2:32:00::","synset_id":"mail.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"mail%2:35:00::","call_ids":["senseval3.d000.s075.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"mail%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":548,"cached_input_tokens":0,"output_tokens":113,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002226,"input_uncached_usd":0.001096,"input_cached_usd":0.0,"output_usd":0.0011300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6704808339709416}
{"item_id":"senseval3.d000.s075.t003","gold_sense_keys":["check%1:21:00::"],"candidates":[{"index":1,"sense_key":"check%1:21:00::","synset_id":"check.n.01"},{"index":2,"sense_key":"check%1:09:00::","synset_id":"assay.n.01"},{"index":3,"sense_key":"check%1:10:00::","synset_id":"check.n.03"},{"index":4,"sense_key":"check%1:26:00::","synset_id":"arrest.n.02"},{"index":5,"sense_key":"check%1:09:01::","synset_id":"confirmation.n.01"},{"index":6,"sense_key":"check%1:04:05::","synset_id":"check.n.06"},{"index":7,"sense_key":"check%1:10:01::","synset_id":"check_mark.n.01"},{"index":8,"sense_key":"check%1:09:02::","synset_id":"hindrance.n.01"},{"index":9,"sense_key":"check%1:07:00::","synset_id":"check.n.09"},{"index":10,"sense_key":"check%1:06:00::","synset_id":"check.n.10"},{"index":11,"sense_key":"check%1:04:01::","synset_id":"bridle.n.02"},{"index":12,"sense_key":"check%1:04:02::","synset_id":"check.n.12"},{"index":13,"sense_key":"check%1:04:04::","synset_id":"check.n.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"check%1:21:00::","call_ids":["senseval3.d000.s075.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"check%1:21:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1235,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00259,"input_uncached_usd":0.00247,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.087439417024143}
{"item_id":"senseval3.d000.s075.t005","gold_sense_keys":["maintenance%1:21:01::"],"candidates":[{"index":1,"sense_key":"maintenance%1:04:00::","synset_id":"care.n.06"},{"index":2,"sense_key":"maintenance%1:21:00::","synset_id":"maintenance.n.02"},{"index":3,"sense_key":"maintenance%1:21:01::","synset_id":"alimony.n.01"},{"index":4,"sense_key":"maintenance%1:04:01::","synset_id":"sustenance.n.03"},{"index":5,"sense_key":"maintenance%1:04:02::","synset_id":"maintenance.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"maintenance%1:21:01::","call_ids":["senseval3.d000.s075.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"maintenance%1:21:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":814,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002188,"input_uncached_usd":0.0016279999999999999,"input_cached_usd":0.0,"output_usd":0.0005600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3538037079852074}
{"item_id":"senseval3.d000.s075.t007","gold_sense_keys":["wish%2:32:02::","wish%2:37:00::"],"candidates":[{"index":1,"sense_key":"wish%2:37:00::","synset_id":"wish.v.01"},{"index":2,"sense_key":"wish%2:37:02::","synset_id":"wish.v.02"},{"index":3,"sense_key":"wish%2:32:02::","synset_id":"wish.v.03"},{"index":4,"sense_key":"wish%2:37:01::","synset_id":"wish.v.04"},{"index":5,"sense_key":"wish%2:32:01::","synset_id":"wish.v.05"},{"index":6,"sense_key":"wish%2:32:00::","synset_id":"wish.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"wish%2:37:00::","call_ids":["senseval3.d000.s075.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"wish%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":711,"cached_input_tokens":0,"output_tokens":138,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002802,"input_uncached_usd":0.0014219999999999999,"input_cached_usd":0.0,"output_usd":0.0013800000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8247140829917043}
{"item_id":"senseval3.d000.s075.t008","gold_sense_keys":["stranger%1:18:01::"],"candidates":[{"index":1,"sense_key":"stranger%1:18:00::","synset_id":"stranger.n.01"},{"index":2,"sense_key":"stranger%1:18:01::","synset_id":"stranger.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"stranger%1:18:01::","call_ids":["senseval3.d000.s075.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"stranger%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":514,"cached_input_tokens":0,"output_tokens":96,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001988,"input_uncached_usd":0.001028,"input_cached_usd":0.0,"output_usd":0.0009600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2333600409328938}
{"item_id":"senseval3.d000.s075.t009","gold_sense_keys":["do%2:36:01::","do%2:41:01::","do%2:41:04::"],"candidates":[{"index":1,"sense_key":"do%2:41:01::","synset_id":"make.v.01"},{"index":2,"sense_key":"do%2:36:01::","synset_id":"perform.v.01"},{"index":3,"sense_key":"do%2:41:04::","synset_id":"do.v.03"},{"index":4,"sense_key":"do%2:42:00::","synset_id":"do.v.04"},{"index":5,"sense_key":"do%2:36:02::","synset_id":"cause.v.01"},{"index":6,"sense_key":"do%2:41:02::","synset_id":"practice.v.01"},{"index":7,"sense_key":"do%2:42:01::","synset_id":"suffice.v.01"},{"index":8,"sense_key":"do%2:36:00::","synset_id":"do.v.08"},{"index":9,"sense_key":"do%2:29:09::","synset_id":"act.v.02"},{"index":10,"sense_key":"do%2:42:02::","synset_id":"serve.v.09"},{"index":11,"sense_key":"do%2:41:03::","synset_id":"do.v.11"},{"index":12,"sense_key":"do%2:29:04::","synset_id":"dress.v.16"},{"index":13,"sense_key":"do%2:38:00::","synset_id":"do.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"do%2:36:01::","call_ids":["senseval3.d000.s075.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"do%2:36:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1234,"cached_input_tokens":0,"output_tokens":92,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0033879999999999995,"input_uncached_usd":0.0024679999999999997,"input_cached_usd":0.0,"output_usd":0.00092,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5993566249962896}
{"item_id":"senseval3.d000.s076.t000","gold_sense_keys":["come%2:38:04::"],"candidates":[{"index":1,"sense_key":"come%2:38:00::","synset_id":"come.v.01"},{"index":2,"sense_key":"come%2:38:04::","synset_id":"arrive.v.01"},{"index":3,"sense_key":"come%2:30:01::","synset_id":"come.v.03"},{"index":4,"sense_key":"come%2:30:03::","synset_id":"come.v.04"},{"index":5,"sense_key":"come%2:42:04::","synset_id":"come.v.05"},{"index":6,"sense_key":"come%2:30:02::","synset_id":"come.v.06"},{"index":7,"sense_key":"come%2:42:13::","synset_id":"issue_forth.v.01"},{"index":8,"sense_key":"come%2:42:09::","synset_id":"hail.v.02"},{"index":9,"sense_key":"come%2:42:03::","synset_id":"come.v.09"},{"index":10,"sense_key":"come%2:42:05::","synset_id":"come.v.10"},{"index":11,"sense_key":"come%2:38:03::","synset_id":"come.v.11"},{"index":12,"sense_key":"come%2:42:00::","synset_id":"fall.v.04"},{"index":13,"sense_key":"come%2:30:13::","synset_id":"come.v.13"},{"index":14,"sense_key":"come%2:42:12::","synset_id":"total.v.01"},{"index":15,"sense_key":"come%2:42:11::","synset_id":"come.v.15"},{"index":16,"sense_key":"come%2:38:02::","synset_id":"come.v.16"},{"index":17,"sense_key":"come%2:31:14::","synset_id":"occur.v.02"},{"index":18,"sense_key":"come%2:42:15::","synset_id":"derive.v.05"},{"index":19,"sense_key":"come%2:42:02::","synset_id":"do.v.04"},{"index":20,"sense_key":"come%2:39:00::","synset_id":"come.v.20"},{"index":21,"sense_key":"come%2:31:13::","synset_id":"come.v.21"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"come%2:38:04::","call_ids":["senseval3.d000.s076.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"come%2:38:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1592,"cached_input_tokens":0,"output_tokens":114,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004324,"input_uncached_usd":0.0031839999999999998,"input_cached_usd":0.0,"output_usd":0.0011400000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7210529589792714}
{"item_id":"senseval3.d000.s076.t001","gold_sense_keys":["home%1:15:04::"],"candidates":[{"index":1,"sense_key":"home%1:15:04::","synset_id":"home.n.01"},{"index":2,"sense_key":"home%1:06:00::","synset_id":"dwelling.n.01"},{"index":3,"sense_key":"home%1:15:02::","synset_id":"home.n.03"},{"index":4,"sense_key":"home%1:06:02::","synset_id":"home_plate.n.01"},{"index":5,"sense_key":"home%1:15:05::","synset_id":"base.n.14"},{"index":6,"sense_key":"home%1:15:03::","synset_id":"home.n.06"},{"index":7,"sense_key":"home%1:26:00::","synset_id":"home.n.07"},{"index":8,"sense_key":"home%1:14:00::","synset_id":"family.n.01"},{"index":9,"sense_key":"home%1:06:01::","synset_id":"home.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"home%1:15:04::","call_ids":["senseval3.d000.s076.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"home%1:15:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":981,"cached_input_tokens":0,"output_tokens":107,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003032,"input_uncached_usd":0.0019619999999999998,"input_cached_usd":0.0,"output_usd":0.00107,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.098537000012584}
{"item_id":"senseval3.d000.s076.t002","gold_sense_keys":["work%1:06:01::"],"candidates":[{"index":1,"sense_key":"work%1:04:00::","synset_id":"work.n.01"},{"index":2,"sense_key":"work%1:06:00::","synset_id":"work.n.02"},{"index":3,"sense_key":"work%1:04:01::","synset_id":"employment.n.02"},{"index":4,"sense_key":"work%1:09:00::","synset_id":"study.n.02"},{"index":5,"sense_key":"work%1:19:00::","synset_id":"work.n.05"},{"index":6,"sense_key":"work%1:06:01::","synset_id":"workplace.n.01"},{"index":7,"sense_key":"work%1:06:02::","synset_id":"oeuvre.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"work%1:06:01::","call_ids":["senseval3.d000.s076.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"work%1:06:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":917,"cached_input_tokens":0,"output_tokens":87,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002704,"input_uncached_usd":0.001834,"input_cached_usd":0.0,"output_usd":0.0008700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9288427920546383}
{"item_id":"senseval3.d000.s076.t004","gold_sense_keys":["see%2:39:00::"],"candidates":[{"index":1,"sense_key":"see%2:39:00::","synset_id":"see.v.01"},{"index":2,"sense_key":"see%2:31:01::","synset_id":"understand.v.02"},{"index":3,"sense_key":"see%2:39:02::","synset_id":"witness.v.02"},{"index":4,"sense_key":"see%2:36:00::","synset_id":"visualize.v.01"},{"index":5,"sense_key":"see%2:31:00::","synset_id":"see.v.05"},{"index":6,"sense_key":"see%2:31:02::","synset_id":"learn.v.02"},{"index":7,"sense_key":"see%2:39:03::","synset_id":"watch.v.03"},{"index":8,"sense_key":"see%2:38:00::","synset_id":"meet.v.01"},{"index":9,"sense_key":"see%2:32:00::","synset_id":"determine.v.08"},{"index":10,"sense_key":"see%2:31:03::","synset_id":"see.v.10"},{"index":11,"sense_key":"see%2:41:11::","synset_id":"see.v.11"},{"index":12,"sense_key":"see%2:41:12::","synset_id":"see.v.12"},{"index":13,"sense_key":"see%2:41:10::","synset_id":"visit.v.01"},{"index":14,"sense_key":"see%2:41:14::","synset_id":"attend.v.02"},{"index":15,"sense_key":"see%2:35:09::","synset_id":"see.v.15"},{"index":16,"sense_key":"see%2:41:00::","synset_id":"go_steady.v.01"},{"index":17,"sense_key":"see%2:39:05::","synset_id":"see.v.17"},{"index":18,"sense_key":"see%2:31:08::","synset_id":"see.v.18"},{"index":19,"sense_key":"see%2:39:11::","synset_id":"see.v.19"},{"index":20,"sense_key":"see%2:39:13::","synset_id":"examine.v.02"},{"index":21,"sense_key":"see%2:39:12::","synset_id":"experience.v.01"},{"index":22,"sense_key":"see%2:38:03::","synset_id":"see.v.22"},{"index":23,"sense_key":"see%2:33:00::","synset_id":"see.v.23"},{"index":24,"sense_key":"see%2:31:13::","synset_id":"interpret.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"see%2:39:00::","call_ids":["senseval3.d000.s076.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"see%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1970,"cached_input_tokens":0,"output_tokens":50,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0044399999999999995,"input_uncached_usd":0.00394,"input_cached_usd":0.0,"output_usd":0.0005,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7354996660724282}
{"item_id":"senseval3.d000.s077.t000","gold_sense_keys":["lobby%1:06:00::"],"candidates":[{"index":1,"sense_key":"lobby%1:06:00::","synset_id":"anteroom.n.01"},{"index":2,"sense_key":"lobby%1:14:01::","synset_id":"lobby.n.02"},{"index":3,"sense_key":"lobby%1:14:00::","synset_id":"lobby.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"lobby%1:06:00::","call_ids":["senseval3.d000.s077.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"lobby%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":558,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0012360000000000001,"input_uncached_usd":0.001116,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.034172625048086}
{"item_id":"senseval3.d000.s077.t001","gold_sense_keys":["people%1:14:00::"],"candidates":[{"index":1,"sense_key":"people%1:14:00::","synset_id":"people.n.01"},{"index":2,"sense_key":"people%1:14:01::","synset_id":"citizenry.n.01"},{"index":3,"sense_key":"people%1:14:02::","synset_id":"people.n.03"},{"index":4,"sense_key":"people%1:14:03::","synset_id":"multitude.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"people%1:14:00::","call_ids":["senseval3.d000.s077.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"people%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":629,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001378,"input_uncached_usd":0.001258,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.081739875022322}
{"item_id":"senseval3.d000.s077.t002","gold_sense_keys":["stand%2:35:00::"],"candidates":[{"index":1,"sense_key":"stand%2:35:00::","synset_id":"stand.v.01"},{"index":2,"sense_key":"stand%2:42:04::","synset_id":"stand.v.02"},{"index":3,"sense_key":"stand%2:42:03::","synset_id":"stand.v.03"},{"index":4,"sense_key":"stand%2:42:00::","synset_id":"stand.v.04"},{"index":5,"sense_key":"stand%2:31:00::","synset_id":"digest.v.03"},{"index":6,"sense_key":"stand%2:31:01::","synset_id":"stand.v.06"},{"index":7,"sense_key":"stand%2:42:06::","synset_id":"stand.v.07"},{"index":8,"sense_key":"stand%2:42:02::","synset_id":"stand.v.08"},{"index":9,"sense_key":"stand%2:42:01::","synset_id":"stand.v.09"},{"index":10,"sense_key":"stand%2:35:01::","synset_id":"stand.v.10"},{"index":11,"sense_key":"stand%2:33:00::","synset_id":"resist.v.04"},{"index":12,"sense_key":"stand%2:35:10::","synset_id":"stand.v.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"stand%2:35:00::","call_ids":["senseval3.d000.s077.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"stand%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1050,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0022199999999999998,"input_uncached_usd":0.0021,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4250667919404805}
{"item_id":"senseval3.d000.s077.t003","gold_sense_keys":["talk%2:32:01::"],"candidates":[{"index":1,"sense_key":"talk%2:32:01::","synset_id":"talk.v.01"},{"index":2,"sense_key":"talk%2:32:00::","synset_id":"talk.v.02"},{"index":3,"sense_key":"talk%2:32:03::","synset_id":"speak.v.03"},{"index":4,"sense_key":"talk%2:32:02::","synset_id":"spill.v.05"},{"index":5,"sense_key":"talk%2:32:06::","synset_id":"spill_the_beans.v.01"},{"index":6,"sense_key":"talk%2:32:05::","synset_id":"lecture.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"talk%2:32:01::","call_ids":["senseval3.d000.s077.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"talk%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":806,"cached_input_tokens":0,"output_tokens":52,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002132,"input_uncached_usd":0.001612,"input_cached_usd":0.0,"output_usd":0.0005200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5813772910041735}
{"item_id":"senseval3.d000.s078.t000","gold_sense_keys":["spine%1:08:00::"],"candidates":[{"index":1,"sense_key":"spine%1:08:00::","synset_id":"spinal_column.n.01"},{"index":2,"sense_key":"spine%1:25:00::","synset_id":"spur.n.02"},{"index":3,"sense_key":"spine%1:20:00::","synset_id":"spine.n.03"},{"index":4,"sense_key":"spine%1:06:00::","synset_id":"spine.n.04"},{"index":5,"sense_key":"spine%1:05:00::","synset_id":"spine.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"spine%1:08:00::","call_ids":["senseval3.d000.s078.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"spine%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":743,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001606,"input_uncached_usd":0.001486,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0872903330018744}
{"item_id":"senseval3.d000.s078.t001","gold_sense_keys":["crawl%2:42:01::"],"candidates":[{"index":1,"sense_key":"crawl%2:38:00::","synset_id":"crawl.v.01"},{"index":2,"sense_key":"crawl%2:42:01::","synset_id":"crawl.v.02"},{"index":3,"sense_key":"crawl%2:42:00::","synset_id":"crawl.v.03"},{"index":4,"sense_key":"crawl%2:38:03::","synset_id":"fawn.v.01"},{"index":5,"sense_key":"crawl%2:38:01::","synset_id":"crawl.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"crawl%2:42:01::","call_ids":["senseval3.d000.s078.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"crawl%2:42:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":664,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001448,"input_uncached_usd":0.001328,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8757083329837769}
{"item_id":"senseval3.d000.s078.t003","gold_sense_keys":["premonition%1:12:00::"],"candidates":[{"index":1,"sense_key":"premonition%1:12:00::","synset_id":"foreboding.n.01"},{"index":2,"sense_key":"premonition%1:10:00::","synset_id":"forewarning.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"premonition%1:12:00::","call_ids":["senseval3.d000.s078.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"premonition%1:12:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":525,"cached_input_tokens":0,"output_tokens":91,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00196,"input_uncached_usd":0.00105,"input_cached_usd":0.0,"output_usd":0.0009100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8306512091076002}
{"item_id":"senseval3.d000.s078.t004","gold_sense_keys":["ask%2:32:04::"],"candidates":[{"index":1,"sense_key":"ask%2:32:00::","synset_id":"ask.v.01"},{"index":2,"sense_key":"ask%2:32:01::","synset_id":"ask.v.02"},{"index":3,"sense_key":"ask%2:32:02::","synset_id":"ask.v.03"},{"index":4,"sense_key":"ask%2:32:05::","synset_id":"ask.v.04"},{"index":5,"sense_key":"ask%2:32:04::","synset_id":"ask.v.05"},{"index":6,"sense_key":"ask%2:42:00::","synset_id":"necessitate.v.01"},{"index":7,"sense_key":"ask%2:32:09::","synset_id":"ask.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"ask%2:32:04::","call_ids":["senseval3.d000.s078.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"ask%2:32:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":824,"cached_input_tokens":0,"output_tokens":84,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0024879999999999998,"input_uncached_usd":0.001648,"input_cached_usd":0.0,"output_usd":0.00084,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5144384589511901}
{"item_id":"senseval3.d000.s078.t005","gold_sense_keys":["fellow%1:18:03::"],"candidates":[{"index":1,"sense_key":"fellow%1:18:00::","synset_id":"chap.n.01"},{"index":2,"sense_key":"fellow%1:18:02::","synset_id":"companion.n.01"},{"index":3,"sense_key":"fellow%1:18:03::","synset_id":"colleague.n.02"},{"index":4,"sense_key":"fellow%1:23:00::","synset_id":"mate.n.06"},{"index":5,"sense_key":"fellow%1:18:05::","synset_id":"fellow.n.05"},{"index":6,"sense_key":"fellow%1:18:04::","synset_id":"fellow.n.06"},{"index":7,"sense_key":"fellow%1:18:01::","synset_id":"boyfriend.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"fellow%1:18:03::","call_ids":["senseval3.d000.s078.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"fellow%1:18:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":889,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002508,"input_uncached_usd":0.0017779999999999998,"input_cached_usd":0.0,"output_usd":0.0007300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.295323041966185}
{"item_id":"senseval3.d000.s078.t006","gold_sense_keys":["tenant%1:18:00::"],"candidates":[{"index":1,"sense_key":"tenant%1:18:00::","synset_id":"tenant.n.01"},{"index":2,"sense_key":"tenant%1:18:02::","synset_id":"tenant.n.02"},{"index":3,"sense_key":"tenant%1:18:01::","synset_id":"tenant.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"tenant%1:18:00::","call_ids":["senseval3.d000.s078.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"tenant%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":555,"cached_input_tokens":0,"output_tokens":121,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00232,"input_uncached_usd":0.0011099999999999999,"input_cached_usd":0.0,"output_usd":0.0012100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9427913340041414}
{"item_id":"senseval3.d000.s078.t007","gold_sense_keys":["happen%2:30:00::"],"candidates":[{"index":1,"sense_key":"happen%2:30:00::","synset_id":"happen.v.01"},{"index":2,"sense_key":"happen%2:30:02::","synset_id":"happen.v.02"},{"index":3,"sense_key":"happen%2:41:00::","synset_id":"happen.v.03"},{"index":4,"sense_key":"happen%2:30:01::","synset_id":"happen.v.04"},{"index":5,"sense_key":"happen%2:40:12::","synset_id":"find.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"happen%2:30:00::","call_ids":["senseval3.d000.s078.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"happen%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":735,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00207,"input_uncached_usd":0.00147,"input_cached_usd":0.0,"output_usd":0.0006000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.439583082916215}
{"item_id":"senseval3.d000.s079.t001","gold_sense_keys":["die%2:30:00::"],"candidates":[{"index":1,"sense_key":"die%2:30:00::","synset_id":"die.v.01"},{"index":2,"sense_key":"die%2:39:00::","synset_id":"die.v.02"},{"index":3,"sense_key":"die%2:37:00::","synset_id":"die.v.03"},{"index":4,"sense_key":"die%2:30:04::","synset_id":"fail.v.04"},{"index":5,"sense_key":"die%2:37:03::","synset_id":"die.v.05"},{"index":6,"sense_key":"die%2:37:01::","synset_id":"die.v.06"},{"index":7,"sense_key":"die%2:35:00::","synset_id":"die.v.07"},{"index":8,"sense_key":"die%2:33:00::","synset_id":"die.v.08"},{"index":9,"sense_key":"die%2:30:05::","synset_id":"die.v.09"},{"index":10,"sense_key":"die%2:30:01::","synset_id":"die.v.10"},{"index":11,"sense_key":"die%2:30:02::","synset_id":"die.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"die%2:30:00::","call_ids":["senseval3.d000.s079.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"die%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1059,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002238,"input_uncached_usd":0.002118,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5046826669713482}
{"item_id":"senseval3.d000.s080.t000","gold_sense_keys":["afternoon%1:28:00::"],"candidates":[{"index":1,"sense_key":"afternoon%1:28:00::","synset_id":"afternoon.n.01"},{"index":2,"sense_key":"afternoon%1:10:00::","synset_id":"good_afternoon.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"afternoon%1:28:00::","call_ids":["senseval3.d000.s080.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"afternoon%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":517,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001154,"input_uncached_usd":0.001034,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4290440829936415}
{"item_id":"senseval3.d000.s080.t001","gold_sense_keys":["seem%2:39:01::"],"candidates":[{"index":1,"sense_key":"seem%2:39:00::","synset_id":"look.v.02"},{"index":2,"sense_key":"seem%2:39:01::","synset_id":"appear.v.04"},{"index":3,"sense_key":"seem%2:42:00::","synset_id":"seem.v.03"},{"index":4,"sense_key":"seem%2:39:02::","synset_id":"seem.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"seem%2:39:01::","call_ids":["senseval3.d000.s080.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"seem%2:39:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":671,"cached_input_tokens":0,"output_tokens":85,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002192,"input_uncached_usd":0.0013419999999999999,"input_cached_usd":0.0,"output_usd":0.0008500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6194963339949027}
{"item_id":"senseval3.d000.s080.t002","gold_sense_keys":["fall%2:38:03::"],"candidates":[{"index":1,"sense_key":"fall%2:38:03::","synset_id":"fall.v.01"},{"index":2,"sense_key":"fall%2:38:00::","synset_id":"descend.v.01"},{"index":3,"sense_key":"fall%2:30:08::","synset_id":"fall.v.03"},{"index":4,"sense_key":"fall%2:42:03::","synset_id":"fall.v.04"},{"index":5,"sense_key":"fall%2:43:00::","synset_id":"precipitate.v.03"},{"index":6,"sense_key":"fall%2:41:02::","synset_id":"fall.v.06"},{"index":7,"sense_key":"fall%2:42:00::","synset_id":"fall.v.07"},{"index":8,"sense_key":"fall%2:42:04::","synset_id":"fall.v.08"},{"index":9,"sense_key":"fall%2:40:05::","synset_id":"fall.v.09"},{"index":10,"sense_key":"fall%2:30:00::","synset_id":"fall.v.10"},{"index":11,"sense_key":"fall%2:30:06::","synset_id":"decrease.v.01"},{"index":12,"sense_key":"fall%2:41:01::","synset_id":"fall.v.12"},{"index":13,"sense_key":"fall%2:41:00::","synset_id":"fall.v.13"},{"index":14,"sense_key":"fall%2:40:03::","synset_id":"fall.v.14"},{"index":15,"sense_key":"fall%2:38:01::","synset_id":"fall.v.15"},{"index":16,"sense_key":"fall%2:42:02::","synset_id":"fall.v.16"},{"index":17,"sense_key":"fall%2:41:15::","synset_id":"fall.v.17"},{"index":18,"sense_key":"fall%2:40:12::","synset_id":"fall.v.18"},{"index":19,"sense_key":"fall%2:40:09::","synset_id":"accrue.v.02"},{"index":20,"sense_key":"fall%2:40:08::","synset_id":"fall.v.20"},{"index":21,"sense_key":"fall%2:40:13::","synset_id":"fall.v.21"},{"index":22,"sense_key":"fall%2:38:11::","synset_id":"fall.v.22"},{"index":23,"sense_key":"fall%2:38:15::","synset_id":"fall.v.23"},{"index":24,"sense_key":"fall%2:38:14::","synset_id":"fall.v.24"},{"index":25,"sense_key":"fall%2:35:00::","synset_id":"hang.v.05"},{"index":26,"sense_key":"fall%2:30:11::","synset_id":"fall.v.26"},{"index":27,"sense_key":"fall%2:30:14::","synset_id":"fall.v.27"},{"index":28,"sense_key":"fall%2:30:10::","synset_id":"fall.v.28"},{"index":29,"sense_key":"fall%2:30:09::","synset_id":"fall.v.29"},{"index":30,"sense_key":"fall%2:30:12::","synset_id":"fall.v.30"},{"index":31,"sense_key":"fall%2:30:15::","synset_id":"fall.v.31"},{"index":32,"sense_key":"fall%2:30:13::","synset_id":"fall.v.32"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"fall%2:38:03::","call_ids":["senseval3.d000.s080.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"fall%2:38:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2201,"cached_input_tokens":0,"output_tokens":70,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.005102,"input_uncached_usd":0.004402,"input_cached_usd":0.0,"output_usd":0.0007000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4451037079561502}
{"item_id":"senseval3.d000.s080.t003","gold_sense_keys":["roof%1:06:00::"],"candidates":[{"index":1,"sense_key":"roof%1:06:00::","synset_id":"roof.n.01"},{"index":2,"sense_key":"roof%1:06:01::","synset_id":"roof.n.02"},{"index":3,"sense_key":"roof%1:17:00::","synset_id":"roof.n.03"},{"index":4,"sense_key":"roof%1:10:00::","synset_id":"ceiling.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"roof%1:06:00::","call_ids":["senseval3.d000.s080.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"roof%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":629,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001378,"input_uncached_usd":0.001258,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2366725830361247}
{"item_id":"senseval3.d000.s080.t004","gold_sense_keys":["obscure%5:00:00:unclear:00"],"candidates":[{"index":1,"sense_key":"obscure%5:00:00:unclear:00","synset_id":"obscure.s.01"},{"index":2,"sense_key":"obscure%5:00:00:incomprehensible:00","synset_id":"dark.s.06"},{"index":3,"sense_key":"obscure%5:00:00:concealed:00","synset_id":"hidden.s.03"},{"index":4,"sense_key":"obscure%5:00:00:inglorious:00","synset_id":"obscure.s.04"},{"index":5,"sense_key":"obscure%5:00:00:inconspicuous:00","synset_id":"obscure.s.05"},{"index":6,"sense_key":"obscure%5:00:00:unconnected:00","synset_id":"apart.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"obscure%5:00:00:unclear:00","call_ids":["senseval3.d000.s080.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"obscure%5:00:00:unclear:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":917,"cached_input_tokens":0,"output_tokens":93,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002764,"input_uncached_usd":0.001834,"input_cached_usd":0.0,"output_usd":0.00093,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.973807624890469}
{"item_id":"senseval3.d000.s082.t001","gold_sense_keys":["witness%2:39:00::"],"candidates":[{"index":1,"sense_key":"witness%2:39:00::","synset_id":"witness.v.01"},{"index":2,"sense_key":"witness%2:39:01::","synset_id":"witness.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"witness%2:39:00::","call_ids":["senseval3.d000.s082.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"witness%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":538,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001196,"input_uncached_usd":0.001076,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0503426250070333}
{"item_id":"senseval3.d000.s082.t002","gold_sense_keys":["fall%1:11:00::"],"candidates":[{"index":1,"sense_key":"fall%1:28:00::","synset_id":"fall.n.01"},{"index":2,"sense_key":"fall%1:04:01::","synset_id":"spill.n.04"},{"index":3,"sense_key":"fall%1:11:03::","synset_id":"fall.n.03"},{"index":4,"sense_key":"fall%1:17:00::","synset_id":"descent.n.05"},{"index":5,"sense_key":"fall%1:04:03::","synset_id":"fall.n.05"},{"index":6,"sense_key":"fall%1:11:01::","synset_id":"fall.n.06"},{"index":7,"sense_key":"fall%1:11:04::","synset_id":"fall.n.07"},{"index":8,"sense_key":"fall%1:04:02::","synset_id":"capitulation.n.03"},{"index":9,"sense_key":"fall%1:28:01::","synset_id":"twilight.n.01"},{"index":10,"sense_key":"fall%1:11:02::","synset_id":"fall.n.10"},{"index":11,"sense_key":"fall%1:11:00::","synset_id":"drop.n.06"},{"index":12,"sense_key":"fall%1:07:00::","synset_id":"drop.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"fall%1:11:00::","call_ids":["senseval3.d000.s082.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"fall%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1174,"cached_input_tokens":0,"output_tokens":161,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003958,"input_uncached_usd":0.002348,"input_cached_usd":0.0,"output_usd":0.00161,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.153025959036313}
{"item_id":"senseval3.d000.s082.t004","gold_sense_keys":["impact%1:11:00::"],"candidates":[{"index":1,"sense_key":"impact%1:11:00::","synset_id":"impact.n.01"},{"index":2,"sense_key":"impact%1:19:00::","synset_id":"impact.n.02"},{"index":3,"sense_key":"impact%1:04:00::","synset_id":"impingement.n.01"},{"index":4,"sense_key":"impact%1:04:01::","synset_id":"shock.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"impact%1:11:00::","call_ids":["senseval3.d000.s082.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"impact%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":639,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001398,"input_uncached_usd":0.0012779999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0042564169270918}
{"item_id":"senseval3.d000.s082.t005","gold_sense_keys":["body%1:08:00::"],"candidates":[{"index":1,"sense_key":"body%1:08:00::","synset_id":"body.n.01"},{"index":2,"sense_key":"body%1:14:00::","synset_id":"body.n.02"},{"index":3,"sense_key":"body%1:08:02::","synset_id":"body.n.03"},{"index":4,"sense_key":"body%1:17:00::","synset_id":"body.n.04"},{"index":5,"sense_key":"body%1:08:01::","synset_id":"torso.n.01"},{"index":6,"sense_key":"body%1:14:02::","synset_id":"body.n.06"},{"index":7,"sense_key":"body%1:07:00::","synset_id":"consistency.n.01"},{"index":8,"sense_key":"body%1:10:00::","synset_id":"body.n.08"},{"index":9,"sense_key":"body%1:07:01::","synset_id":"body.n.09"},{"index":10,"sense_key":"body%1:06:01::","synset_id":"soundbox.n.01"},{"index":11,"sense_key":"body%1:06:00::","synset_id":"body.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"body%1:08:00::","call_ids":["senseval3.d000.s082.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"body%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1033,"cached_input_tokens":0,"output_tokens":218,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004246,"input_uncached_usd":0.0020659999999999997,"input_cached_usd":0.0,"output_usd":0.00218,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.5232786670094356}
{"item_id":"senseval3.d000.s082.t006","gold_sense_keys":["smash%2:35:05::"],"candidates":[{"index":1,"sense_key":"smash%2:35:00::","synset_id":"smash.v.01"},{"index":2,"sense_key":"smash%2:30:00::","synset_id":"smash.v.02"},{"index":3,"sense_key":"smash%2:40:00::","synset_id":"bankrupt.v.01"},{"index":4,"sense_key":"smash%2:38:02::","synset_id":"smash.v.04"},{"index":5,"sense_key":"smash%2:37:00::","synset_id":"crush.v.05"},{"index":6,"sense_key":"smash%2:35:08::","synset_id":"bang_up.v.01"},{"index":7,"sense_key":"smash%2:35:06::","synset_id":"smash.v.07"},{"index":8,"sense_key":"smash%2:35:05::","synset_id":"smash.v.08"},{"index":9,"sense_key":"smash%2:33:00::","synset_id":"smash.v.09"},{"index":10,"sense_key":"smash%2:30:10::","synset_id":"smash.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"smash%2:35:05::","call_ids":["senseval3.d000.s082.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"smash%2:35:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":995,"cached_input_tokens":0,"output_tokens":147,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0034600000000000004,"input_uncached_usd":0.00199,"input_cached_usd":0.0,"output_usd":0.0014700000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8303587909322232}
{"item_id":"senseval3.d000.s082.t007","gold_sense_keys":["pavement%1:06:00::","pavement%1:06:01::"],"candidates":[{"index":1,"sense_key":"pavement%1:06:00::","synset_id":"pavement.n.01"},{"index":2,"sense_key":"pavement%1:27:00::","synset_id":"paving.n.01"},{"index":3,"sense_key":"pavement%1:06:01::","synset_id":"sidewalk.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"pavement%1:06:00::","call_ids":["senseval3.d000.s082.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"pavement%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":566,"cached_input_tokens":0,"output_tokens":156,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002692,"input_uncached_usd":0.001132,"input_cached_usd":0.0,"output_usd":0.0015600000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9069541250355542}
{"item_id":"senseval3.d000.s082.t008","gold_sense_keys":["basement%1:06:00::"],"candidates":[{"index":1,"sense_key":"basement%1:06:00::","synset_id":"basement.n.01"},{"index":2,"sense_key":"basement%1:06:01::","synset_id":"basement.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"basement%1:06:00::","call_ids":["senseval3.d000.s082.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"basement%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":516,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001152,"input_uncached_usd":0.001032,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9698134590871632}
{"item_id":"senseval3.d000.s082.t009","gold_sense_keys":["delivery%1:04:04::"],"candidates":[{"index":1,"sense_key":"delivery%1:04:04::","synset_id":"delivery.n.01"},{"index":2,"sense_key":"delivery%1:11:00::","synset_id":"delivery.n.02"},{"index":3,"sense_key":"delivery%1:10:00::","synset_id":"manner_of_speaking.n.01"},{"index":4,"sense_key":"delivery%1:04:01::","synset_id":"delivery.n.04"},{"index":5,"sense_key":"delivery%1:04:03::","synset_id":"pitch.n.02"},{"index":6,"sense_key":"delivery%1:04:00::","synset_id":"rescue.n.01"},{"index":7,"sense_key":"delivery%1:04:02::","synset_id":"delivery.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"delivery%1:04:04::","call_ids":["senseval3.d000.s082.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"delivery%1:04:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":858,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002356,"input_uncached_usd":0.0017159999999999999,"input_cached_usd":0.0,"output_usd":0.00064,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7302480830112472}
{"item_id":"senseval3.d000.s082.t010","gold_sense_keys":["entrance%1:06:00::"],"candidates":[{"index":1,"sense_key":"entrance%1:06:00::","synset_id":"entrance.n.01"},{"index":2,"sense_key":"entrance%1:11:00::","synset_id":"entrance.n.02"},{"index":3,"sense_key":"entrance%1:04:00::","synset_id":"entrance.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"entrance%1:06:00::","call_ids":["senseval3.d000.s082.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"entrance%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":607,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0013340000000000001,"input_uncached_usd":0.001214,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1128762919688597}
{"item_id":"senseval3.d000.s083.t002","gold_sense_keys":["notice%2:39:00::"],"candidates":[{"index":1,"sense_key":"notice%2:39:04::","synset_id":"detect.v.01"},{"index":2,"sense_key":"notice%2:39:00::","synset_id":"notice.v.02"},{"index":3,"sense_key":"notice%2:32:00::","synset_id":"comment.v.01"},{"index":4,"sense_key":"notice%2:32:01::","synset_id":"notice.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"notice%2:39:00::","call_ids":["senseval3.d000.s083.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"notice%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":719,"cached_input_tokens":0,"output_tokens":115,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002588,"input_uncached_usd":0.001438,"input_cached_usd":0.0,"output_usd":0.0011500000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1524847919354215}
{"item_id":"senseval3.d000.s083.t005","gold_sense_keys":["feel%2:39:00::"],"candidates":[{"index":1,"sense_key":"feel%2:37:00::","synset_id":"feel.v.01"},{"index":2,"sense_key":"feel%2:31:00::","synset_id":"find.v.05"},{"index":3,"sense_key":"feel%2:39:00::","synset_id":"feel.v.03"},{"index":4,"sense_key":"feel%2:29:00::","synset_id":"feel.v.04"},{"index":5,"sense_key":"feel%2:31:01::","synset_id":"feel.v.05"},{"index":6,"sense_key":"feel%2:39:01::","synset_id":"feel.v.06"},{"index":7,"sense_key":"feel%2:42:00::","synset_id":"feel.v.07"},{"index":8,"sense_key":"feel%2:39:10::","synset_id":"feel.v.08"},{"index":9,"sense_key":"feel%2:35:00::","synset_id":"feel.v.09"},{"index":10,"sense_key":"feel%2:35:01::","synset_id":"palpate.v.01"},{"index":11,"sense_key":"feel%2:40:00::","synset_id":"feel.v.11"},{"index":12,"sense_key":"feel%2:39:09::","synset_id":"feel.v.12"},{"index":13,"sense_key":"feel%2:35:11::","synset_id":"feel.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"feel%2:39:00::","call_ids":["senseval3.d000.s083.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"feel%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1216,"cached_input_tokens":0,"output_tokens":120,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003632,"input_uncached_usd":0.0024319999999999997,"input_cached_usd":0.0,"output_usd":0.0012000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6013579999562353}
{"item_id":"senseval3.d000.s083.t006","gold_sense_keys":["blood%1:08:00::"],"candidates":[{"index":1,"sense_key":"blood%1:08:00::","synset_id":"blood.n.01"},{"index":2,"sense_key":"blood%1:07:00::","synset_id":"blood.n.02"},{"index":3,"sense_key":"blood%1:18:00::","synset_id":"rake.n.01"},{"index":4,"sense_key":"blood%1:14:00::","synset_id":"lineage.n.01"},{"index":5,"sense_key":"blood%1:14:01::","synset_id":"blood.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"blood%1:08:00::","call_ids":["senseval3.d000.s083.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"blood%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":753,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001626,"input_uncached_usd":0.001506,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.123859916930087}
{"item_id":"senseval3.d000.s083.t007","gold_sense_keys":["drain%2:38:00::"],"candidates":[{"index":1,"sense_key":"drain%2:38:00::","synset_id":"drain.v.01"},{"index":2,"sense_key":"drain%2:34:00::","synset_id":"drain.v.02"},{"index":3,"sense_key":"drain%2:30:01::","synset_id":"drain.v.03"},{"index":4,"sense_key":"drain%2:30:04::","synset_id":"enfeeble.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"drain%2:38:00::","call_ids":["senseval3.d000.s083.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"drain%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":622,"cached_input_tokens":0,"output_tokens":115,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0023940000000000003,"input_uncached_usd":0.0012439999999999999,"input_cached_usd":0.0,"output_usd":0.0011500000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.137574875028804}
{"item_id":"senseval3.d000.s083.t008","gold_sense_keys":["cheek%1:08:00::"],"candidates":[{"index":1,"sense_key":"cheek%1:08:00::","synset_id":"cheek.n.01"},{"index":2,"sense_key":"cheek%1:10:00::","synset_id":"impudence.n.01"},{"index":3,"sense_key":"cheek%1:08:01::","synset_id":"buttock.n.01"},{"index":4,"sense_key":"cheek%1:07:00::","synset_id":"boldness.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cheek%1:08:00::","call_ids":["senseval3.d000.s083.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cheek%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":640,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0014,"input_uncached_usd":0.0012799999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9802128329174593}
{"item_id":"senseval3.d000.s084.t000","gold_sense_keys":["mutter%2:32:01::"],"candidates":[{"index":1,"sense_key":"mutter%2:32:01::","synset_id":"mumble.v.01"},{"index":2,"sense_key":"mutter%2:32:00::","synset_id":"murmur.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"mutter%2:32:01::","call_ids":["senseval3.d000.s084.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"mutter%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":558,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001666,"input_uncached_usd":0.001116,"input_cached_usd":0.0,"output_usd":0.00055,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4395512080518529}
{"item_id":"senseval3.d000.s084.t001","gold_sense_keys":["terrible%5:00:00:alarming:00"],"candidates":[{"index":1,"sense_key":"terrible%5:00:00:alarming:00","synset_id":"awful.s.02"},{"index":2,"sense_key":"terrible%5:00:01:bad:00","synset_id":"atrocious.s.02"},{"index":3,"sense_key":"terrible%5:00:00:intense:00","synset_id":"severe.s.01"},{"index":4,"sense_key":"terrible%5:00:00:extraordinary:00","synset_id":"frightful.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"terrible%5:00:00:intense:00","call_ids":["senseval3.d000.s084.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"terrible%5:00:00:intense:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":755,"cached_input_tokens":0,"output_tokens":214,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0036499999999999996,"input_uncached_usd":0.0015099999999999998,"input_cached_usd":0.0,"output_usd":0.00214,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.8774089590879157}
{"item_id":"senseval3.d000.s084.t002","gold_sense_keys":["walk%2:38:00::"],"candidates":[{"index":1,"sense_key":"walk%2:38:00::","synset_id":"walk.v.01"},{"index":2,"sense_key":"walk%2:38:01::","synset_id":"walk.v.02"},{"index":3,"sense_key":"walk%2:33:00::","synset_id":"walk.v.03"},{"index":4,"sense_key":"walk%2:38:03::","synset_id":"walk.v.04"},{"index":5,"sense_key":"walk%2:33:01::","synset_id":"walk.v.05"},{"index":6,"sense_key":"walk%2:41:00::","synset_id":"walk.v.06"},{"index":7,"sense_key":"walk%2:41:01::","synset_id":"walk.v.07"},{"index":8,"sense_key":"walk%2:38:02::","synset_id":"walk.v.08"},{"index":9,"sense_key":"walk%2:38:05::","synset_id":"walk.v.09"},{"index":10,"sense_key":"walk%2:38:04::","synset_id":"walk.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"walk%2:38:00::","call_ids":["senseval3.d000.s084.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"walk%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":904,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001928,"input_uncached_usd":0.001808,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9556497080484405}
{"item_id":"senseval3.d000.s084.t003","gold_sense_keys":["deliberate%5:00:00:unhurried:00"],"candidates":[{"index":1,"sense_key":"deliberate%5:00:00:intended:00","synset_id":"deliberate.s.01"},{"index":2,"sense_key":"deliberate%5:00:00:unhurried:00","synset_id":"careful.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"deliberate%5:00:00:unhurried:00","call_ids":["senseval3.d000.s084.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"deliberate%5:00:00:unhurried:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":568,"cached_input_tokens":0,"output_tokens":211,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003246,"input_uncached_usd":0.0011359999999999999,"input_cached_usd":0.0,"output_usd":0.0021100000000000003,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":4.4320503330091015}
{"item_id":"senseval3.d000.s084.t004","gold_sense_keys":["slowness%1:07:00::"],"candidates":[{"index":1,"sense_key":"slowness%1:09:02::","synset_id":"awkwardness.n.01"},{"index":2,"sense_key":"slowness%1:07:00::","synset_id":"slowness.n.02"},{"index":3,"sense_key":"slowness%1:09:01::","synset_id":"retardation.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"slowness%1:07:00::","call_ids":["senseval3.d000.s084.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"slowness%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":625,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0013700000000000001,"input_uncached_usd":0.00125,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.141021459014155}
{"item_id":"senseval3.d000.s084.t005","gold_sense_keys":["elevator%1:06:00::"],"candidates":[{"index":1,"sense_key":"elevator%1:06:00::","synset_id":"elevator.n.01"},{"index":2,"sense_key":"elevator%1:06:01::","synset_id":"elevator.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"elevator%1:06:00::","call_ids":["senseval3.d000.s084.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"elevator%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":556,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001232,"input_uncached_usd":0.001112,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4836905840784311}
{"item_id":"senseval3.d000.s085.t001","gold_sense_keys":["pour%2:38:04::"],"candidates":[{"index":1,"sense_key":"pour%2:38:03::","synset_id":"pour.v.01"},{"index":2,"sense_key":"pour%2:38:00::","synset_id":"pour.v.02"},{"index":3,"sense_key":"pour%2:38:04::","synset_id":"decant.v.01"},{"index":4,"sense_key":"pour%2:38:02::","synset_id":"pour.v.04"},{"index":5,"sense_key":"pour%2:40:00::","synset_id":"pour.v.05"},{"index":6,"sense_key":"pour%2:43:00::","synset_id":"pour.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"pour%2:38:03::","call_ids":["senseval3.d000.s085.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"pour%2:38:03::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":774,"cached_input_tokens":0,"output_tokens":203,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003578,"input_uncached_usd":0.001548,"input_cached_usd":0.0,"output_usd":0.00203,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.3380539580248296}
{"item_id":"senseval3.d000.s085.t002","gold_sense_keys":["drink%1:13:04::"],"candidates":[{"index":1,"sense_key":"drink%1:13:04::","synset_id":"drink.n.01"},{"index":2,"sense_key":"drink%1:04:00::","synset_id":"drink.n.02"},{"index":3,"sense_key":"drink%1:13:00::","synset_id":"beverage.n.01"},{"index":4,"sense_key":"drink%1:17:00::","synset_id":"drink.n.04"},{"index":5,"sense_key":"drink%1:04:01::","synset_id":"swallow.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"drink%1:13:04::","call_ids":["senseval3.d000.s085.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"drink%1:13:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":760,"cached_input_tokens":0,"output_tokens":81,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00233,"input_uncached_usd":0.0015199999999999999,"input_cached_usd":0.0,"output_usd":0.0008100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4078201670199633}
{"item_id":"senseval3.d000.s085.t004","gold_sense_keys":["hand%1:08:00::"],"candidates":[{"index":1,"sense_key":"hand%1:08:00::","synset_id":"hand.n.01"},{"index":2,"sense_key":"hand%1:18:00::","synset_id":"hired_hand.n.01"},{"index":3,"sense_key":"hand%1:10:00::","synset_id":"handwriting.n.01"},{"index":4,"sense_key":"hand%1:09:00::","synset_id":"hand.n.04"},{"index":5,"sense_key":"hand%1:15:00::","synset_id":"hand.n.05"},{"index":6,"sense_key":"hand%1:14:00::","synset_id":"hand.n.06"},{"index":7,"sense_key":"hand%1:09:01::","synset_id":"hand.n.07"},{"index":8,"sense_key":"hand%1:06:00::","synset_id":"hand.n.08"},{"index":9,"sense_key":"hand%1:23:00::","synset_id":"hand.n.09"},{"index":10,"sense_key":"hand%1:18:02::","synset_id":"hand.n.10"},{"index":11,"sense_key":"hand%1:18:01::","synset_id":"bridge_player.n.01"},{"index":12,"sense_key":"hand%1:10:01::","synset_id":"hand.n.12"},{"index":13,"sense_key":"hand%1:05:01::","synset_id":"hand.n.13"},{"index":14,"sense_key":"hand%1:04:00::","synset_id":"hand.n.14"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"hand%1:08:00::","call_ids":["senseval3.d000.s085.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"hand%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1281,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002682,"input_uncached_usd":0.002562,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6770683750510216}
{"item_id":"senseval3.d000.s085.t005","gold_sense_keys":["flop%2:38:01::"],"candidates":[{"index":1,"sense_key":"flop%2:38:01::","synset_id":"flop.v.01"},{"index":2,"sense_key":"flop%2:38:00::","synset_id":"flop.v.02"},{"index":3,"sense_key":"flop%2:41:00::","synset_id":"fall_through.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"flop%2:38:01::","call_ids":["senseval3.d000.s085.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"flop%2:38:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":575,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00127,"input_uncached_usd":0.00115,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4458072090055794}
{"item_id":"senseval3.d000.s085.t006","gold_sense_keys":["chair%1:06:00::"],"candidates":[{"index":1,"sense_key":"chair%1:06:00::","synset_id":"chair.n.01"},{"index":2,"sense_key":"chair%1:04:00::","synset_id":"professorship.n.01"},{"index":3,"sense_key":"chair%1:18:00::","synset_id":"president.n.04"},{"index":4,"sense_key":"chair%1:06:01::","synset_id":"electric_chair.n.01"},{"index":5,"sense_key":"chair%1:06:02::","synset_id":"chair.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"chair%1:06:00::","call_ids":["senseval3.d000.s085.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"chair%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":758,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001636,"input_uncached_usd":0.001516,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1172887498978525}
{"item_id":"senseval3.d000.s086.t002","gold_sense_keys":["feel%2:29:00::","feel%2:37:00::"],"candidates":[{"index":1,"sense_key":"feel%2:37:00::","synset_id":"feel.v.01"},{"index":2,"sense_key":"feel%2:31:00::","synset_id":"find.v.05"},{"index":3,"sense_key":"feel%2:39:00::","synset_id":"feel.v.03"},{"index":4,"sense_key":"feel%2:29:00::","synset_id":"feel.v.04"},{"index":5,"sense_key":"feel%2:31:01::","synset_id":"feel.v.05"},{"index":6,"sense_key":"feel%2:39:01::","synset_id":"feel.v.06"},{"index":7,"sense_key":"feel%2:42:00::","synset_id":"feel.v.07"},{"index":8,"sense_key":"feel%2:39:10::","synset_id":"feel.v.08"},{"index":9,"sense_key":"feel%2:35:00::","synset_id":"feel.v.09"},{"index":10,"sense_key":"feel%2:35:01::","synset_id":"palpate.v.01"},{"index":11,"sense_key":"feel%2:40:00::","synset_id":"feel.v.11"},{"index":12,"sense_key":"feel%2:39:09::","synset_id":"feel.v.12"},{"index":13,"sense_key":"feel%2:35:11::","synset_id":"feel.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"feel%2:37:00::","call_ids":["senseval3.d000.s086.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"feel%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1208,"cached_input_tokens":0,"output_tokens":226,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004676,"input_uncached_usd":0.0024159999999999997,"input_cached_usd":0.0,"output_usd":0.0022600000000000003,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.8979273330187425}
{"item_id":"senseval3.d000.s086.t003","gold_sense_keys":["bother%2:41:00::"],"candidates":[{"index":1,"sense_key":"bother%2:41:00::","synset_id":"trouble_oneself.v.01"},{"index":2,"sense_key":"bother%2:37:00::","synset_id":"annoy.v.01"},{"index":3,"sense_key":"bother%2:41:01::","synset_id":"trouble.v.02"},{"index":4,"sense_key":"bother%2:38:00::","synset_id":"bother.v.04"},{"index":5,"sense_key":"bother%2:37:03::","synset_id":"bother.v.05"},{"index":6,"sense_key":"bother%2:37:02::","synset_id":"bother.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bother%2:41:00::","call_ids":["senseval3.d000.s086.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bother%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":841,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001802,"input_uncached_usd":0.001682,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.387010833946988}
{"item_id":"senseval3.d000.s086.t004","gold_sense_keys":["ask%2:32:02::"],"candidates":[{"index":1,"sense_key":"ask%2:32:00::","synset_id":"ask.v.01"},{"index":2,"sense_key":"ask%2:32:01::","synset_id":"ask.v.02"},{"index":3,"sense_key":"ask%2:32:02::","synset_id":"ask.v.03"},{"index":4,"sense_key":"ask%2:32:05::","synset_id":"ask.v.04"},{"index":5,"sense_key":"ask%2:32:04::","synset_id":"ask.v.05"},{"index":6,"sense_key":"ask%2:42:00::","synset_id":"necessitate.v.01"},{"index":7,"sense_key":"ask%2:32:09::","synset_id":"ask.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"ask%2:32:02::","call_ids":["senseval3.d000.s086.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"ask%2:32:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":851,"cached_input_tokens":0,"output_tokens":51,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002212,"input_uncached_usd":0.001702,"input_cached_usd":0.0,"output_usd":0.00051,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3020494590746239}
{"item_id":"senseval3.d000.s086.t005","gold_sense_keys":["question%1:10:03::"],"candidates":[{"index":1,"sense_key":"question%1:10:00::","synset_id":"question.n.01"},{"index":2,"sense_key":"question%1:10:01::","synset_id":"question.n.02"},{"index":3,"sense_key":"question%1:10:03::","synset_id":"question.n.03"},{"index":4,"sense_key":"question%1:07:00::","synset_id":"doubt.n.02"},{"index":5,"sense_key":"question%1:10:02::","synset_id":"motion.n.05"},{"index":6,"sense_key":"question%1:10:04::","synset_id":"question.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"question%1:10:03::","call_ids":["senseval3.d000.s086.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"question%1:10:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":852,"cached_input_tokens":0,"output_tokens":126,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002964,"input_uncached_usd":0.001704,"input_cached_usd":0.0,"output_usd":0.00126,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0332012500148267}
{"item_id":"senseval3.d000.s088.t002","gold_sense_keys":["coincidence%1:11:00::"],"candidates":[{"index":1,"sense_key":"coincidence%1:11:00::","synset_id":"coincidence.n.01"},{"index":2,"sense_key":"coincidence%1:07:02::","synset_id":"coincidence.n.02"},{"index":3,"sense_key":"coincidence%1:07:01::","synset_id":"concurrence.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"coincidence%1:11:00::","call_ids":["senseval3.d000.s088.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"coincidence%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":604,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001328,"input_uncached_usd":0.0012079999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1590639998903498}
{"item_id":"senseval3.d000.s089.t000","gold_sense_keys":["stranger%1:18:01::"],"candidates":[{"index":1,"sense_key":"stranger%1:18:00::","synset_id":"stranger.n.01"},{"index":2,"sense_key":"stranger%1:18:01::","synset_id":"stranger.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"stranger%1:18:01::","call_ids":["senseval3.d000.s089.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"stranger%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":486,"cached_input_tokens":0,"output_tokens":131,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002282,"input_uncached_usd":0.000972,"input_cached_usd":0.0,"output_usd":0.0013100000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7694675420643762}
{"item_id":"senseval3.d000.s090.t001","gold_sense_keys":["happy%3:00:00::"],"candidates":[{"index":1,"sense_key":"happy%3:00:00::","synset_id":"happy.a.01"},{"index":2,"sense_key":"happy%5:00:00:fortunate:00","synset_id":"felicitous.s.01"},{"index":3,"sense_key":"happy%5:00:00:willing:00","synset_id":"glad.s.01"},{"index":4,"sense_key":"happy%5:00:00:felicitous:00","synset_id":"happy.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"happy%3:00:00::","call_ids":["senseval3.d000.s090.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"happy%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":640,"cached_input_tokens":0,"output_tokens":125,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00253,"input_uncached_usd":0.0012799999999999999,"input_cached_usd":0.0,"output_usd":0.00125,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9085507079726085}
{"item_id":"senseval3.d000.s090.t002","gold_sense_keys":["lousy%5:00:00:nasty:00"],"candidates":[{"index":1,"sense_key":"lousy%5:00:00:bad:00","synset_id":"icky.s.01"},{"index":2,"sense_key":"lousy%5:00:00:dirty:01","synset_id":"lousy.s.02"},{"index":3,"sense_key":"lousy%5:00:00:nasty:00","synset_id":"dirty.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"lousy%5:00:00:nasty:00","call_ids":["senseval3.d000.s090.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"lousy%5:00:00:nasty:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":587,"cached_input_tokens":0,"output_tokens":166,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002834,"input_uncached_usd":0.001174,"input_cached_usd":0.0,"output_usd":0.0016600000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.5041856670286506}
{"item_id":"senseval3.d000.s091.t000","gold_sense_keys":["man%1:18:00::"],"candidates":[{"index":1,"sense_key":"man%1:18:00::","synset_id":"man.n.01"},{"index":2,"sense_key":"man%1:18:03::","synset_id":"serviceman.n.01"},{"index":3,"sense_key":"man%1:18:04::","synset_id":"man.n.03"},{"index":4,"sense_key":"man%1:05:01::","synset_id":"homo.n.02"},{"index":5,"sense_key":"man%1:18:08::","synset_id":"man.n.05"},{"index":6,"sense_key":"man%1:18:06::","synset_id":"man.n.06"},{"index":7,"sense_key":"man%1:18:05::","synset_id":"valet.n.01"},{"index":8,"sense_key":"man%1:18:07::","synset_id":"man.n.08"},{"index":9,"sense_key":"man%1:15:00::","synset_id":"man.n.09"},{"index":10,"sense_key":"man%1:06:00::","synset_id":"man.n.10"},{"index":11,"sense_key":"man%1:05:00::","synset_id":"world.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"man%1:18:00::","call_ids":["senseval3.d000.s091.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"man%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1123,"cached_input_tokens":0,"output_tokens":47,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0027159999999999997,"input_uncached_usd":0.0022459999999999997,"input_cached_usd":0.0,"output_usd":0.00047000000000000004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8907352919923142}
{"item_id":"senseval3.d000.s091.t001","gold_sense_keys":["have%2:40:00::"],"candidates":[{"index":1,"sense_key":"have%2:40:00::","synset_id":"have.v.01"},{"index":2,"sense_key":"have%2:42:00::","synset_id":"have.v.02"},{"index":3,"sense_key":"have%2:39:06::","synset_id":"experience.v.03"},{"index":4,"sense_key":"have%2:40:04::","synset_id":"own.v.01"},{"index":5,"sense_key":"have%2:30:00::","synset_id":"get.v.03"},{"index":6,"sense_key":"have%2:34:00::","synset_id":"consume.v.02"},{"index":7,"sense_key":"have%2:41:00::","synset_id":"have.v.07"},{"index":8,"sense_key":"have%2:36:00::","synset_id":"hold.v.03"},{"index":9,"sense_key":"have%2:40:01::","synset_id":"have.v.09"},{"index":10,"sense_key":"have%2:42:12::","synset_id":"have.v.10"},{"index":11,"sense_key":"have%2:30:01::","synset_id":"have.v.11"},{"index":12,"sense_key":"have%2:29:02::","synset_id":"have.v.12"},{"index":13,"sense_key":"have%2:32:00::","synset_id":"induce.v.02"},{"index":14,"sense_key":"have%2:40:05::","synset_id":"accept.v.02"},{"index":15,"sense_key":"have%2:40:03::","synset_id":"receive.v.01"},{"index":16,"sense_key":"have%2:29:05::","synset_id":"suffer.v.02"},{"index":17,"sense_key":"have%2:40:02::","synset_id":"have.v.17"},{"index":18,"sense_key":"have%2:29:00::","synset_id":"give_birth.v.01"},{"index":19,"sense_key":"have%2:35:00::","synset_id":"take.v.35"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"have%2:40:00::","call_ids":["senseval3.d000.s091.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"have%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1551,"cached_input_tokens":0,"output_tokens":182,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004922,"input_uncached_usd":0.0031019999999999997,"input_cached_usd":0.0,"output_usd":0.0018200000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0964710830012336}
{"item_id":"senseval3.d000.s091.t002","gold_sense_keys":["family%1:14:00::","family%1:14:02::"],"candidates":[{"index":1,"sense_key":"family%1:14:02::","synset_id":"family.n.01"},{"index":2,"sense_key":"family%1:14:00::","synset_id":"family.n.02"},{"index":3,"sense_key":"family%1:14:04::","synset_id":"class.n.01"},{"index":4,"sense_key":"family%1:14:01::","synset_id":"family.n.04"},{"index":5,"sense_key":"family%1:18:00::","synset_id":"kin.n.01"},{"index":6,"sense_key":"family%1:14:03::","synset_id":"family.n.06"},{"index":7,"sense_key":"family%1:14:06::","synset_id":"syndicate.n.01"},{"index":8,"sense_key":"family%1:14:05::","synset_id":"family.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"family%1:14:00::","call_ids":["senseval3.d000.s091.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"family%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":901,"cached_input_tokens":0,"output_tokens":240,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004202,"input_uncached_usd":0.001802,"input_cached_usd":0.0,"output_usd":0.0024000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.529935083934106}
{"item_id":"senseval3.d000.s091.t003","gold_sense_keys":["suffer%2:29:00::","suffer%2:37:00::"],"candidates":[{"index":1,"sense_key":"suffer%2:39:01::","synset_id":"suffer.v.01"},{"index":2,"sense_key":"suffer%2:29:01::","synset_id":"suffer.v.02"},{"index":3,"sense_key":"suffer%2:37:00::","synset_id":"suffer.v.03"},{"index":4,"sense_key":"suffer%2:31:00::","synset_id":"digest.v.03"},{"index":5,"sense_key":"suffer%2:30:00::","synset_id":"suffer.v.05"},{"index":6,"sense_key":"suffer%2:29:00::","synset_id":"suffer.v.06"},{"index":7,"sense_key":"suffer%2:39:00::","synset_id":"hurt.v.06"},{"index":8,"sense_key":"suffer%2:29:03::","synset_id":"suffer.v.08"},{"index":9,"sense_key":"suffer%2:42:04::","synset_id":"suffer.v.09"},{"index":10,"sense_key":"suffer%2:39:02::","synset_id":"suffer.v.10"},{"index":11,"sense_key":"suffer%2:30:02::","synset_id":"suffer.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"suffer%2:37:00::","call_ids":["senseval3.d000.s091.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"suffer%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1024,"cached_input_tokens":0,"output_tokens":78,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002828,"input_uncached_usd":0.002048,"input_cached_usd":0.0,"output_usd":0.0007800000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4921964589739218}
{"item_id":"senseval3.d000.s092.t000","gold_sense_keys":["say%2:32:00::"],"candidates":[{"index":1,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":6,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":9,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":10,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":11,"sense_key":"say%2:32:09::","synset_id":"say.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"say%2:32:00::","call_ids":["senseval3.d000.s092.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"say%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1028,"cached_input_tokens":0,"output_tokens":162,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0036760000000000004,"input_uncached_usd":0.002056,"input_cached_usd":0.0,"output_usd":0.0016200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.450664790929295}
{"item_id":"senseval3.d000.s092.t001","gold_sense_keys":["kill%2:35:00::"],"candidates":[{"index":1,"sense_key":"kill%2:35:00::","synset_id":"kill.v.01"},{"index":2,"sense_key":"kill%2:41:01::","synset_id":"kill.v.02"},{"index":3,"sense_key":"kill%2:30:08::","synset_id":"stamp_out.v.01"},{"index":4,"sense_key":"kill%2:42:00::","synset_id":"kill.v.04"},{"index":5,"sense_key":"kill%2:39:00::","synset_id":"kill.v.05"},{"index":6,"sense_key":"kill%2:37:00::","synset_id":"kill.v.06"},{"index":7,"sense_key":"kill%2:35:06::","synset_id":"kill.v.07"},{"index":8,"sense_key":"kill%2:35:05::","synset_id":"kill.v.08"},{"index":9,"sense_key":"kill%2:35:02::","synset_id":"kill.v.09"},{"index":10,"sense_key":"kill%2:35:01::","synset_id":"kill.v.10"},{"index":11,"sense_key":"kill%2:34:00::","synset_id":"toss_off.v.02"},{"index":12,"sense_key":"kill%2:30:00::","synset_id":"kill.v.12"},{"index":13,"sense_key":"kill%2:30:04::","synset_id":"kill.v.13"},{"index":14,"sense_key":"kill%2:30:03::","synset_id":"kill.v.14"},{"index":15,"sense_key":"kill%2:30:02::","synset_id":"kill.v.15"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"kill%2:35:00::","call_ids":["senseval3.d000.s092.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"kill%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1193,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0029460000000000003,"input_uncached_usd":0.002386,"input_cached_usd":0.0,"output_usd":0.0005600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2796199589502066}
{"item_id":"senseval3.d000.s092.t002","gold_sense_keys":["man%1:18:00::","man%1:18:04::"],"candidates":[{"index":1,"sense_key":"man%1:18:00::","synset_id":"man.n.01"},{"index":2,"sense_key":"man%1:18:03::","synset_id":"serviceman.n.01"},{"index":3,"sense_key":"man%1:18:04::","synset_id":"man.n.03"},{"index":4,"sense_key":"man%1:05:01::","synset_id":"homo.n.02"},{"index":5,"sense_key":"man%1:18:08::","synset_id":"man.n.05"},{"index":6,"sense_key":"man%1:18:06::","synset_id":"man.n.06"},{"index":7,"sense_key":"man%1:18:05::","synset_id":"valet.n.01"},{"index":8,"sense_key":"man%1:18:07::","synset_id":"man.n.08"},{"index":9,"sense_key":"man%1:15:00::","synset_id":"man.n.09"},{"index":10,"sense_key":"man%1:06:00::","synset_id":"man.n.10"},{"index":11,"sense_key":"man%1:05:00::","synset_id":"world.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"man%1:18:00::","call_ids":["senseval3.d000.s092.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"man%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1110,"cached_input_tokens":0,"output_tokens":132,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0035399999999999997,"input_uncached_usd":0.0022199999999999998,"input_cached_usd":0.0,"output_usd":0.0013200000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8080205420264974}
{"item_id":"senseval3.d000.s092.t003","gold_sense_keys":["favor%1:04:00::"],"candidates":[{"index":1,"sense_key":"favor%1:04:00::","synset_id":"favor.n.01"},{"index":2,"sense_key":"favor%1:07:00::","synset_id":"favor.n.02"},{"index":3,"sense_key":"favor%1:09:00::","synset_id":"favor.n.03"},{"index":4,"sense_key":"favor%1:12:00::","synset_id":"favor.n.04"},{"index":5,"sense_key":"favor%1:06:00::","synset_id":"party_favor.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"favor%1:04:00::","call_ids":["senseval3.d000.s092.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"favor%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":614,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001348,"input_uncached_usd":0.001228,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0705165000399575}
{"item_id":"senseval3.d000.s093.t000","gold_sense_keys":["think%2:31:03::"],"candidates":[{"index":1,"sense_key":"think%2:31:01::","synset_id":"think.v.01"},{"index":2,"sense_key":"think%2:31:03::","synset_id":"think.v.02"},{"index":3,"sense_key":"think%2:31:00::","synset_id":"think.v.03"},{"index":4,"sense_key":"think%2:31:02::","synset_id":"remember.v.01"},{"index":5,"sense_key":"think%2:36:00::","synset_id":"think.v.05"},{"index":6,"sense_key":"think%2:31:09::","synset_id":"think.v.06"},{"index":7,"sense_key":"think%2:31:06::","synset_id":"intend.v.01"},{"index":8,"sense_key":"think%2:31:12::","synset_id":"think.v.08"},{"index":9,"sense_key":"think%2:31:13::","synset_id":"think.v.09"},{"index":10,"sense_key":"think%2:31:15::","synset_id":"think.v.10"},{"index":11,"sense_key":"think%2:31:14::","synset_id":"think.v.11"},{"index":12,"sense_key":"think%2:31:08::","synset_id":"think.v.12"},{"index":13,"sense_key":"think%2:30:00::","synset_id":"think.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"think%2:31:03::","call_ids":["senseval3.d000.s093.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"think%2:31:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1183,"cached_input_tokens":0,"output_tokens":132,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003686,"input_uncached_usd":0.002366,"input_cached_usd":0.0,"output_usd":0.0013200000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5929142909590155}
{"item_id":"senseval3.d000.s093.t001","gold_sense_keys":["mean%2:32:01::"],"candidates":[{"index":1,"sense_key":"mean%2:32:01::","synset_id":"mean.v.01"},{"index":2,"sense_key":"mean%2:42:00::","synset_id":"entail.v.01"},{"index":3,"sense_key":"mean%2:32:03::","synset_id":"mean.v.03"},{"index":4,"sense_key":"mean%2:31:00::","synset_id":"intend.v.01"},{"index":5,"sense_key":"mean%2:42:03::","synset_id":"mean.v.05"},{"index":6,"sense_key":"mean%2:31:01::","synset_id":"think_of.v.04"},{"index":7,"sense_key":"mean%2:31:05::","synset_id":"mean.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"mean%2:32:01::","call_ids":["senseval3.d000.s093.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"mean%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":826,"cached_input_tokens":0,"output_tokens":129,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002942,"input_uncached_usd":0.0016519999999999998,"input_cached_usd":0.0,"output_usd":0.0012900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1998685830039904}
{"item_id":"senseval3.d000.s093.t002","gold_sense_keys":["say%2:32:00::"],"candidates":[{"index":1,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":6,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":9,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":10,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":11,"sense_key":"say%2:32:09::","synset_id":"say.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"say%2:32:00::","call_ids":["senseval3.d000.s093.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"say%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1051,"cached_input_tokens":0,"output_tokens":121,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003312,"input_uncached_usd":0.0021019999999999997,"input_cached_usd":0.0,"output_usd":0.0012100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6335402500117198}
{"item_id":"senseval3.d000.s094.t000","gold_sense_keys":["stranger%1:18:01::"],"candidates":[{"index":1,"sense_key":"stranger%1:18:00::","synset_id":"stranger.n.01"},{"index":2,"sense_key":"stranger%1:18:01::","synset_id":"stranger.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"stranger%1:18:01::","call_ids":["senseval3.d000.s094.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"stranger%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":494,"cached_input_tokens":0,"output_tokens":82,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0018080000000000001,"input_uncached_usd":0.000988,"input_cached_usd":0.0,"output_usd":0.0008200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5835652499226853}
{"item_id":"senseval3.d000.s094.t001","gold_sense_keys":["eye%1:08:00::"],"candidates":[{"index":1,"sense_key":"eye%1:08:00::","synset_id":"eye.n.01"},{"index":2,"sense_key":"eye%1:09:00::","synset_id":"eye.n.02"},{"index":3,"sense_key":"eye%1:09:01::","synset_id":"eye.n.03"},{"index":4,"sense_key":"eye%1:15:00::","synset_id":"center.n.01"},{"index":5,"sense_key":"eye%1:06:00::","synset_id":"eye.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"eye%1:08:00::","call_ids":["senseval3.d000.s094.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"eye%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":677,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001474,"input_uncached_usd":0.001354,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.205538250040263}
{"item_id":"senseval3.d000.s094.t002","gold_sense_keys":["large%3:00:00::"],"candidates":[{"index":1,"sense_key":"large%3:00:00::","synset_id":"large.a.01"},{"index":2,"sense_key":"large%5:00:00:significant:00","synset_id":"large.s.01"},{"index":3,"sense_key":"large%5:00:00:rhetorical:00","synset_id":"bombastic.s.01"},{"index":4,"sense_key":"large%5:00:00:generous:02","synset_id":"big.s.10"},{"index":5,"sense_key":"large%5:00:00:conspicuous:00","synset_id":"big.s.04"},{"index":6,"sense_key":"large%5:00:00:comprehensive:00","synset_id":"large.s.05"},{"index":7,"sense_key":"large%5:00:00:pregnant:00","synset_id":"big.s.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"large%3:00:00::","call_ids":["senseval3.d000.s094.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"large%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":874,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001868,"input_uncached_usd":0.001748,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0348489169264212}
{"item_id":"senseval3.d000.s094.t003","gold_sense_keys":["sad%3:00:00::"],"candidates":[{"index":1,"sense_key":"sad%3:00:00::","synset_id":"sad.a.01"},{"index":2,"sense_key":"sad%5:00:00:sorrowful:00","synset_id":"sad.s.01"},{"index":3,"sense_key":"sad%5:00:00:bad:00","synset_id":"deplorable.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"sad%3:00:00::","call_ids":["senseval3.d000.s094.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"sad%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":652,"cached_input_tokens":0,"output_tokens":122,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002524,"input_uncached_usd":0.001304,"input_cached_usd":0.0,"output_usd":0.0012200000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0283225000603124}
{"item_id":"senseval3.d000.s094.t004","gold_sense_keys":["hurt%2:37:00::"],"candidates":[{"index":1,"sense_key":"hurt%2:39:00::","synset_id":"ache.v.03"},{"index":2,"sense_key":"hurt%2:29:01::","synset_id":"hurt.v.02"},{"index":3,"sense_key":"hurt%2:37:01::","synset_id":"pain.v.02"},{"index":4,"sense_key":"hurt%2:30:04::","synset_id":"hurt.v.04"},{"index":5,"sense_key":"hurt%2:37:00::","synset_id":"hurt.v.05"},{"index":6,"sense_key":"hurt%2:39:01::","synset_id":"hurt.v.06"},{"index":7,"sense_key":"hurt%2:29:02::","synset_id":"suffer.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"hurt%2:37:00::","call_ids":["senseval3.d000.s094.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"hurt%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":794,"cached_input_tokens":0,"output_tokens":94,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002528,"input_uncached_usd":0.001588,"input_cached_usd":0.0,"output_usd":0.0009400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6338174170814455}
{"item_id":"senseval3.d000.s095.t002","gold_sense_keys":["dream%1:09:01::"],"candidates":[{"index":1,"sense_key":"dream%1:09:01::","synset_id":"dream.n.01"},{"index":2,"sense_key":"dream%1:09:02::","synset_id":"dream.n.02"},{"index":3,"sense_key":"dream%1:12:00::","synset_id":"ambition.n.01"},{"index":4,"sense_key":"dream%1:09:03::","synset_id":"pipe_dream.n.01"},{"index":5,"sense_key":"dream%1:09:04::","synset_id":"dream.n.05"},{"index":6,"sense_key":"dream%1:26:00::","synset_id":"dream.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"dream%1:09:01::","call_ids":["senseval3.d000.s095.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"dream%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":765,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00225,"input_uncached_usd":0.00153,"input_cached_usd":0.0,"output_usd":0.00072,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9091827080119401}
{"item_id":"senseval3.d000.s095.t003","gold_sense_keys":["dream%1:09:01::"],"candidates":[{"index":1,"sense_key":"dream%1:09:01::","synset_id":"dream.n.01"},{"index":2,"sense_key":"dream%1:09:02::","synset_id":"dream.n.02"},{"index":3,"sense_key":"dream%1:12:00::","synset_id":"ambition.n.01"},{"index":4,"sense_key":"dream%1:09:03::","synset_id":"pipe_dream.n.01"},{"index":5,"sense_key":"dream%1:09:04::","synset_id":"dream.n.05"},{"index":6,"sense_key":"dream%1:26:00::","synset_id":"dream.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"dream%1:09:01::","call_ids":["senseval3.d000.s095.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"dream%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":765,"cached_input_tokens":0,"output_tokens":71,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00224,"input_uncached_usd":0.00153,"input_cached_usd":0.0,"output_usd":0.00071,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4122254999820143}
{"item_id":"senseval3.d000.s095.t004","gold_sense_keys":["begin%2:30:00::"],"candidates":[{"index":1,"sense_key":"begin%2:30:00::","synset_id":"get_down.v.07"},{"index":2,"sense_key":"begin%2:42:00::","synset_id":"begin.v.02"},{"index":3,"sense_key":"begin%2:30:01::","synset_id":"begin.v.03"},{"index":4,"sense_key":"begin%2:32:03::","synset_id":"begin.v.04"},{"index":5,"sense_key":"begin%2:42:02::","synset_id":"begin.v.05"},{"index":6,"sense_key":"begin%2:42:04::","synset_id":"begin.v.06"},{"index":7,"sense_key":"begin%2:42:03::","synset_id":"begin.v.07"},{"index":8,"sense_key":"begin%2:41:04::","synset_id":"begin.v.08"},{"index":9,"sense_key":"begin%2:41:00::","synset_id":"begin.v.09"},{"index":10,"sense_key":"begin%2:32:04::","synset_id":"begin.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"begin%2:30:00::","call_ids":["senseval3.d000.s095.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"begin%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1116,"cached_input_tokens":0,"output_tokens":244,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004672000000000001,"input_uncached_usd":0.002232,"input_cached_usd":0.0,"output_usd":0.0024400000000000003,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.654509500018321}
{"item_id":"senseval3.d000.s095.t005","gold_sense_keys":["overtone%1:10:00::"],"candidates":[{"index":1,"sense_key":"overtone%1:10:00::","synset_id":"overtone.n.01"},{"index":2,"sense_key":"overtone%1:09:00::","synset_id":"overtone.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"overtone%1:10:00::","call_ids":["senseval3.d000.s095.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"overtone%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":518,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0011560000000000001,"input_uncached_usd":0.001036,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.577187583083287}
{"item_id":"senseval3.d000.s095.t006","gold_sense_keys":["nightmare%1:09:00::"],"candidates":[{"index":1,"sense_key":"nightmare%1:26:00::","synset_id":"nightmare.n.01"},{"index":2,"sense_key":"nightmare%1:09:00::","synset_id":"nightmare.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"nightmare%1:26:00::","call_ids":["senseval3.d000.s095.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"nightmare%1:26:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":491,"cached_input_tokens":0,"output_tokens":93,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001912,"input_uncached_usd":0.000982,"input_cached_usd":0.0,"output_usd":0.00093,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.256343291956}
{"item_id":"senseval3.d000.s096.t000","gold_sense_keys":["know%2:31:01::","know%2:31:02::"],"candidates":[{"index":1,"sense_key":"know%2:31:01::","synset_id":"know.v.01"},{"index":2,"sense_key":"know%2:31:03::","synset_id":"know.v.02"},{"index":3,"sense_key":"know%2:31:02::","synset_id":"know.v.03"},{"index":4,"sense_key":"know%2:31:00::","synset_id":"know.v.04"},{"index":5,"sense_key":"know%2:31:04::","synset_id":"know.v.05"},{"index":6,"sense_key":"know%2:31:06::","synset_id":"acknowledge.v.06"},{"index":7,"sense_key":"know%2:31:15::","synset_id":"know.v.07"},{"index":8,"sense_key":"know%2:35:00::","synset_id":"sleep_together.v.01"},{"index":9,"sense_key":"know%2:31:11::","synset_id":"know.v.09"},{"index":10,"sense_key":"know%2:31:12::","synset_id":"know.v.10"},{"index":11,"sense_key":"know%2:31:14::","synset_id":"know.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"know%2:31:02::","call_ids":["senseval3.d000.s096.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"know%2:31:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1112,"cached_input_tokens":0,"output_tokens":173,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003954,"input_uncached_usd":0.002224,"input_cached_usd":0.0,"output_usd":0.0017300000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2924461669754237}
{"item_id":"senseval3.d000.s096.t001","gold_sense_keys":["dream%1:09:01::"],"candidates":[{"index":1,"sense_key":"dream%1:09:01::","synset_id":"dream.n.01"},{"index":2,"sense_key":"dream%1:09:02::","synset_id":"dream.n.02"},{"index":3,"sense_key":"dream%1:12:00::","synset_id":"ambition.n.01"},{"index":4,"sense_key":"dream%1:09:03::","synset_id":"pipe_dream.n.01"},{"index":5,"sense_key":"dream%1:09:04::","synset_id":"dream.n.05"},{"index":6,"sense_key":"dream%1:26:00::","synset_id":"dream.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"dream%1:09:01::","call_ids":["senseval3.d000.s096.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"dream%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":754,"cached_input_tokens":0,"output_tokens":107,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002578,"input_uncached_usd":0.001508,"input_cached_usd":0.0,"output_usd":0.00107,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.079363208031282}
{"item_id":"senseval3.d000.s097.t000","gold_sense_keys":["tight%5:00:00:intoxicated:00"],"candidates":[{"index":1,"sense_key":"tight%3:00:01::","synset_id":"tight.a.01"},{"index":2,"sense_key":"tight%5:00:00:tense:01","synset_id":"taut.s.01"},{"index":3,"sense_key":"tight%5:00:00:invulnerable:00","synset_id":"tight.s.02"},{"index":4,"sense_key":"tight%5:00:00:closed:03","synset_id":"compressed.s.01"},{"index":5,"sense_key":"tight%5:00:00:stingy:00","synset_id":"mean.s.06"},{"index":6,"sense_key":"tight%5:00:00:scarce:00","synset_id":"tight.s.05"},{"index":7,"sense_key":"tight%3:00:02::","synset_id":"tight.a.07"},{"index":8,"sense_key":"tight%5:00:00:fine:00","synset_id":"close.s.06"},{"index":9,"sense_key":"tight%5:00:00:secure:03","synset_id":"tight.s.07"},{"index":10,"sense_key":"tight%5:00:00:equal:00","synset_id":"close.s.03"},{"index":11,"sense_key":"tight%5:00:00:intoxicated:00","synset_id":"besotted.s.01"},{"index":12,"sense_key":"tight%5:00:00:difficult:00","synset_id":"nasty.s.01"},{"index":13,"sense_key":"tight%5:00:00:demanding:01","synset_id":"rigorous.s.02"},{"index":14,"sense_key":"tight%5:00:00:compact:00","synset_id":"tight.s.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"tight%5:00:00:intoxicated:00","call_ids":["senseval3.d000.s097.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"tight%5:00:00:intoxicated:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1259,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0026379999999999997,"input_uncached_usd":0.002518,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9738062919350341}
{"item_id":"senseval3.d000.s097.t001","gold_sense_keys":["know%2:31:01::","know%2:31:02::"],"candidates":[{"index":1,"sense_key":"know%2:31:01::","synset_id":"know.v.01"},{"index":2,"sense_key":"know%2:31:03::","synset_id":"know.v.02"},{"index":3,"sense_key":"know%2:31:02::","synset_id":"know.v.03"},{"index":4,"sense_key":"know%2:31:00::","synset_id":"know.v.04"},{"index":5,"sense_key":"know%2:31:04::","synset_id":"know.v.05"},{"index":6,"sense_key":"know%2:31:06::","synset_id":"acknowledge.v.06"},{"index":7,"sense_key":"know%2:31:15::","synset_id":"know.v.07"},{"index":8,"sense_key":"know%2:35:00::","synset_id":"sleep_together.v.01"},{"index":9,"sense_key":"know%2:31:11::","synset_id":"know.v.09"},{"index":10,"sense_key":"know%2:31:12::","synset_id":"know.v.10"},{"index":11,"sense_key":"know%2:31:14::","synset_id":"know.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"know%2:31:01::","call_ids":["senseval3.d000.s097.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"know%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1126,"cached_input_tokens":0,"output_tokens":111,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0033620000000000004,"input_uncached_usd":0.002252,"input_cached_usd":0.0,"output_usd":0.00111,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8306510839611292}
{"item_id":"senseval3.d000.s098.t000","gold_sense_keys":["same%3:00:02::"],"candidates":[{"index":1,"sense_key":"same%3:00:02::","synset_id":"same.a.01"},{"index":2,"sense_key":"same%3:00:00::","synset_id":"same.a.02"},{"index":3,"sense_key":"same%3:00:04::","synset_id":"like.a.02"},{"index":4,"sense_key":"same%5:00:00:unchanged:00","synset_id":"same.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"same%3:00:02::","call_ids":["senseval3.d000.s098.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"same%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":637,"cached_input_tokens":0,"output_tokens":48,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001754,"input_uncached_usd":0.001274,"input_cached_usd":0.0,"output_usd":0.00048000000000000007,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.250797540997155}
{"item_id":"senseval3.d000.s098.t001","gold_sense_keys":["bar%1:06:04::"],"candidates":[{"index":1,"sense_key":"bar%1:06:04::","synset_id":"barroom.n.01"},{"index":2,"sense_key":"bar%1:06:05::","synset_id":"bar.n.02"},{"index":3,"sense_key":"bar%1:06:00::","synset_id":"bar.n.03"},{"index":4,"sense_key":"bar%1:10:00::","synset_id":"measure.n.07"},{"index":5,"sense_key":"bar%1:06:02::","synset_id":"bar.n.05"},{"index":6,"sense_key":"bar%1:04:00::","synset_id":"prevention.n.01"},{"index":7,"sense_key":"bar%1:23:00::","synset_id":"bar.n.07"},{"index":8,"sense_key":"bar%1:17:00::","synset_id":"bar.n.08"},{"index":9,"sense_key":"bar%1:14:00::","synset_id":"legal_profession.n.01"},{"index":10,"sense_key":"bar%1:07:00::","synset_id":"stripe.n.05"},{"index":11,"sense_key":"bar%1:06:06::","synset_id":"cake.n.01"},{"index":12,"sense_key":"bar%1:06:01::","synset_id":"browning_automatic_rifle.n.01"},{"index":13,"sense_key":"bar%1:06:09::","synset_id":"bar.n.13"},{"index":14,"sense_key":"bar%1:06:08::","synset_id":"bar.n.14"},{"index":15,"sense_key":"bar%1:06:07::","synset_id":"bar.n.15"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bar%1:06:04::","call_ids":["senseval3.d000.s098.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bar%1:06:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1452,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0030239999999999998,"input_uncached_usd":0.002904,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0193262910470366}
{"item_id":"senseval3.d000.s098.t002","gold_sense_keys":["week%1:28:00::"],"candidates":[{"index":1,"sense_key":"week%1:28:00::","synset_id":"week.n.01"},{"index":2,"sense_key":"week%1:28:01::","synset_id":"workweek.n.01"},{"index":3,"sense_key":"week%1:28:02::","synset_id":"week.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"week%1:28:00::","call_ids":["senseval3.d000.s098.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"week%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":573,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001266,"input_uncached_usd":0.0011459999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3885939590400085}
{"item_id":"senseval3.d000.s098.t003","gold_sense_keys":["night%1:28:03::"],"candidates":[{"index":1,"sense_key":"night%1:28:00::","synset_id":"night.n.01"},{"index":2,"sense_key":"night%1:28:01::","synset_id":"night.n.02"},{"index":3,"sense_key":"night%1:28:04::","synset_id":"night.n.03"},{"index":4,"sense_key":"night%1:28:02::","synset_id":"night.n.04"},{"index":5,"sense_key":"night%1:26:00::","synset_id":"night.n.05"},{"index":6,"sense_key":"night%1:28:05::","synset_id":"night.n.06"},{"index":7,"sense_key":"night%1:28:03::","synset_id":"night.n.07"},{"index":8,"sense_key":"night%1:18:00::","synset_id":"nox.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"night%1:28:03::","call_ids":["senseval3.d000.s098.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"night%1:28:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":775,"cached_input_tokens":0,"output_tokens":226,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00381,"input_uncached_usd":0.00155,"input_cached_usd":0.0,"output_usd":0.0022600000000000003,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.281513374997303}
{"item_id":"senseval3.d000.s098.t004","gold_sense_keys":["have%2:40:00::"],"candidates":[{"index":1,"sense_key":"have%2:40:00::","synset_id":"have.v.01"},{"index":2,"sense_key":"have%2:42:00::","synset_id":"have.v.02"},{"index":3,"sense_key":"have%2:39:06::","synset_id":"experience.v.03"},{"index":4,"sense_key":"have%2:40:04::","synset_id":"own.v.01"},{"index":5,"sense_key":"have%2:30:00::","synset_id":"get.v.03"},{"index":6,"sense_key":"have%2:34:00::","synset_id":"consume.v.02"},{"index":7,"sense_key":"have%2:41:00::","synset_id":"have.v.07"},{"index":8,"sense_key":"have%2:36:00::","synset_id":"hold.v.03"},{"index":9,"sense_key":"have%2:40:01::","synset_id":"have.v.09"},{"index":10,"sense_key":"have%2:42:12::","synset_id":"have.v.10"},{"index":11,"sense_key":"have%2:30:01::","synset_id":"have.v.11"},{"index":12,"sense_key":"have%2:29:02::","synset_id":"have.v.12"},{"index":13,"sense_key":"have%2:32:00::","synset_id":"induce.v.02"},{"index":14,"sense_key":"have%2:40:05::","synset_id":"accept.v.02"},{"index":15,"sense_key":"have%2:40:03::","synset_id":"receive.v.01"},{"index":16,"sense_key":"have%2:29:05::","synset_id":"suffer.v.02"},{"index":17,"sense_key":"have%2:40:02::","synset_id":"have.v.17"},{"index":18,"sense_key":"have%2:29:00::","synset_id":"give_birth.v.01"},{"index":19,"sense_key":"have%2:35:00::","synset_id":"take.v.35"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"have%2:40:00::","call_ids":["senseval3.d000.s098.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"have%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1584,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0037979999999999997,"input_uncached_usd":0.003168,"input_cached_usd":0.0,"output_usd":0.00063,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7827056249370798}
{"item_id":"senseval3.d000.s098.t005","gold_sense_keys":["excuse%1:10:00::"],"candidates":[{"index":1,"sense_key":"excuse%1:10:00::","synset_id":"excuse.n.01"},{"index":2,"sense_key":"excuse%1:10:01::","synset_id":"excuse.n.02"},{"index":3,"sense_key":"excuse%1:09:00::","synset_id":"apology.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"excuse%1:10:00::","call_ids":["senseval3.d000.s098.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"excuse%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":626,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001372,"input_uncached_usd":0.0012519999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.475645332946442}
{"item_id":"senseval3.d000.s098.t006","gold_sense_keys":["drink%2:34:01::"],"candidates":[{"index":1,"sense_key":"drink%2:34:00::","synset_id":"drink.v.01"},{"index":2,"sense_key":"drink%2:34:01::","synset_id":"drink.v.02"},{"index":3,"sense_key":"drink%2:34:12::","synset_id":"toast.v.02"},{"index":4,"sense_key":"drink%2:31:00::","synset_id":"drink_in.v.01"},{"index":5,"sense_key":"drink%2:34:02::","synset_id":"drink.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"drink%2:34:01::","call_ids":["senseval3.d000.s098.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"drink%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":742,"cached_input_tokens":0,"output_tokens":230,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003784,"input_uncached_usd":0.0014839999999999999,"input_cached_usd":0.0,"output_usd":0.0023000000000000004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.277414125041105}
{"item_id":"senseval3.d000.s098.t007","gold_sense_keys":["usual%3:00:00::"],"candidates":[{"index":1,"sense_key":"usual%3:00:00::","synset_id":"usual.a.01"},{"index":2,"sense_key":"usual%5:00:00:familiar:02","synset_id":"common.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"usual%3:00:00::","call_ids":["senseval3.d000.s098.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"usual%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":553,"cached_input_tokens":0,"output_tokens":147,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002576,"input_uncached_usd":0.001106,"input_cached_usd":0.0,"output_usd":0.0014700000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.080086417030543}
{"item_id":"senseval3.d000.s099.t000","gold_sense_keys":["sit%2:35:00::","sit%2:38:00::"],"candidates":[{"index":1,"sense_key":"sit%2:35:00::","synset_id":"sit.v.01"},{"index":2,"sense_key":"sit%2:42:00::","synset_id":"sit.v.02"},{"index":3,"sense_key":"sit%2:38:00::","synset_id":"sit_down.v.01"},{"index":4,"sense_key":"sit%2:38:01::","synset_id":"sit.v.04"},{"index":5,"sense_key":"sit%2:39:00::","synset_id":"model.v.03"},{"index":6,"sense_key":"sit%2:38:03::","synset_id":"ride.v.01"},{"index":7,"sense_key":"sit%2:42:05::","synset_id":"sit.v.07"},{"index":8,"sense_key":"sit%2:41:05::","synset_id":"baby-sit.v.02"},{"index":9,"sense_key":"sit%2:35:01::","synset_id":"seat.v.01"},{"index":10,"sense_key":"sit%2:33:00::","synset_id":"sit.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"sit%2:35:00::","call_ids":["senseval3.d000.s099.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"sit%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1000,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00267,"input_uncached_usd":0.002,"input_cached_usd":0.0,"output_usd":0.00067,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8931349170161411}
{"item_id":"senseval3.d000.s099.t001","gold_sense_keys":["usual%3:00:00::"],"candidates":[{"index":1,"sense_key":"usual%3:00:00::","synset_id":"usual.a.01"},{"index":2,"sense_key":"usual%5:00:00:familiar:02","synset_id":"common.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"usual%3:00:00::","call_ids":["senseval3.d000.s099.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"usual%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":576,"cached_input_tokens":0,"output_tokens":85,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0020020000000000003,"input_uncached_usd":0.001152,"input_cached_usd":0.0,"output_usd":0.0008500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4721499580191448}
{"item_id":"senseval3.d000.s099.t002","gold_sense_keys":["corner%1:06:02::"],"candidates":[{"index":1,"sense_key":"corner%1:15:02::","synset_id":"corner.n.01"},{"index":2,"sense_key":"corner%1:15:00::","synset_id":"corner.n.02"},{"index":3,"sense_key":"corner%1:06:02::","synset_id":"corner.n.03"},{"index":4,"sense_key":"corner%1:06:00::","synset_id":"corner.n.04"},{"index":5,"sense_key":"corner%1:15:01::","synset_id":"corner.n.05"},{"index":6,"sense_key":"corner%1:25:00::","synset_id":"recess.n.02"},{"index":7,"sense_key":"corner%1:26:00::","synset_id":"corner.n.07"},{"index":8,"sense_key":"corner%1:26:01::","synset_id":"corner.n.08"},{"index":9,"sense_key":"corner%1:17:00::","synset_id":"corner.n.09"},{"index":10,"sense_key":"corner%1:15:03::","synset_id":"corner.n.10"},{"index":11,"sense_key":"corner%1:06:01::","synset_id":"corner.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"corner%1:06:02::","call_ids":["senseval3.d000.s099.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"corner%1:06:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1028,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002726,"input_uncached_usd":0.002056,"input_cached_usd":0.0,"output_usd":0.00067,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7467123749665916}
{"item_id":"senseval3.d000.s099.t003","gold_sense_keys":["little%3:00:01::"],"candidates":[{"index":1,"sense_key":"little%3:00:01::","synset_id":"small.a.01"},{"index":2,"sense_key":"little%3:00:03::","synset_id":"little.a.02"},{"index":3,"sense_key":"little%5:00:00:young:00","synset_id":"little.s.01"},{"index":4,"sense_key":"little%5:00:00:unimportant:00","synset_id":"fiddling.s.01"},{"index":5,"sense_key":"little%5:00:00:soft:04","synset_id":"little.s.03"},{"index":6,"sense_key":"little%3:00:00::","synset_id":"short.a.03"},{"index":7,"sense_key":"little%5:00:00:lowercase:00","synset_id":"little.s.04"},{"index":8,"sense_key":"little%5:00:00:emotional:00","synset_id":"little.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"little%3:00:01::","call_ids":["senseval3.d000.s099.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"little%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":955,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0024999999999999996,"input_uncached_usd":0.0019099999999999998,"input_cached_usd":0.0,"output_usd":0.00059,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4227030420443043}
{"item_id":"senseval3.d000.s099.t004","gold_sense_keys":["table%1:06:01::"],"candidates":[{"index":1,"sense_key":"table%1:14:00::","synset_id":"table.n.01"},{"index":2,"sense_key":"table%1:06:01::","synset_id":"table.n.02"},{"index":3,"sense_key":"table%1:06:02::","synset_id":"table.n.03"},{"index":4,"sense_key":"table%1:17:00::","synset_id":"mesa.n.01"},{"index":5,"sense_key":"table%1:14:01::","synset_id":"table.n.05"},{"index":6,"sense_key":"table%1:13:01::","synset_id":"board.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"table%1:06:01::","call_ids":["senseval3.d000.s099.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"table%1:06:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":789,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001698,"input_uncached_usd":0.001578,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3737107500201091}
{"item_id":"senseval3.d000.s099.t006","gold_sense_keys":["talkative%5:00:00:communicative:00","talkative%5:00:00:voluble:00"],"candidates":[{"index":1,"sense_key":"talkative%5:00:00:voluble:00","synset_id":"chatty.s.01"},{"index":2,"sense_key":"talkative%5:00:00:indiscreet:00","synset_id":"bigmouthed.s.01"},{"index":3,"sense_key":"talkative%5:00:00:communicative:00","synset_id":"expansive.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"talkative%5:00:00:voluble:00","call_ids":["senseval3.d000.s099.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"talkative%5:00:00:voluble:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":645,"cached_input_tokens":0,"output_tokens":129,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00258,"input_uncached_usd":0.00129,"input_cached_usd":0.0,"output_usd":0.0012900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8229436250403523}
{"item_id":"senseval3.d000.s099.t007","gold_sense_keys":["friendly%3:00:01::"],"candidates":[{"index":1,"sense_key":"friendly%3:00:01::","synset_id":"friendly.a.01"},{"index":2,"sense_key":"friendly%5:00:00:amicable:00","synset_id":"friendly.s.01"},{"index":3,"sense_key":"friendly%3:00:03::","synset_id":"friendly.a.03"},{"index":4,"sense_key":"friendly%3:00:02::","synset_id":"friendly.a.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"friendly%3:00:01::","call_ids":["senseval3.d000.s099.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"friendly%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":696,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001512,"input_uncached_usd":0.001392,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8717312089866027}
{"item_id":"senseval3.d000.s100.t000","gold_sense_keys":["enjoy%2:34:00::","enjoy%2:37:00::","enjoy%2:37:01::"],"candidates":[{"index":1,"sense_key":"enjoy%2:37:00::","synset_id":"enjoy.v.01"},{"index":2,"sense_key":"enjoy%2:34:01::","synset_id":"enjoy.v.02"},{"index":3,"sense_key":"enjoy%2:37:01::","synset_id":"love.v.02"},{"index":4,"sense_key":"enjoy%2:39:00::","synset_id":"enjoy.v.04"},{"index":5,"sense_key":"enjoy%2:34:00::","synset_id":"delight.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"enjoy%2:37:00::","call_ids":["senseval3.d000.s100.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"enjoy%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":695,"cached_input_tokens":0,"output_tokens":154,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0029300000000000003,"input_uncached_usd":0.00139,"input_cached_usd":0.0,"output_usd":0.0015400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.127723874989897}
{"item_id":"senseval3.d000.s100.t002","gold_sense_keys":["ritual%1:04:00::"],"candidates":[{"index":1,"sense_key":"ritual%1:04:00::","synset_id":"ritual.n.01"},{"index":2,"sense_key":"ritual%1:04:01::","synset_id":"ritual.n.02"},{"index":3,"sense_key":"ritual%1:04:02::","synset_id":"ritual.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ritual%1:04:00::","call_ids":["senseval3.d000.s100.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ritual%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":537,"cached_input_tokens":0,"output_tokens":129,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0023639999999999998,"input_uncached_usd":0.0010739999999999999,"input_cached_usd":0.0,"output_usd":0.0012900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7576540000736713}
{"item_id":"senseval3.d000.s100.t003","gold_sense_keys":["wash_down%2:34:00::"],"candidates":[{"index":1,"sense_key":"wash_down%2:34:00::","synset_id":"wash_down.v.01"},{"index":2,"sense_key":"wash_down%2:30:00::","synset_id":"wash_down.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"wash_down%2:34:00::","call_ids":["senseval3.d000.s100.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"wash_down%2:34:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":560,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00124,"input_uncached_usd":0.00112,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.213498083059676}
{"item_id":"senseval3.d000.s100.t004","gold_sense_keys":["pet%1:18:00::"],"candidates":[{"index":1,"sense_key":"pet%1:05:00::","synset_id":"pet.n.01"},{"index":2,"sense_key":"pet%1:18:00::","synset_id":"darling.n.01"},{"index":3,"sense_key":"pet%1:12:00::","synset_id":"pet.n.03"},{"index":4,"sense_key":"pet%1:04:00::","synset_id":"positron_emission_tomography.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"pet%1:18:00::","call_ids":["senseval3.d000.s100.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"pet%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":634,"cached_input_tokens":0,"output_tokens":114,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0024080000000000004,"input_uncached_usd":0.001268,"input_cached_usd":0.0,"output_usd":0.0011400000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.175093541969545}
{"item_id":"senseval3.d000.s100.t005","gold_sense_keys":["grievance%1:12:00::"],"candidates":[{"index":1,"sense_key":"grievance%1:12:00::","synset_id":"grudge.n.01"},{"index":2,"sense_key":"grievance%1:10:01::","synset_id":"grievance.n.02"},{"index":3,"sense_key":"grievance%1:10:00::","synset_id":"grievance.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"grievance%1:12:00::","call_ids":["senseval3.d000.s100.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"grievance%1:12:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":594,"cached_input_tokens":0,"output_tokens":146,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002648,"input_uncached_usd":0.001188,"input_cached_usd":0.0,"output_usd":0.0014600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8828628329792991}
{"item_id":"senseval3.d000.s100.t006","gold_sense_keys":["bourbon%1:13:00::"],"candidates":[{"index":1,"sense_key":"bourbon%1:18:01::","synset_id":"bourbon.n.01"},{"index":2,"sense_key":"bourbon%1:13:00::","synset_id":"bourbon.n.02"},{"index":3,"sense_key":"bourbon%1:18:00::","synset_id":"bourbon.n.03"},{"index":4,"sense_key":"bourbon%1:14:00::","synset_id":"bourbon.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bourbon%1:13:00::","call_ids":["senseval3.d000.s100.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bourbon%1:13:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":633,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001386,"input_uncached_usd":0.001266,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.837863374967128}
{"item_id":"senseval3.d000.s100.t007","gold_sense_keys":["water%1:13:00::"],"candidates":[{"index":1,"sense_key":"water%1:27:00::","synset_id":"water.n.01"},{"index":2,"sense_key":"water%1:17:00::","synset_id":"body_of_water.n.01"},{"index":3,"sense_key":"water%1:27:02::","synset_id":"water.n.03"},{"index":4,"sense_key":"water%1:06:00::","synset_id":"water_system.n.02"},{"index":5,"sense_key":"water%1:27:01::","synset_id":"urine.n.01"},{"index":6,"sense_key":"water%1:13:00::","synset_id":"water.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"water%1:13:00::","call_ids":["senseval3.d000.s100.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"water%1:13:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":836,"cached_input_tokens":0,"output_tokens":103,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002702,"input_uncached_usd":0.0016719999999999999,"input_cached_usd":0.0,"output_usd":0.00103,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.201092208037153}
{"item_id":"senseval3.d000.s101.t000","gold_sense_keys":["favorite%5:00:00:loved:00"],"candidates":[{"index":1,"sense_key":"favorite%5:00:00:popular:00","synset_id":"favorite.s.01"},{"index":2,"sense_key":"favorite%5:00:00:loved:00","synset_id":"favored.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"favorite%5:00:00:loved:00","call_ids":["senseval3.d000.s101.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"favorite%5:00:00:loved:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":533,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0017859999999999998,"input_uncached_usd":0.0010659999999999999,"input_cached_usd":0.0,"output_usd":0.00072,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9289152501150966}
{"item_id":"senseval3.d000.s101.t001","gold_sense_keys":["grievance%1:12:00::"],"candidates":[{"index":1,"sense_key":"grievance%1:12:00::","synset_id":"grudge.n.01"},{"index":2,"sense_key":"grievance%1:10:01::","synset_id":"grievance.n.02"},{"index":3,"sense_key":"grievance%1:10:00::","synset_id":"grievance.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"grievance%1:12:00::","call_ids":["senseval3.d000.s101.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"grievance%1:12:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":569,"cached_input_tokens":0,"output_tokens":101,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0021479999999999997,"input_uncached_usd":0.001138,"input_cached_usd":0.0,"output_usd":0.00101,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9579382500378415}
{"item_id":"senseval3.d000.s102.t000","gold_sense_keys":["forget%2:31:01::"],"candidates":[{"index":1,"sense_key":"forget%2:31:01::","synset_id":"forget.v.01"},{"index":2,"sense_key":"forget%2:31:00::","synset_id":"forget.v.02"},{"index":3,"sense_key":"forget%2:31:03::","synset_id":"forget.v.03"},{"index":4,"sense_key":"forget%2:31:02::","synset_id":"forget.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"forget%2:31:01::","call_ids":["senseval3.d000.s102.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"forget%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":666,"cached_input_tokens":0,"output_tokens":99,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002322,"input_uncached_usd":0.0013319999999999999,"input_cached_usd":0.0,"output_usd":0.00099,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2069332499522716}
{"item_id":"senseval3.d000.s103.t000","gold_sense_keys":["mention%2:32:02::"],"candidates":[{"index":1,"sense_key":"mention%2:32:02::","synset_id":"mention.v.01"},{"index":2,"sense_key":"mention%2:32:00::","synset_id":"note.v.01"},{"index":3,"sense_key":"mention%2:32:01::","synset_id":"mention.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"mention%2:32:02::","call_ids":["senseval3.d000.s103.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"mention%2:32:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":588,"cached_input_tokens":0,"output_tokens":122,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002396,"input_uncached_usd":0.001176,"input_cached_usd":0.0,"output_usd":0.0012200000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.778111582971178}
{"item_id":"senseval3.d000.s104.t000","gold_sense_keys":["real%3:00:02::","real%5:00:00:true:00"],"candidates":[{"index":1,"sense_key":"real%3:00:00::","synset_id":"real.a.01"},{"index":2,"sense_key":"real%3:00:02::","synset_id":"real.a.02"},{"index":3,"sense_key":"real%5:00:00:serious:00","synset_id":"real.s.01"},{"index":4,"sense_key":"real%5:00:00:concrete:00","synset_id":"real.s.02"},{"index":5,"sense_key":"real%5:00:00:true:00","synset_id":"actual.s.02"},{"index":6,"sense_key":"real%3:00:01::","synset_id":"real.a.06"},{"index":7,"sense_key":"real%3:00:03::","synset_id":"substantial.a.03"},{"index":8,"sense_key":"real%5:00:00:tangible:02","synset_id":"real.s.04"},{"index":9,"sense_key":"real%5:00:02:realistic:00","synset_id":"veridical.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"real%5:00:00:true:00","call_ids":["senseval3.d000.s104.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"real%5:00:00:true:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1056,"cached_input_tokens":0,"output_tokens":168,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003792,"input_uncached_usd":0.0021119999999999997,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0416091250954196}
{"item_id":"senseval3.d000.s104.t001","gold_sense_keys":["grievance%1:12:00::"],"candidates":[{"index":1,"sense_key":"grievance%1:12:00::","synset_id":"grudge.n.01"},{"index":2,"sense_key":"grievance%1:10:01::","synset_id":"grievance.n.02"},{"index":3,"sense_key":"grievance%1:10:00::","synset_id":"grievance.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"grievance%1:12:00::","call_ids":["senseval3.d000.s104.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"grievance%1:12:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":585,"cached_input_tokens":0,"output_tokens":85,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00202,"input_uncached_usd":0.00117,"input_cached_usd":0.0,"output_usd":0.0008500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7719318749150261}
{"item_id":"senseval3.d000.s105.t000","gold_sense_keys":["end%1:28:00::"],"candidates":[{"index":1,"sense_key":"end%1:15:00::","synset_id":"end.n.01"},{"index":2,"sense_key":"end%1:28:00::","synset_id":"end.n.02"},{"index":3,"sense_key":"end%1:11:00::","synset_id":"end.n.03"},{"index":4,"sense_key":"end%1:09:02::","synset_id":"goal.n.01"},{"index":5,"sense_key":"end%1:09:00::","synset_id":"end.n.05"},{"index":6,"sense_key":"end%1:26:00::","synset_id":"end.n.06"},{"index":7,"sense_key":"end%1:15:02::","synset_id":"end.n.07"},{"index":8,"sense_key":"end%1:18:00::","synset_id":"end.n.08"},{"index":9,"sense_key":"end%1:15:01::","synset_id":"end.n.09"},{"index":10,"sense_key":"end%1:15:03::","synset_id":"end.n.10"},{"index":11,"sense_key":"end%1:04:00::","synset_id":"end.n.11"},{"index":12,"sense_key":"end%1:10:00::","synset_id":"conclusion.n.08"},{"index":13,"sense_key":"end%1:06:02::","synset_id":"end.n.13"},{"index":14,"sense_key":"end%1:04:02::","synset_id":"end.n.14"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"end%1:11:00::","call_ids":["senseval3.d000.s105.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"end%1:11:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1237,"cached_input_tokens":0,"output_tokens":122,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0036940000000000002,"input_uncached_usd":0.002474,"input_cached_usd":0.0,"output_usd":0.0012200000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7601728329900652}
{"item_id":"senseval3.d000.s105.t003","gold_sense_keys":["make%2:36:00::","make%2:36:08::"],"candidates":[{"index":1,"sense_key":"make%2:41:00::","synset_id":"make.v.01"},{"index":2,"sense_key":"make%2:30:00::","synset_id":"make.v.02"},{"index":3,"sense_key":"make%2:36:00::","synset_id":"make.v.03"},{"index":4,"sense_key":"make%2:32:00::","synset_id":"induce.v.02"},{"index":5,"sense_key":"make%2:36:08::","synset_id":"cause.v.01"},{"index":6,"sense_key":"make%2:36:01::","synset_id":"produce.v.02"},{"index":7,"sense_key":"make%2:31:00::","synset_id":"draw.v.04"},{"index":8,"sense_key":"make%2:36:15::","synset_id":"make.v.08"},{"index":9,"sense_key":"make%2:36:09::","synset_id":"create.v.05"},{"index":10,"sense_key":"make%2:40:01::","synset_id":"gain.v.08"},{"index":11,"sense_key":"make%2:36:05::","synset_id":"do.v.08"},{"index":12,"sense_key":"make%2:42:05::","synset_id":"form.v.02"},{"index":13,"sense_key":"make%2:38:02::","synset_id":"reach.v.07"},{"index":14,"sense_key":"make%2:42:00::","synset_id":"make.v.14"},{"index":15,"sense_key":"make%2:36:11::","synset_id":"make.v.15"},{"index":16,"sense_key":"make%2:36:12::","synset_id":"make.v.16"},{"index":17,"sense_key":"make%2:36:13::","synset_id":"construct.v.01"},{"index":18,"sense_key":"make%2:30:15::","synset_id":"make.v.18"},{"index":19,"sense_key":"make%2:29:08::","synset_id":"make.v.19"},{"index":20,"sense_key":"make%2:41:03::","synset_id":"name.v.03"},{"index":21,"sense_key":"make%2:40:02::","synset_id":"have.v.17"},{"index":22,"sense_key":"make%2:38:00::","synset_id":"reach.v.01"},{"index":23,"sense_key":"make%2:31:13::","synset_id":"lay_down.v.01"},{"index":24,"sense_key":"make%2:41:13::","synset_id":"make.v.24"},{"index":25,"sense_key":"make%2:41:01::","synset_id":"make.v.25"},{"index":26,"sense_key":"make%2:36:04::","synset_id":"hold.v.03"},{"index":27,"sense_key":"make%2:30:02::","synset_id":"make.v.27"},{"index":28,"sense_key":"make%2:38:05::","synset_id":"take.v.27"},{"index":29,"sense_key":"make%2:29:00::","synset_id":"stool.v.04"},{"index":30,"sense_key":"make%2:42:09::","synset_id":"make.v.30"},{"index":31,"sense_key":"make%2:42:13::","synset_id":"make.v.31"},{"index":32,"sense_key":"make%2:42:12::","synset_id":"make.v.32"},{"index":33,"sense_key":"make%2:42:11::","synset_id":"make.v.33"},{"index":34,"sense_key":"make%2:42:15::","synset_id":"make.v.34"},{"index":35,"sense_key":"make%2:39:12::","synset_id":"make.v.35"},{"index":36,"sense_key":"make%2:38:10::","synset_id":"make.v.36"},{"index":37,"sense_key":"make%2:38:15::","synset_id":"make.v.37"},{"index":38,"sense_key":"make%2:36:10::","synset_id":"make.v.38"},{"index":39,"sense_key":"make%2:36:07::","synset_id":"cook.v.02"},{"index":40,"sense_key":"make%2:35:13::","synset_id":"seduce.v.01"},{"index":41,"sense_key":"make%2:32:14::","synset_id":"make.v.41"},{"index":42,"sense_key":"make%2:32:09::","synset_id":"make.v.42"},{"index":43,"sense_key":"make%2:31:10::","synset_id":"make.v.43"},{"index":44,"sense_key":"make%2:31:12::","synset_id":"make.v.44"},{"index":45,"sense_key":"make%2:30:10::","synset_id":"make.v.45"},{"index":46,"sense_key":"make%2:30:14::","synset_id":"make.v.46"},{"index":47,"sense_key":"make%2:30:12::","synset_id":"make.v.47"},{"index":48,"sense_key":"make%2:29:06::","synset_id":"make.v.48"},{"index":49,"sense_key":"make%2:29:01::","synset_id":"make.v.49"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"make%2:36:00::","call_ids":["senseval3.d000.s105.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"make%2:36:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":3095,"cached_input_tokens":0,"output_tokens":267,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00886,"input_uncached_usd":0.006189999999999999,"input_cached_usd":0.0,"output_usd":0.00267,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.224708958994597}
{"item_id":"senseval3.d000.s105.t004","gold_sense_keys":["wet%3:00:01::"],"candidates":[{"index":1,"sense_key":"wet%3:00:01::","synset_id":"wet.a.01"},{"index":2,"sense_key":"wet%3:00:05::","synset_id":"wet.a.02"},{"index":3,"sense_key":"wet%3:00:04::","synset_id":"wet.a.03"},{"index":4,"sense_key":"wet%3:00:02::","synset_id":"wet.a.04"},{"index":5,"sense_key":"wet%5:00:00:alcoholic:00","synset_id":"wet.s.01"},{"index":6,"sense_key":"wet%5:00:00:intoxicated:00","synset_id":"besotted.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"wet%3:00:01::","call_ids":["senseval3.d000.s105.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"wet%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":777,"cached_input_tokens":0,"output_tokens":48,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002034,"input_uncached_usd":0.0015539999999999998,"input_cached_usd":0.0,"output_usd":0.00048000000000000007,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4574161670170724}
{"item_id":"senseval3.d000.s105.t005","gold_sense_keys":["ring%1:25:00::"],"candidates":[{"index":1,"sense_key":"ring%1:07:00::","synset_id":"ring.n.01"},{"index":2,"sense_key":"ring%1:25:00::","synset_id":"ring.n.02"},{"index":3,"sense_key":"ring%1:06:02::","synset_id":"hoop.n.02"},{"index":4,"sense_key":"ring%1:17:00::","synset_id":"closed_chain.n.01"},{"index":5,"sense_key":"ring%1:14:00::","synset_id":"gang.n.01"},{"index":6,"sense_key":"ring%1:11:00::","synset_id":"ring.n.06"},{"index":7,"sense_key":"ring%1:06:03::","synset_id":"ring.n.07"},{"index":8,"sense_key":"ring%1:06:00::","synset_id":"ring.n.08"},{"index":9,"sense_key":"ring%1:06:01::","synset_id":"band.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"ring%1:25:00::","call_ids":["senseval3.d000.s105.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"ring%1:25:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1031,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002732,"input_uncached_usd":0.002062,"input_cached_usd":0.0,"output_usd":0.00067,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2712501250207424}
{"item_id":"senseval3.d000.s105.t006","gold_sense_keys":["bottom%1:15:00::"],"candidates":[{"index":1,"sense_key":"bottom%1:15:00::","synset_id":"bottom.n.01"},{"index":2,"sense_key":"bottom%1:15:01::","synset_id":"bottom.n.02"},{"index":3,"sense_key":"bottom%1:08:00::","synset_id":"buttocks.n.01"},{"index":4,"sense_key":"bottom%1:28:00::","synset_id":"bottom.n.04"},{"index":5,"sense_key":"bottom%1:17:00::","synset_id":"bed.n.03"},{"index":6,"sense_key":"bottom%1:17:01::","synset_id":"bottomland.n.01"},{"index":7,"sense_key":"bottom%1:06:00::","synset_id":"bottom.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bottom%1:15:00::","call_ids":["senseval3.d000.s105.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bottom%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":848,"cached_input_tokens":0,"output_tokens":104,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002736,"input_uncached_usd":0.001696,"input_cached_usd":0.0,"output_usd":0.0010400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.621898000012152}
{"item_id":"senseval3.d000.s105.t007","gold_sense_keys":["glass%1:06:00::"],"candidates":[{"index":1,"sense_key":"glass%1:27:00::","synset_id":"glass.n.01"},{"index":2,"sense_key":"glass%1:06:00::","synset_id":"glass.n.02"},{"index":3,"sense_key":"glass%1:23:00::","synset_id":"glass.n.03"},{"index":4,"sense_key":"glass%1:06:03::","synset_id":"field_glass.n.01"},{"index":5,"sense_key":"glass%1:06:01::","synset_id":"methamphetamine.n.01"},{"index":6,"sense_key":"glass%1:06:02::","synset_id":"looking_glass.n.01"},{"index":7,"sense_key":"glass%1:06:04::","synset_id":"glass.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"glass%1:06:00::","call_ids":["senseval3.d000.s105.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"glass%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":816,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001752,"input_uncached_usd":0.001632,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9970500839408487}
{"item_id":"senseval3.d000.s105.t008","gold_sense_keys":["become%2:30:00::"],"candidates":[{"index":1,"sense_key":"become%2:30:00::","synset_id":"become.v.01"},{"index":2,"sense_key":"become%2:42:01::","synset_id":"become.v.02"},{"index":3,"sense_key":"become%2:42:00::","synset_id":"become.v.03"},{"index":4,"sense_key":"become%2:42:03::","synset_id":"become.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"become%2:30:00::","call_ids":["senseval3.d000.s105.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"become%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":657,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001914,"input_uncached_usd":0.0013139999999999998,"input_cached_usd":0.0,"output_usd":0.0006000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4413172909989953}
{"item_id":"senseval3.d000.s105.t009","gold_sense_keys":["aware%3:00:00::"],"candidates":[{"index":1,"sense_key":"aware%3:00:00::","synset_id":"aware.a.01"},{"index":2,"sense_key":"aware%3:00:04::","synset_id":"mindful.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"aware%3:00:00::","call_ids":["senseval3.d000.s105.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"aware%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":587,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001294,"input_uncached_usd":0.001174,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5609265840612352}
{"item_id":"senseval3.d000.s106.t000","gold_sense_keys":["look%2:39:00::"],"candidates":[{"index":1,"sense_key":"look%2:39:00::","synset_id":"look.v.01"},{"index":2,"sense_key":"look%2:39:01::","synset_id":"look.v.02"},{"index":3,"sense_key":"look%2:29:00::","synset_id":"look.v.03"},{"index":4,"sense_key":"look%2:39:02::","synset_id":"search.v.02"},{"index":5,"sense_key":"look%2:42:00::","synset_id":"front.v.01"},{"index":6,"sense_key":"look%2:41:13::","synset_id":"attend.v.02"},{"index":7,"sense_key":"look%2:32:00::","synset_id":"look.v.07"},{"index":8,"sense_key":"look%2:31:00::","synset_id":"expect.v.03"},{"index":9,"sense_key":"look%2:42:02::","synset_id":"look.v.09"},{"index":10,"sense_key":"look%2:31:02::","synset_id":"count.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"look%2:39:00::","call_ids":["senseval3.d000.s106.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"look%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1041,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002202,"input_uncached_usd":0.002082,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9805902499938384}
{"item_id":"senseval3.d000.s106.t002","gold_sense_keys":["eye%1:08:00::"],"candidates":[{"index":1,"sense_key":"eye%1:08:00::","synset_id":"eye.n.01"},{"index":2,"sense_key":"eye%1:09:00::","synset_id":"eye.n.02"},{"index":3,"sense_key":"eye%1:09:01::","synset_id":"eye.n.03"},{"index":4,"sense_key":"eye%1:15:00::","synset_id":"center.n.01"},{"index":5,"sense_key":"eye%1:06:00::","synset_id":"eye.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"eye%1:08:00::","call_ids":["senseval3.d000.s106.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"eye%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":700,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00152,"input_uncached_usd":0.0014,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.141917916946113}
{"item_id":"senseval3.d000.s106.t004","gold_sense_keys":["stranger%1:18:01::"],"candidates":[{"index":1,"sense_key":"stranger%1:18:00::","synset_id":"stranger.n.01"},{"index":2,"sense_key":"stranger%1:18:01::","synset_id":"stranger.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"stranger%1:18:01::","call_ids":["senseval3.d000.s106.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"stranger%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":516,"cached_input_tokens":0,"output_tokens":84,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001872,"input_uncached_usd":0.001032,"input_cached_usd":0.0,"output_usd":0.00084,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6177591250743717}
{"item_id":"senseval3.d000.s106.t005","gold_sense_keys":["sit%2:35:00::"],"candidates":[{"index":1,"sense_key":"sit%2:35:00::","synset_id":"sit.v.01"},{"index":2,"sense_key":"sit%2:42:00::","synset_id":"sit.v.02"},{"index":3,"sense_key":"sit%2:38:00::","synset_id":"sit_down.v.01"},{"index":4,"sense_key":"sit%2:38:01::","synset_id":"sit.v.04"},{"index":5,"sense_key":"sit%2:39:00::","synset_id":"model.v.03"},{"index":6,"sense_key":"sit%2:38:03::","synset_id":"ride.v.01"},{"index":7,"sense_key":"sit%2:42:05::","synset_id":"sit.v.07"},{"index":8,"sense_key":"sit%2:41:05::","synset_id":"baby-sit.v.02"},{"index":9,"sense_key":"sit%2:35:01::","synset_id":"seat.v.01"},{"index":10,"sense_key":"sit%2:33:00::","synset_id":"sit.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"sit%2:35:00::","call_ids":["senseval3.d000.s106.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"sit%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":987,"cached_input_tokens":0,"output_tokens":96,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002934,"input_uncached_usd":0.001974,"input_cached_usd":0.0,"output_usd":0.0009600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.532921916921623}
{"item_id":"senseval3.d000.s106.t006","gold_sense_keys":["table%1:06:01::"],"candidates":[{"index":1,"sense_key":"table%1:14:00::","synset_id":"table.n.01"},{"index":2,"sense_key":"table%1:06:01::","synset_id":"table.n.02"},{"index":3,"sense_key":"table%1:06:02::","synset_id":"table.n.03"},{"index":4,"sense_key":"table%1:17:00::","synset_id":"mesa.n.01"},{"index":5,"sense_key":"table%1:14:01::","synset_id":"table.n.05"},{"index":6,"sense_key":"table%1:13:01::","synset_id":"board.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"table%1:06:01::","call_ids":["senseval3.d000.s106.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"table%1:06:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":776,"cached_input_tokens":0,"output_tokens":91,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002462,"input_uncached_usd":0.001552,"input_cached_usd":0.0,"output_usd":0.0009100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6473702499642968}
{"item_id":"senseval3.d000.s106.t008","gold_sense_keys":["secret%5:00:03:covert:00"],"candidates":[{"index":1,"sense_key":"secret%5:00:00:concealed:00","synset_id":"secret.s.01"},{"index":2,"sense_key":"secret%5:00:01:covert:00","synset_id":"clandestine.s.01"},{"index":3,"sense_key":"secret%5:00:00:unacknowledged:00","synset_id":"unavowed.s.03"},{"index":4,"sense_key":"secret%5:00:03:covert:00","synset_id":"secret.s.04"},{"index":5,"sense_key":"secret%5:00:00:inward:00","synset_id":"secret.s.05"},{"index":6,"sense_key":"secret%5:00:02:concealed:00","synset_id":"hidden.s.02"},{"index":7,"sense_key":"secret%5:00:02:private:00","synset_id":"privy.s.01"},{"index":8,"sense_key":"secret%5:00:00:private:00","synset_id":"confidential.s.02"},{"index":9,"sense_key":"secret%5:00:02:covert:00","synset_id":"secret.s.09"},{"index":10,"sense_key":"secret%5:00:00:esoteric:00","synset_id":"mysterious.s.02"},{"index":11,"sense_key":"secret%5:00:00:classified:02","synset_id":"secret.s.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"secret%5:00:00:inward:00","call_ids":["senseval3.d000.s106.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"secret%5:00:00:inward:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1167,"cached_input_tokens":0,"output_tokens":216,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004494,"input_uncached_usd":0.0023339999999999997,"input_cached_usd":0.0,"output_usd":0.00216,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0848017500247806}
{"item_id":"senseval3.d000.s107.t000","gold_sense_keys":["notice%2:39:00::"],"candidates":[{"index":1,"sense_key":"notice%2:39:04::","synset_id":"detect.v.01"},{"index":2,"sense_key":"notice%2:39:00::","synset_id":"notice.v.02"},{"index":3,"sense_key":"notice%2:32:00::","synset_id":"comment.v.01"},{"index":4,"sense_key":"notice%2:32:01::","synset_id":"notice.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"notice%2:39:00::","call_ids":["senseval3.d000.s107.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"notice%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":732,"cached_input_tokens":0,"output_tokens":106,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002524,"input_uncached_usd":0.001464,"input_cached_usd":0.0,"output_usd":0.0010600000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5403757089516148}
{"item_id":"senseval3.d000.s107.t001","gold_sense_keys":["name%1:10:00::"],"candidates":[{"index":1,"sense_key":"name%1:10:00::","synset_id":"name.n.01"},{"index":2,"sense_key":"name%1:26:00::","synset_id":"name.n.02"},{"index":3,"sense_key":"name%1:14:00::","synset_id":"name.n.03"},{"index":4,"sense_key":"name%1:18:00::","synset_id":"name.n.04"},{"index":5,"sense_key":"name%1:04:00::","synset_id":"name.n.05"},{"index":6,"sense_key":"name%1:10:01::","synset_id":"name.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"name%1:10:00::","call_ids":["senseval3.d000.s107.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"name%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":743,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001606,"input_uncached_usd":0.001486,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9946754169650376}
{"item_id":"senseval3.d000.s107.t002","gold_sense_keys":["see%2:39:00::"],"candidates":[{"index":1,"sense_key":"see%2:39:00::","synset_id":"see.v.01"},{"index":2,"sense_key":"see%2:31:01::","synset_id":"understand.v.02"},{"index":3,"sense_key":"see%2:39:02::","synset_id":"witness.v.02"},{"index":4,"sense_key":"see%2:36:00::","synset_id":"visualize.v.01"},{"index":5,"sense_key":"see%2:31:00::","synset_id":"see.v.05"},{"index":6,"sense_key":"see%2:31:02::","synset_id":"learn.v.02"},{"index":7,"sense_key":"see%2:39:03::","synset_id":"watch.v.03"},{"index":8,"sense_key":"see%2:38:00::","synset_id":"meet.v.01"},{"index":9,"sense_key":"see%2:32:00::","synset_id":"determine.v.08"},{"index":10,"sense_key":"see%2:31:03::","synset_id":"see.v.10"},{"index":11,"sense_key":"see%2:41:11::","synset_id":"see.v.11"},{"index":12,"sense_key":"see%2:41:12::","synset_id":"see.v.12"},{"index":13,"sense_key":"see%2:41:10::","synset_id":"visit.v.01"},{"index":14,"sense_key":"see%2:41:14::","synset_id":"attend.v.02"},{"index":15,"sense_key":"see%2:35:09::","synset_id":"see.v.15"},{"index":16,"sense_key":"see%2:41:00::","synset_id":"go_steady.v.01"},{"index":17,"sense_key":"see%2:39:05::","synset_id":"see.v.17"},{"index":18,"sense_key":"see%2:31:08::","synset_id":"see.v.18"},{"index":19,"sense_key":"see%2:39:11::","synset_id":"see.v.19"},{"index":20,"sense_key":"see%2:39:13::","synset_id":"examine.v.02"},{"index":21,"sense_key":"see%2:39:12::","synset_id":"experience.v.01"},{"index":22,"sense_key":"see%2:38:03::","synset_id":"see.v.22"},{"index":23,"sense_key":"see%2:33:00::","synset_id":"see.v.23"},{"index":24,"sense_key":"see%2:31:13::","synset_id":"interpret.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"see%2:39:00::","call_ids":["senseval3.d000.s107.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"see%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1993,"cached_input_tokens":0,"output_tokens":93,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004916,"input_uncached_usd":0.003986,"input_cached_usd":0.0,"output_usd":0.00093,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9381190410349518}
{"item_id":"senseval3.d000.s107.t003","gold_sense_keys":["sit_down%2:38:00::"],"candidates":[{"index":1,"sense_key":"sit_down%2:38:00::","synset_id":"sit_down.v.01"},{"index":2,"sense_key":"sit_down%2:35:00::","synset_id":"seat.v.01"},{"index":3,"sense_key":"sit_down%2:35:03::","synset_id":"sit.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"sit_down%2:38:00::","call_ids":["senseval3.d000.s107.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"sit_down%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":579,"cached_input_tokens":0,"output_tokens":106,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0022180000000000004,"input_uncached_usd":0.001158,"input_cached_usd":0.0,"output_usd":0.0010600000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7452493750024587}
{"item_id":"senseval3.d000.s108.t000","gold_sense_keys":["man%1:18:00::"],"candidates":[{"index":1,"sense_key":"man%1:18:00::","synset_id":"man.n.01"},{"index":2,"sense_key":"man%1:18:03::","synset_id":"serviceman.n.01"},{"index":3,"sense_key":"man%1:18:04::","synset_id":"man.n.03"},{"index":4,"sense_key":"man%1:05:01::","synset_id":"homo.n.02"},{"index":5,"sense_key":"man%1:18:08::","synset_id":"man.n.05"},{"index":6,"sense_key":"man%1:18:06::","synset_id":"man.n.06"},{"index":7,"sense_key":"man%1:18:05::","synset_id":"valet.n.01"},{"index":8,"sense_key":"man%1:18:07::","synset_id":"man.n.08"},{"index":9,"sense_key":"man%1:15:00::","synset_id":"man.n.09"},{"index":10,"sense_key":"man%1:06:00::","synset_id":"man.n.10"},{"index":11,"sense_key":"man%1:05:00::","synset_id":"world.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"man%1:18:00::","call_ids":["senseval3.d000.s108.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"man%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1203,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0025259999999999996,"input_uncached_usd":0.0024059999999999997,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.433098457986489}
{"item_id":"senseval3.d000.s108.t001","gold_sense_keys":["uncanny%5:00:00:supernatural:00"],"candidates":[{"index":1,"sense_key":"uncanny%5:00:00:supernatural:00","synset_id":"eldritch.s.01"},{"index":2,"sense_key":"uncanny%5:00:00:extraordinary:00","synset_id":"preternatural.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"uncanny%5:00:00:supernatural:00","call_ids":["senseval3.d000.s108.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"uncanny%5:00:00:supernatural:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":668,"cached_input_tokens":0,"output_tokens":101,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002346,"input_uncached_usd":0.001336,"input_cached_usd":0.0,"output_usd":0.00101,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.943450832972303}
{"item_id":"senseval3.d000.s108.t002","gold_sense_keys":["shadow%1:09:00::","shadow%1:26:01::"],"candidates":[{"index":1,"sense_key":"shadow%1:26:01::","synset_id":"shadow.n.01"},{"index":2,"sense_key":"shadow%1:15:00::","synset_id":"darkness.n.02"},{"index":3,"sense_key":"shadow%1:09:00::","synset_id":"apparition.n.03"},{"index":4,"sense_key":"shadow%1:12:00::","synset_id":"shadow.n.04"},{"index":5,"sense_key":"shadow%1:10:00::","synset_id":"trace.n.02"},{"index":6,"sense_key":"shadow%1:07:00::","synset_id":"shadow.n.06"},{"index":7,"sense_key":"shadow%1:26:00::","synset_id":"shadow.n.07"},{"index":8,"sense_key":"shadow%1:18:00::","synset_id":"tail.n.05"},{"index":9,"sense_key":"shadow%1:18:01::","synset_id":"shadow.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"shadow%1:09:00::","call_ids":["senseval3.d000.s108.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"shadow%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":945,"cached_input_tokens":0,"output_tokens":175,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00364,"input_uncached_usd":0.00189,"input_cached_usd":0.0,"output_usd":0.00175,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":7.4474542499519885}
{"item_id":"senseval3.d000.s108.t003","gold_sense_keys":["make%2:36:00::","make%2:36:08::"],"candidates":[{"index":1,"sense_key":"make%2:41:00::","synset_id":"make.v.01"},{"index":2,"sense_key":"make%2:30:00::","synset_id":"make.v.02"},{"index":3,"sense_key":"make%2:36:00::","synset_id":"make.v.03"},{"index":4,"sense_key":"make%2:32:00::","synset_id":"induce.v.02"},{"index":5,"sense_key":"make%2:36:08::","synset_id":"cause.v.01"},{"index":6,"sense_key":"make%2:36:01::","synset_id":"produce.v.02"},{"index":7,"sense_key":"make%2:31:00::","synset_id":"draw.v.04"},{"index":8,"sense_key":"make%2:36:15::","synset_id":"make.v.08"},{"index":9,"sense_key":"make%2:36:09::","synset_id":"create.v.05"},{"index":10,"sense_key":"make%2:40:01::","synset_id":"gain.v.08"},{"index":11,"sense_key":"make%2:36:05::","synset_id":"do.v.08"},{"index":12,"sense_key":"make%2:42:05::","synset_id":"form.v.02"},{"index":13,"sense_key":"make%2:38:02::","synset_id":"reach.v.07"},{"index":14,"sense_key":"make%2:42:00::","synset_id":"make.v.14"},{"index":15,"sense_key":"make%2:36:11::","synset_id":"make.v.15"},{"index":16,"sense_key":"make%2:36:12::","synset_id":"make.v.16"},{"index":17,"sense_key":"make%2:36:13::","synset_id":"construct.v.01"},{"index":18,"sense_key":"make%2:30:15::","synset_id":"make.v.18"},{"index":19,"sense_key":"make%2:29:08::","synset_id":"make.v.19"},{"index":20,"sense_key":"make%2:41:03::","synset_id":"name.v.03"},{"index":21,"sense_key":"make%2:40:02::","synset_id":"have.v.17"},{"index":22,"sense_key":"make%2:38:00::","synset_id":"reach.v.01"},{"index":23,"sense_key":"make%2:31:13::","synset_id":"lay_down.v.01"},{"index":24,"sense_key":"make%2:41:13::","synset_id":"make.v.24"},{"index":25,"sense_key":"make%2:41:01::","synset_id":"make.v.25"},{"index":26,"sense_key":"make%2:36:04::","synset_id":"hold.v.03"},{"index":27,"sense_key":"make%2:30:02::","synset_id":"make.v.27"},{"index":28,"sense_key":"make%2:38:05::","synset_id":"take.v.27"},{"index":29,"sense_key":"make%2:29:00::","synset_id":"stool.v.04"},{"index":30,"sense_key":"make%2:42:09::","synset_id":"make.v.30"},{"index":31,"sense_key":"make%2:42:13::","synset_id":"make.v.31"},{"index":32,"sense_key":"make%2:42:12::","synset_id":"make.v.32"},{"index":33,"sense_key":"make%2:42:11::","synset_id":"make.v.33"},{"index":34,"sense_key":"make%2:42:15::","synset_id":"make.v.34"},{"index":35,"sense_key":"make%2:39:12::","synset_id":"make.v.35"},{"index":36,"sense_key":"make%2:38:10::","synset_id":"make.v.36"},{"index":37,"sense_key":"make%2:38:15::","synset_id":"make.v.37"},{"index":38,"sense_key":"make%2:36:10::","synset_id":"make.v.38"},{"index":39,"sense_key":"make%2:36:07::","synset_id":"cook.v.02"},{"index":40,"sense_key":"make%2:35:13::","synset_id":"seduce.v.01"},{"index":41,"sense_key":"make%2:32:14::","synset_id":"make.v.41"},{"index":42,"sense_key":"make%2:32:09::","synset_id":"make.v.42"},{"index":43,"sense_key":"make%2:31:10::","synset_id":"make.v.43"},{"index":44,"sense_key":"make%2:31:12::","synset_id":"make.v.44"},{"index":45,"sense_key":"make%2:30:10::","synset_id":"make.v.45"},{"index":46,"sense_key":"make%2:30:14::","synset_id":"make.v.46"},{"index":47,"sense_key":"make%2:30:12::","synset_id":"make.v.47"},{"index":48,"sense_key":"make%2:29:06::","synset_id":"make.v.48"},{"index":49,"sense_key":"make%2:29:01::","synset_id":"make.v.49"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"make%2:36:08::","call_ids":["senseval3.d000.s108.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"make%2:36:08::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":3129,"cached_input_tokens":0,"output_tokens":173,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.007988,"input_uncached_usd":0.006258,"input_cached_usd":0.0,"output_usd":0.0017300000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9707796250004321}
{"item_id":"senseval3.d000.s108.t005","gold_sense_keys":["noise%1:11:00::"],"candidates":[{"index":1,"sense_key":"noise%1:11:00::","synset_id":"noise.n.01"},{"index":2,"sense_key":"noise%1:09:00::","synset_id":"noise.n.02"},{"index":3,"sense_key":"noise%1:11:01::","synset_id":"noise.n.03"},{"index":4,"sense_key":"noise%1:10:00::","synset_id":"noise.n.04"},{"index":5,"sense_key":"noise%1:07:01::","synset_id":"noise.n.05"},{"index":6,"sense_key":"noise%1:07:00::","synset_id":"randomness.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"noise%1:11:00::","call_ids":["senseval3.d000.s108.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"noise%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":892,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001904,"input_uncached_usd":0.001784,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.15098495897837}
{"item_id":"senseval3.d000.s108.t006","gold_sense_keys":["shadow%1:09:00::","shadow%1:26:01::"],"candidates":[{"index":1,"sense_key":"shadow%1:26:01::","synset_id":"shadow.n.01"},{"index":2,"sense_key":"shadow%1:15:00::","synset_id":"darkness.n.02"},{"index":3,"sense_key":"shadow%1:09:00::","synset_id":"apparition.n.03"},{"index":4,"sense_key":"shadow%1:12:00::","synset_id":"shadow.n.04"},{"index":5,"sense_key":"shadow%1:10:00::","synset_id":"trace.n.02"},{"index":6,"sense_key":"shadow%1:07:00::","synset_id":"shadow.n.06"},{"index":7,"sense_key":"shadow%1:26:00::","synset_id":"shadow.n.07"},{"index":8,"sense_key":"shadow%1:18:00::","synset_id":"tail.n.05"},{"index":9,"sense_key":"shadow%1:18:01::","synset_id":"shadow.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"shadow%1:26:01::","call_ids":["senseval3.d000.s108.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"shadow%1:26:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":945,"cached_input_tokens":0,"output_tokens":321,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0051,"input_uncached_usd":0.00189,"input_cached_usd":0.0,"output_usd":0.00321,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.54940879100468}
{"item_id":"senseval3.d000.s109.t001","gold_sense_keys":["shrink%2:30:00::","shrink%2:38:00::"],"candidates":[{"index":1,"sense_key":"shrink%2:30:02::","synset_id":"shrivel.v.01"},{"index":2,"sense_key":"shrink%2:38:00::","synset_id":"flinch.v.01"},{"index":3,"sense_key":"shrink%2:30:01::","synset_id":"shrink.v.03"},{"index":4,"sense_key":"shrink%2:30:00::","synset_id":"shrink.v.04"},{"index":5,"sense_key":"shrink%2:30:11::","synset_id":"shrink.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"shrink%2:38:00::","call_ids":["senseval3.d000.s109.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"shrink%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":828,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001776,"input_uncached_usd":0.001656,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4001810409827158}
{"item_id":"senseval3.d000.s109.t003","gold_sense_keys":["trap%2:35:00::","trap%2:35:02::","trap%2:41:00::"],"candidates":[{"index":1,"sense_key":"trap%2:41:00::","synset_id":"trap.v.01"},{"index":2,"sense_key":"trap%2:35:00::","synset_id":"trap.v.02"},{"index":3,"sense_key":"trap%2:35:01::","synset_id":"trap.v.03"},{"index":4,"sense_key":"trap%2:35:02::","synset_id":"trap.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"trap%2:41:00::","call_ids":["senseval3.d000.s109.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"trap%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":718,"cached_input_tokens":0,"output_tokens":84,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0022760000000000002,"input_uncached_usd":0.001436,"input_cached_usd":0.0,"output_usd":0.00084,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.55536162503995}
{"item_id":"senseval3.d000.s109.t004","gold_sense_keys":["corner%1:06:02::"],"candidates":[{"index":1,"sense_key":"corner%1:15:02::","synset_id":"corner.n.01"},{"index":2,"sense_key":"corner%1:15:00::","synset_id":"corner.n.02"},{"index":3,"sense_key":"corner%1:06:02::","synset_id":"corner.n.03"},{"index":4,"sense_key":"corner%1:06:00::","synset_id":"corner.n.04"},{"index":5,"sense_key":"corner%1:15:01::","synset_id":"corner.n.05"},{"index":6,"sense_key":"corner%1:25:00::","synset_id":"recess.n.02"},{"index":7,"sense_key":"corner%1:26:00::","synset_id":"corner.n.07"},{"index":8,"sense_key":"corner%1:26:01::","synset_id":"corner.n.08"},{"index":9,"sense_key":"corner%1:17:00::","synset_id":"corner.n.09"},{"index":10,"sense_key":"corner%1:15:03::","synset_id":"corner.n.10"},{"index":11,"sense_key":"corner%1:06:01::","synset_id":"corner.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"corner%1:06:02::","call_ids":["senseval3.d000.s109.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"corner%1:06:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1062,"cached_input_tokens":0,"output_tokens":129,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0034140000000000004,"input_uncached_usd":0.002124,"input_cached_usd":0.0,"output_usd":0.0012900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.101717874989845}
{"item_id":"senseval3.d000.s109.t005","gold_sense_keys":["wiry%5:00:00:thin:03"],"candidates":[{"index":1,"sense_key":"wiry%5:00:00:thin:03","synset_id":"stringy.s.01"},{"index":2,"sense_key":"wiry%3:01:00::","synset_id":"wiry.a.02"},{"index":3,"sense_key":"wiry%5:00:00:hairy:00","synset_id":"wiry.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"wiry%5:00:00:thin:03","call_ids":["senseval3.d000.s109.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"wiry%5:00:00:thin:03","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":601,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001322,"input_uncached_usd":0.001202,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1366683749947697}
{"item_id":"senseval3.d000.s109.t007","gold_sense_keys":["little%3:00:00::","little%3:00:01::"],"candidates":[{"index":1,"sense_key":"little%3:00:01::","synset_id":"small.a.01"},{"index":2,"sense_key":"little%3:00:03::","synset_id":"little.a.02"},{"index":3,"sense_key":"little%5:00:00:young:00","synset_id":"little.s.01"},{"index":4,"sense_key":"little%5:00:00:unimportant:00","synset_id":"fiddling.s.01"},{"index":5,"sense_key":"little%5:00:00:soft:04","synset_id":"little.s.03"},{"index":6,"sense_key":"little%3:00:00::","synset_id":"short.a.03"},{"index":7,"sense_key":"little%5:00:00:lowercase:00","synset_id":"little.s.04"},{"index":8,"sense_key":"little%5:00:00:emotional:00","synset_id":"little.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"little%3:00:00::","call_ids":["senseval3.d000.s109.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"little%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":989,"cached_input_tokens":0,"output_tokens":111,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0030879999999999996,"input_uncached_usd":0.0019779999999999997,"input_cached_usd":0.0,"output_usd":0.00111,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7602507910924032}
{"item_id":"senseval3.d000.s109.t008","gold_sense_keys":["man%1:18:00::"],"candidates":[{"index":1,"sense_key":"man%1:18:00::","synset_id":"man.n.01"},{"index":2,"sense_key":"man%1:18:03::","synset_id":"serviceman.n.01"},{"index":3,"sense_key":"man%1:18:04::","synset_id":"man.n.03"},{"index":4,"sense_key":"man%1:05:01::","synset_id":"homo.n.02"},{"index":5,"sense_key":"man%1:18:08::","synset_id":"man.n.05"},{"index":6,"sense_key":"man%1:18:06::","synset_id":"man.n.06"},{"index":7,"sense_key":"man%1:18:05::","synset_id":"valet.n.01"},{"index":8,"sense_key":"man%1:18:07::","synset_id":"man.n.08"},{"index":9,"sense_key":"man%1:15:00::","synset_id":"man.n.09"},{"index":10,"sense_key":"man%1:06:00::","synset_id":"man.n.10"},{"index":11,"sense_key":"man%1:05:00::","synset_id":"world.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"man%1:18:00::","call_ids":["senseval3.d000.s109.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"man%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1213,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0025459999999999997,"input_uncached_usd":0.0024259999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.348980167065747}
{"item_id":"senseval3.d000.s110.t000","gold_sense_keys":["begin%2:30:00::"],"candidates":[{"index":1,"sense_key":"begin%2:30:00::","synset_id":"get_down.v.07"},{"index":2,"sense_key":"begin%2:42:00::","synset_id":"begin.v.02"},{"index":3,"sense_key":"begin%2:30:01::","synset_id":"begin.v.03"},{"index":4,"sense_key":"begin%2:32:03::","synset_id":"begin.v.04"},{"index":5,"sense_key":"begin%2:42:02::","synset_id":"begin.v.05"},{"index":6,"sense_key":"begin%2:42:04::","synset_id":"begin.v.06"},{"index":7,"sense_key":"begin%2:42:03::","synset_id":"begin.v.07"},{"index":8,"sense_key":"begin%2:41:04::","synset_id":"begin.v.08"},{"index":9,"sense_key":"begin%2:41:00::","synset_id":"begin.v.09"},{"index":10,"sense_key":"begin%2:32:04::","synset_id":"begin.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"begin%2:30:00::","call_ids":["senseval3.d000.s110.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"begin%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1200,"cached_input_tokens":0,"output_tokens":111,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00351,"input_uncached_usd":0.0024,"input_cached_usd":0.0,"output_usd":0.00111,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5857670420082286}
{"item_id":"senseval3.d000.s110.t001","gold_sense_keys":["wish%2:32:02::","wish%2:37:00::"],"candidates":[{"index":1,"sense_key":"wish%2:37:00::","synset_id":"wish.v.01"},{"index":2,"sense_key":"wish%2:37:02::","synset_id":"wish.v.02"},{"index":3,"sense_key":"wish%2:32:02::","synset_id":"wish.v.03"},{"index":4,"sense_key":"wish%2:37:01::","synset_id":"wish.v.04"},{"index":5,"sense_key":"wish%2:32:01::","synset_id":"wish.v.05"},{"index":6,"sense_key":"wish%2:32:00::","synset_id":"wish.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"wish%2:37:00::","call_ids":["senseval3.d000.s110.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"wish%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":774,"cached_input_tokens":0,"output_tokens":184,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003388,"input_uncached_usd":0.001548,"input_cached_usd":0.0,"output_usd":0.00184,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.159665958955884}
{"item_id":"senseval3.d000.s110.t002","gold_sense_keys":["shout%2:32:01::"],"candidates":[{"index":1,"sense_key":"shout%2:32:00::","synset_id":"shout.v.01"},{"index":2,"sense_key":"shout%2:32:01::","synset_id":"shout.v.02"},{"index":3,"sense_key":"shout%2:32:08::","synset_id":"exclaim.v.01"},{"index":4,"sense_key":"shout%2:32:12::","synset_id":"abuse.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"shout%2:32:01::","call_ids":["senseval3.d000.s110.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"shout%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":842,"cached_input_tokens":0,"output_tokens":106,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002744,"input_uncached_usd":0.001684,"input_cached_usd":0.0,"output_usd":0.0010600000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.103184042032808}
{"item_id":"senseval3.d000.s110.t003","gold_sense_keys":["other%5:00:00:past:00"],"candidates":[{"index":1,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":2,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":3,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"other%5:00:00:past:00","call_ids":["senseval3.d000.s110.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"other%5:00:00:past:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":721,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001562,"input_uncached_usd":0.001442,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0858172910520807}
{"item_id":"senseval3.d000.s110.t004","gold_sense_keys":["evening%1:28:00::"],"candidates":[{"index":1,"sense_key":"evening%1:28:00::","synset_id":"evening.n.01"},{"index":2,"sense_key":"evening%1:28:01::","synset_id":"evening.n.02"},{"index":3,"sense_key":"evening%1:28:02::","synset_id":"evening.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"evening%1:28:00::","call_ids":["senseval3.d000.s110.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"evening%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":673,"cached_input_tokens":0,"output_tokens":150,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002846,"input_uncached_usd":0.001346,"input_cached_usd":0.0,"output_usd":0.0015,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.662289291038178}
{"item_id":"senseval3.d000.s110.t005","gold_sense_keys":["truck%1:06:00::"],"candidates":[{"index":1,"sense_key":"truck%1:06:00::","synset_id":"truck.n.01"},{"index":2,"sense_key":"truck%1:06:01::","synset_id":"hand_truck.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"truck%1:06:00::","call_ids":["senseval3.d000.s110.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"truck%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":611,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001342,"input_uncached_usd":0.001222,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5017815419705585}
{"item_id":"senseval3.d000.s110.t006","gold_sense_keys":["bear_down%2:33:03::"],"candidates":[{"index":1,"sense_key":"bear_down%2:35:00::","synset_id":"drag_down.v.01"},{"index":2,"sense_key":"bear_down%2:35:01::","synset_id":"bear_down.v.02"},{"index":3,"sense_key":"bear_down%2:33:03::","synset_id":"charge.v.01"},{"index":4,"sense_key":"bear_down%2:33:00::","synset_id":"bear_down.v.04"},{"index":5,"sense_key":"bear_down%2:32:12::","synset_id":"bear_down.v.05"},{"index":6,"sense_key":"bear_down%2:32:01::","synset_id":"bear_down.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"bear_down%2:33:03::","call_ids":["senseval3.d000.s110.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"bear_down%2:33:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":828,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002286,"input_uncached_usd":0.001656,"input_cached_usd":0.0,"output_usd":0.00063,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8225769170094281}
{"item_id":"senseval3.d000.s110.t007","gold_sense_keys":["crossing%1:06:00::","crossing%1:06:01::"],"candidates":[{"index":1,"sense_key":"crossing%1:04:01::","synset_id":"crossing.n.01"},{"index":2,"sense_key":"crossing%1:17:00::","synset_id":"ford.n.07"},{"index":3,"sense_key":"crossing%1:15:00::","synset_id":"crossing.n.03"},{"index":4,"sense_key":"crossing%1:06:01::","synset_id":"intersection.n.02"},{"index":5,"sense_key":"crossing%1:06:00::","synset_id":"crossing.n.05"},{"index":6,"sense_key":"crossing%1:04:02::","synset_id":"hybridization.n.01"},{"index":7,"sense_key":"crossing%1:04:00::","synset_id":"crossing.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"crossing%1:06:01::","call_ids":["senseval3.d000.s110.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"crossing%1:06:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":873,"cached_input_tokens":0,"output_tokens":199,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0037359999999999997,"input_uncached_usd":0.001746,"input_cached_usd":0.0,"output_usd":0.00199,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.5221170419827104}
{"item_id":"senseval3.d000.s111.t000","gold_sense_keys":["saddle%2:35:10::"],"candidates":[{"index":1,"sense_key":"saddle%2:35:01::","synset_id":"saddle.v.01"},{"index":2,"sense_key":"saddle%2:35:10::","synset_id":"saddle.v.02"},{"index":3,"sense_key":"saddle%2:32:00::","synset_id":"charge.v.18"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"saddle%2:35:10::","call_ids":["senseval3.d000.s111.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"saddle%2:35:10::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":611,"cached_input_tokens":0,"output_tokens":153,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002752,"input_uncached_usd":0.001222,"input_cached_usd":0.0,"output_usd":0.0015300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8692033329280093}
{"item_id":"senseval3.d000.s111.t001","gold_sense_keys":["creep%1:18:00::"],"candidates":[{"index":1,"sense_key":"creep%1:18:00::","synset_id":"creep.n.01"},{"index":2,"sense_key":"creep%1:11:00::","synset_id":"creep.n.02"},{"index":3,"sense_key":"creep%1:06:00::","synset_id":"creep.n.03"},{"index":4,"sense_key":"creep%1:04:00::","synset_id":"crawl.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"creep%1:18:00::","call_ids":["senseval3.d000.s111.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"creep%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":680,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00148,"input_uncached_usd":0.0013599999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9721256250049919}
{"item_id":"senseval3.d000.s112.t000","gold_sense_keys":["start%2:30:00::"],"candidates":[{"index":1,"sense_key":"start%2:30:00::","synset_id":"get_down.v.07"},{"index":2,"sense_key":"start%2:30:01::","synset_id":"begin.v.03"},{"index":3,"sense_key":"start%2:38:02::","synset_id":"depart.v.03"},{"index":4,"sense_key":"start%2:42:00::","synset_id":"begin.v.02"},{"index":5,"sense_key":"start%2:36:00::","synset_id":"originate.v.02"},{"index":6,"sense_key":"start%2:36:01::","synset_id":"start.v.06"},{"index":7,"sense_key":"start%2:38:04::","synset_id":"startle.v.02"},{"index":8,"sense_key":"start%2:38:01::","synset_id":"start.v.08"},{"index":9,"sense_key":"start%2:38:00::","synset_id":"start.v.09"},{"index":10,"sense_key":"start%2:41:00::","synset_id":"start.v.10"},{"index":11,"sense_key":"start%2:33:00::","synset_id":"start.v.11"},{"index":12,"sense_key":"start%2:42:03::","synset_id":"begin.v.07"},{"index":13,"sense_key":"start%2:41:04::","synset_id":"begin.v.08"},{"index":14,"sense_key":"start%2:38:03::","synset_id":"start.v.14"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"start%2:30:00::","call_ids":["senseval3.d000.s112.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"start%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1322,"cached_input_tokens":0,"output_tokens":81,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003454,"input_uncached_usd":0.002644,"input_cached_usd":0.0,"output_usd":0.0008100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6550187080865726}
{"item_id":"senseval3.d000.s112.t001","gold_sense_keys":["go%2:38:00::"],"candidates":[{"index":1,"sense_key":"go%2:38:00::","synset_id":"travel.v.01"},{"index":2,"sense_key":"go%2:41:00::","synset_id":"go.v.02"},{"index":3,"sense_key":"go%2:38:01::","synset_id":"go.v.03"},{"index":4,"sense_key":"go%2:30:04::","synset_id":"become.v.01"},{"index":5,"sense_key":"go%2:42:06::","synset_id":"go.v.05"},{"index":6,"sense_key":"go%2:42:02::","synset_id":"run.v.05"},{"index":7,"sense_key":"go%2:42:00::","synset_id":"run.v.03"},{"index":8,"sense_key":"go%2:30:01::","synset_id":"proceed.v.04"},{"index":9,"sense_key":"go%2:30:08::","synset_id":"go.v.09"},{"index":10,"sense_key":"go%2:42:07::","synset_id":"go.v.10"},{"index":11,"sense_key":"go%2:39:00::","synset_id":"sound.v.02"},{"index":12,"sense_key":"go%2:35:00::","synset_id":"function.v.01"},{"index":13,"sense_key":"go%2:34:00::","synset_id":"run_low.v.01"},{"index":14,"sense_key":"go%2:30:02::","synset_id":"move.v.13"},{"index":15,"sense_key":"go%2:42:14::","synset_id":"survive.v.01"},{"index":16,"sense_key":"go%2:42:03::","synset_id":"go.v.16"},{"index":17,"sense_key":"go%2:30:00::","synset_id":"die.v.01"},{"index":18,"sense_key":"go%2:42:04::","synset_id":"belong.v.03"},{"index":19,"sense_key":"go%2:42:10::","synset_id":"go.v.19"},{"index":20,"sense_key":"go%2:38:09::","synset_id":"start.v.09"},{"index":21,"sense_key":"go%2:33:00::","synset_id":"move.v.15"},{"index":22,"sense_key":"go%2:42:13::","synset_id":"go.v.22"},{"index":23,"sense_key":"go%2:42:12::","synset_id":"go.v.23"},{"index":24,"sense_key":"go%2:42:11::","synset_id":"blend.v.02"},{"index":25,"sense_key":"go%2:42:08::","synset_id":"go.v.25"},{"index":26,"sense_key":"go%2:42:05::","synset_id":"fit.v.02"},{"index":27,"sense_key":"go%2:35:13::","synset_id":"rifle.v.02"},{"index":28,"sense_key":"go%2:34:09::","synset_id":"go.v.28"},{"index":29,"sense_key":"go%2:31:12::","synset_id":"plump.v.04"},{"index":30,"sense_key":"go%2:30:03::","synset_id":"fail.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"go%2:38:00::","call_ids":["senseval3.d000.s112.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"go%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2340,"cached_input_tokens":0,"output_tokens":109,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00577,"input_uncached_usd":0.00468,"input_cached_usd":0.0,"output_usd":0.00109,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.822026624926366}
{"item_id":"senseval3.d000.s112.t002","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":2,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":3,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"other%3:00:00::","call_ids":["senseval3.d000.s112.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":672,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002114,"input_uncached_usd":0.001344,"input_cached_usd":0.0,"output_usd":0.0007700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6249578749993816}
{"item_id":"senseval3.d000.s112.t003","gold_sense_keys":["place%1:15:04::"],"candidates":[{"index":1,"sense_key":"place%1:15:00::","synset_id":"topographic_point.n.01"},{"index":2,"sense_key":"place%1:15:04::","synset_id":"place.n.02"},{"index":3,"sense_key":"place%1:09:00::","synset_id":"place.n.03"},{"index":4,"sense_key":"place%1:15:01::","synset_id":"place.n.04"},{"index":5,"sense_key":"place%1:04:01::","synset_id":"stead.n.01"},{"index":6,"sense_key":"place%1:26:01::","synset_id":"place.n.06"},{"index":7,"sense_key":"place%1:15:06::","synset_id":"home.n.01"},{"index":8,"sense_key":"place%1:04:00::","synset_id":"position.n.06"},{"index":9,"sense_key":"place%1:15:05::","synset_id":"position.n.01"},{"index":10,"sense_key":"place%1:26:00::","synset_id":"place.n.10"},{"index":11,"sense_key":"place%1:15:03::","synset_id":"seat.n.01"},{"index":12,"sense_key":"place%1:10:00::","synset_id":"place.n.12"},{"index":13,"sense_key":"place%1:26:02::","synset_id":"place.n.13"},{"index":14,"sense_key":"place%1:15:02::","synset_id":"plaza.n.01"},{"index":15,"sense_key":"place%1:10:02::","synset_id":"place.n.15"},{"index":16,"sense_key":"place%1:10:01::","synset_id":"space.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"place%1:15:00::","call_ids":["senseval3.d000.s112.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"place%1:15:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1359,"cached_input_tokens":0,"output_tokens":138,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004098,"input_uncached_usd":0.002718,"input_cached_usd":0.0,"output_usd":0.0013800000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1722362090367824}
{"item_id":"senseval3.d000.s113.t000","gold_sense_keys":["low%5:00:00:soft:04"],"candidates":[{"index":1,"sense_key":"low%3:00:02::","synset_id":"low.a.01"},{"index":2,"sense_key":"low%3:00:01::","synset_id":"low.a.02"},{"index":3,"sense_key":"low%5:00:00:soft:04","synset_id":"low.s.01"},{"index":4,"sense_key":"low%5:00:00:unrefined:01","synset_id":"low.s.02"},{"index":5,"sense_key":"low%3:00:03::","synset_id":"low.a.05"},{"index":6,"sense_key":"low%5:00:00:contemptible:00","synset_id":"abject.s.01"},{"index":7,"sense_key":"low%5:00:00:inferior:01","synset_id":"humble.s.01"},{"index":8,"sense_key":"low%5:00:00:insufficient:00","synset_id":"depleted.s.01"},{"index":9,"sense_key":"low%5:00:00:humble:00","synset_id":"broken.s.01"},{"index":10,"sense_key":"low%5:00:00:dejected:00","synset_id":"gloomy.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"low%5:00:00:soft:04","call_ids":["senseval3.d000.s113.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"low%5:00:00:soft:04","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1075,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00281,"input_uncached_usd":0.00215,"input_cached_usd":0.0,"output_usd":0.0006600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2626776669640094}
{"item_id":"senseval3.d000.s113.t001","gold_sense_keys":["voice%1:07:00::"],"candidates":[{"index":1,"sense_key":"voice%1:07:00::","synset_id":"voice.n.01"},{"index":2,"sense_key":"voice%1:10:00::","synset_id":"voice.n.02"},{"index":3,"sense_key":"voice%1:11:00::","synset_id":"voice.n.03"},{"index":4,"sense_key":"voice%1:10:01::","synset_id":"articulation.n.03"},{"index":5,"sense_key":"voice%1:04:00::","synset_id":"voice.n.05"},{"index":6,"sense_key":"voice%1:10:03::","synset_id":"voice.n.06"},{"index":7,"sense_key":"voice%1:18:01::","synset_id":"voice.n.07"},{"index":8,"sense_key":"voice%1:18:00::","synset_id":"spokesperson.n.01"},{"index":9,"sense_key":"voice%1:07:01::","synset_id":"voice.n.09"},{"index":10,"sense_key":"voice%1:24:00::","synset_id":"voice.n.10"},{"index":11,"sense_key":"voice%1:10:02::","synset_id":"part.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"voice%1:07:00::","call_ids":["senseval3.d000.s113.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"voice%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1161,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003042,"input_uncached_usd":0.002322,"input_cached_usd":0.0,"output_usd":0.00072,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.4104519999818876}
{"item_id":"senseval3.d000.s113.t003","gold_sense_keys":["man%1:18:00::"],"candidates":[{"index":1,"sense_key":"man%1:18:00::","synset_id":"man.n.01"},{"index":2,"sense_key":"man%1:18:03::","synset_id":"serviceman.n.01"},{"index":3,"sense_key":"man%1:18:04::","synset_id":"man.n.03"},{"index":4,"sense_key":"man%1:05:01::","synset_id":"homo.n.02"},{"index":5,"sense_key":"man%1:18:08::","synset_id":"man.n.05"},{"index":6,"sense_key":"man%1:18:06::","synset_id":"man.n.06"},{"index":7,"sense_key":"man%1:18:05::","synset_id":"valet.n.01"},{"index":8,"sense_key":"man%1:18:07::","synset_id":"man.n.08"},{"index":9,"sense_key":"man%1:15:00::","synset_id":"man.n.09"},{"index":10,"sense_key":"man%1:06:00::","synset_id":"man.n.10"},{"index":11,"sense_key":"man%1:05:00::","synset_id":"world.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"man%1:18:00::","call_ids":["senseval3.d000.s113.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"man%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1183,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002486,"input_uncached_usd":0.002366,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8037089999997988}
{"item_id":"senseval3.d000.s113.t004","gold_sense_keys":["ask%2:32:04::"],"candidates":[{"index":1,"sense_key":"ask%2:32:00::","synset_id":"ask.v.01"},{"index":2,"sense_key":"ask%2:32:01::","synset_id":"ask.v.02"},{"index":3,"sense_key":"ask%2:32:02::","synset_id":"ask.v.03"},{"index":4,"sense_key":"ask%2:32:05::","synset_id":"ask.v.04"},{"index":5,"sense_key":"ask%2:32:04::","synset_id":"ask.v.05"},{"index":6,"sense_key":"ask%2:42:00::","synset_id":"necessitate.v.01"},{"index":7,"sense_key":"ask%2:32:09::","synset_id":"ask.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"ask%2:32:04::","call_ids":["senseval3.d000.s113.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"ask%2:32:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":877,"cached_input_tokens":0,"output_tokens":129,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003044,"input_uncached_usd":0.001754,"input_cached_usd":0.0,"output_usd":0.0012900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.085912915994413}
{"item_id":"senseval3.d000.s113.t005","gold_sense_keys":["happy%3:00:00::"],"candidates":[{"index":1,"sense_key":"happy%3:00:00::","synset_id":"happy.a.01"},{"index":2,"sense_key":"happy%5:00:00:fortunate:00","synset_id":"felicitous.s.01"},{"index":3,"sense_key":"happy%5:00:00:willing:00","synset_id":"glad.s.01"},{"index":4,"sense_key":"happy%5:00:00:felicitous:00","synset_id":"happy.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"happy%3:00:00::","call_ids":["senseval3.d000.s113.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"happy%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":691,"cached_input_tokens":0,"output_tokens":133,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002712,"input_uncached_usd":0.001382,"input_cached_usd":0.0,"output_usd":0.00133,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8336683749221265}
{"item_id":"senseval3.d000.s113.t006","gold_sense_keys":["way%1:04:01::","way%1:07:01::"],"candidates":[{"index":1,"sense_key":"way%1:07:01::","synset_id":"manner.n.01"},{"index":2,"sense_key":"way%1:04:01::","synset_id":"means.n.01"},{"index":3,"sense_key":"way%1:15:00::","synset_id":"direction.n.01"},{"index":4,"sense_key":"way%1:26:00::","synset_id":"way.n.04"},{"index":5,"sense_key":"way%1:04:00::","synset_id":"way.n.05"},{"index":6,"sense_key":"way%1:06:00::","synset_id":"way.n.06"},{"index":7,"sense_key":"way%1:04:02::","synset_id":"way.n.07"},{"index":8,"sense_key":"way%1:23:00::","synset_id":"room.n.02"},{"index":9,"sense_key":"way%1:07:02::","synset_id":"way.n.09"},{"index":10,"sense_key":"way%1:09:00::","synset_id":"way.n.10"},{"index":11,"sense_key":"way%1:09:01::","synset_id":"way.n.11"},{"index":12,"sense_key":"way%1:21:00::","synset_id":"way.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"way%1:07:01::","call_ids":["senseval3.d000.s113.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"way%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1061,"cached_input_tokens":0,"output_tokens":151,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003632,"input_uncached_usd":0.0021219999999999998,"input_cached_usd":0.0,"output_usd":0.00151,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0442005830118433}
{"item_id":"senseval3.d000.s114.t000","gold_sense_keys":["make%2:41:13::"],"candidates":[{"index":1,"sense_key":"make%2:41:00::","synset_id":"make.v.01"},{"index":2,"sense_key":"make%2:30:00::","synset_id":"make.v.02"},{"index":3,"sense_key":"make%2:36:00::","synset_id":"make.v.03"},{"index":4,"sense_key":"make%2:32:00::","synset_id":"induce.v.02"},{"index":5,"sense_key":"make%2:36:08::","synset_id":"cause.v.01"},{"index":6,"sense_key":"make%2:36:01::","synset_id":"produce.v.02"},{"index":7,"sense_key":"make%2:31:00::","synset_id":"draw.v.04"},{"index":8,"sense_key":"make%2:36:15::","synset_id":"make.v.08"},{"index":9,"sense_key":"make%2:36:09::","synset_id":"create.v.05"},{"index":10,"sense_key":"make%2:40:01::","synset_id":"gain.v.08"},{"index":11,"sense_key":"make%2:36:05::","synset_id":"do.v.08"},{"index":12,"sense_key":"make%2:42:05::","synset_id":"form.v.02"},{"index":13,"sense_key":"make%2:38:02::","synset_id":"reach.v.07"},{"index":14,"sense_key":"make%2:42:00::","synset_id":"make.v.14"},{"index":15,"sense_key":"make%2:36:11::","synset_id":"make.v.15"},{"index":16,"sense_key":"make%2:36:12::","synset_id":"make.v.16"},{"index":17,"sense_key":"make%2:36:13::","synset_id":"construct.v.01"},{"index":18,"sense_key":"make%2:30:15::","synset_id":"make.v.18"},{"index":19,"sense_key":"make%2:29:08::","synset_id":"make.v.19"},{"index":20,"sense_key":"make%2:41:03::","synset_id":"name.v.03"},{"index":21,"sense_key":"make%2:40:02::","synset_id":"have.v.17"},{"index":22,"sense_key":"make%2:38:00::","synset_id":"reach.v.01"},{"index":23,"sense_key":"make%2:31:13::","synset_id":"lay_down.v.01"},{"index":24,"sense_key":"make%2:41:13::","synset_id":"make.v.24"},{"index":25,"sense_key":"make%2:41:01::","synset_id":"make.v.25"},{"index":26,"sense_key":"make%2:36:04::","synset_id":"hold.v.03"},{"index":27,"sense_key":"make%2:30:02::","synset_id":"make.v.27"},{"index":28,"sense_key":"make%2:38:05::","synset_id":"take.v.27"},{"index":29,"sense_key":"make%2:29:00::","synset_id":"stool.v.04"},{"index":30,"sense_key":"make%2:42:09::","synset_id":"make.v.30"},{"index":31,"sense_key":"make%2:42:13::","synset_id":"make.v.31"},{"index":32,"sense_key":"make%2:42:12::","synset_id":"make.v.32"},{"index":33,"sense_key":"make%2:42:11::","synset_id":"make.v.33"},{"index":34,"sense_key":"make%2:42:15::","synset_id":"make.v.34"},{"index":35,"sense_key":"make%2:39:12::","synset_id":"make.v.35"},{"index":36,"sense_key":"make%2:38:10::","synset_id":"make.v.36"},{"index":37,"sense_key":"make%2:38:15::","synset_id":"make.v.37"},{"index":38,"sense_key":"make%2:36:10::","synset_id":"make.v.38"},{"index":39,"sense_key":"make%2:36:07::","synset_id":"cook.v.02"},{"index":40,"sense_key":"make%2:35:13::","synset_id":"seduce.v.01"},{"index":41,"sense_key":"make%2:32:14::","synset_id":"make.v.41"},{"index":42,"sense_key":"make%2:32:09::","synset_id":"make.v.42"},{"index":43,"sense_key":"make%2:31:10::","synset_id":"make.v.43"},{"index":44,"sense_key":"make%2:31:12::","synset_id":"make.v.44"},{"index":45,"sense_key":"make%2:30:10::","synset_id":"make.v.45"},{"index":46,"sense_key":"make%2:30:14::","synset_id":"make.v.46"},{"index":47,"sense_key":"make%2:30:12::","synset_id":"make.v.47"},{"index":48,"sense_key":"make%2:29:06::","synset_id":"make.v.48"},{"index":49,"sense_key":"make%2:29:01::","synset_id":"make.v.49"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":24,"chosen_sense_key":"make%2:41:13::","call_ids":["senseval3.d000.s114.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":24,"predicted_sense_key":"make%2:41:13::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":3101,"cached_input_tokens":0,"output_tokens":46,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.006662,"input_uncached_usd":0.006202,"input_cached_usd":0.0,"output_usd":0.00046,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.967142458073795}
{"item_id":"senseval3.d000.s114.t001","gold_sense_keys":["mistake%1:04:00::"],"candidates":[{"index":1,"sense_key":"mistake%1:04:00::","synset_id":"mistake.n.01"},{"index":2,"sense_key":"mistake%1:09:00::","synset_id":"mistake.n.02"},{"index":3,"sense_key":"mistake%1:10:00::","synset_id":"error.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"mistake%1:04:00::","call_ids":["senseval3.d000.s114.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"mistake%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":648,"cached_input_tokens":0,"output_tokens":40,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0016959999999999998,"input_uncached_usd":0.0012959999999999998,"input_cached_usd":0.0,"output_usd":0.0004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3066395000787452}
{"item_id":"senseval3.d000.s114.t002","gold_sense_keys":["answer%2:32:00::"],"candidates":[{"index":1,"sense_key":"answer%2:32:00::","synset_id":"answer.v.01"},{"index":2,"sense_key":"answer%2:31:02::","synset_id":"answer.v.02"},{"index":3,"sense_key":"answer%2:31:00::","synset_id":"answer.v.03"},{"index":4,"sense_key":"answer%2:31:03::","synset_id":"answer.v.04"},{"index":5,"sense_key":"answer%2:32:06::","synset_id":"answer.v.05"},{"index":6,"sense_key":"answer%2:42:01::","synset_id":"answer.v.06"},{"index":7,"sense_key":"answer%2:42:02::","synset_id":"suffice.v.01"},{"index":8,"sense_key":"answer%2:42:00::","synset_id":"answer.v.08"},{"index":9,"sense_key":"answer%2:34:00::","synset_id":"answer.v.09"},{"index":10,"sense_key":"answer%2:31:05::","synset_id":"answer.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"answer%2:32:00::","call_ids":["senseval3.d000.s114.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"answer%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":976,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002072,"input_uncached_usd":0.001952,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4921487499959767}
{"item_id":"senseval3.d000.s114.t004","gold_sense_keys":["way%1:04:01::","way%1:07:01::"],"candidates":[{"index":1,"sense_key":"way%1:07:01::","synset_id":"manner.n.01"},{"index":2,"sense_key":"way%1:04:01::","synset_id":"means.n.01"},{"index":3,"sense_key":"way%1:15:00::","synset_id":"direction.n.01"},{"index":4,"sense_key":"way%1:26:00::","synset_id":"way.n.04"},{"index":5,"sense_key":"way%1:04:00::","synset_id":"way.n.05"},{"index":6,"sense_key":"way%1:06:00::","synset_id":"way.n.06"},{"index":7,"sense_key":"way%1:04:02::","synset_id":"way.n.07"},{"index":8,"sense_key":"way%1:23:00::","synset_id":"room.n.02"},{"index":9,"sense_key":"way%1:07:02::","synset_id":"way.n.09"},{"index":10,"sense_key":"way%1:09:00::","synset_id":"way.n.10"},{"index":11,"sense_key":"way%1:09:01::","synset_id":"way.n.11"},{"index":12,"sense_key":"way%1:21:00::","synset_id":"way.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"way%1:07:01::","call_ids":["senseval3.d000.s114.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"way%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1053,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0022259999999999997,"input_uncached_usd":0.002106,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.5986858339747414}
{"item_id":"senseval3.d000.s114.t005","gold_sense_keys":["realize%2:31:00::"],"candidates":[{"index":1,"sense_key":"realize%2:31:01::","synset_id":"recognize.v.02"},{"index":2,"sense_key":"realize%2:31:00::","synset_id":"understand.v.02"},{"index":3,"sense_key":"realize%2:36:00::","synset_id":"realize.v.03"},{"index":4,"sense_key":"realize%2:40:01::","synset_id":"gain.v.08"},{"index":5,"sense_key":"realize%2:40:00::","synset_id":"realize.v.05"},{"index":6,"sense_key":"realize%2:36:07::","synset_id":"realize.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"realize%2:31:00::","call_ids":["senseval3.d000.s114.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"realize%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":832,"cached_input_tokens":0,"output_tokens":255,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004214,"input_uncached_usd":0.001664,"input_cached_usd":0.0,"output_usd":0.00255,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.277158125070855}
{"item_id":"senseval3.d000.s114.t006","gold_sense_keys":["skepticism%1:09:00::"],"candidates":[{"index":1,"sense_key":"skepticism%1:09:00::","synset_id":"incredulity.n.01"},{"index":2,"sense_key":"skepticism%1:09:01::","synset_id":"agnosticism.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"skepticism%1:09:00::","call_ids":["senseval3.d000.s114.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"skepticism%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":552,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001224,"input_uncached_usd":0.001104,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6430145420599729}
{"item_id":"senseval3.d000.s114.t007","gold_sense_keys":["show%2:39:01::"],"candidates":[{"index":1,"sense_key":"show%2:39:02::","synset_id":"show.v.01"},{"index":2,"sense_key":"show%2:31:00::","synset_id":"prove.v.02"},{"index":3,"sense_key":"show%2:32:00::","synset_id":"testify.v.02"},{"index":4,"sense_key":"show%2:39:00::","synset_id":"show.v.04"},{"index":5,"sense_key":"show%2:36:00::","synset_id":"picture.v.02"},{"index":6,"sense_key":"show%2:32:02::","synset_id":"express.v.01"},{"index":7,"sense_key":"show%2:32:01::","synset_id":"indicate.v.02"},{"index":8,"sense_key":"show%2:39:01::","synset_id":"show.v.08"},{"index":9,"sense_key":"show%2:32:04::","synset_id":"read.v.08"},{"index":10,"sense_key":"show%2:32:03::","synset_id":"show.v.10"},{"index":11,"sense_key":"show%2:38:00::","synset_id":"usher.v.01"},{"index":12,"sense_key":"show%2:33:00::","synset_id":"show.v.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"show%2:39:01::","call_ids":["senseval3.d000.s114.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"show%2:39:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1249,"cached_input_tokens":0,"output_tokens":80,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0032979999999999997,"input_uncached_usd":0.002498,"input_cached_usd":0.0,"output_usd":0.0008,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3327479580184445}
{"item_id":"senseval3.d000.s114.t008","gold_sense_keys":["face%1:07:03::","face%1:08:00::"],"candidates":[{"index":1,"sense_key":"face%1:08:00::","synset_id":"face.n.01"},{"index":2,"sense_key":"face%1:07:03::","synset_id":"expression.n.01"},{"index":3,"sense_key":"face%1:07:00::","synset_id":"face.n.03"},{"index":4,"sense_key":"face%1:06:02::","synset_id":"face.n.04"},{"index":5,"sense_key":"face%1:18:00::","synset_id":"face.n.05"},{"index":6,"sense_key":"face%1:15:00::","synset_id":"side.n.04"},{"index":7,"sense_key":"face%1:08:01::","synset_id":"face.n.07"},{"index":8,"sense_key":"face%1:06:01::","synset_id":"face.n.08"},{"index":9,"sense_key":"face%1:10:01::","synset_id":"grimace.n.01"},{"index":10,"sense_key":"face%1:10:00::","synset_id":"font.n.01"},{"index":11,"sense_key":"face%1:07:01::","synset_id":"face.n.11"},{"index":12,"sense_key":"face%1:07:02::","synset_id":"boldness.n.02"},{"index":13,"sense_key":"face%1:06:00::","synset_id":"face.n.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"face%1:08:00::","call_ids":["senseval3.d000.s114.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"face%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1135,"cached_input_tokens":0,"output_tokens":98,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0032500000000000003,"input_uncached_usd":0.00227,"input_cached_usd":0.0,"output_usd":0.0009800000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6343645000597462}
{"item_id":"senseval3.d000.s114.t009","gold_sense_keys":["voice%1:07:00::"],"candidates":[{"index":1,"sense_key":"voice%1:07:00::","synset_id":"voice.n.01"},{"index":2,"sense_key":"voice%1:10:00::","synset_id":"voice.n.02"},{"index":3,"sense_key":"voice%1:11:00::","synset_id":"voice.n.03"},{"index":4,"sense_key":"voice%1:10:01::","synset_id":"articulation.n.03"},{"index":5,"sense_key":"voice%1:04:00::","synset_id":"voice.n.05"},{"index":6,"sense_key":"voice%1:10:03::","synset_id":"voice.n.06"},{"index":7,"sense_key":"voice%1:18:01::","synset_id":"voice.n.07"},{"index":8,"sense_key":"voice%1:18:00::","synset_id":"spokesperson.n.01"},{"index":9,"sense_key":"voice%1:07:01::","synset_id":"voice.n.09"},{"index":10,"sense_key":"voice%1:24:00::","synset_id":"voice.n.10"},{"index":11,"sense_key":"voice%1:10:02::","synset_id":"part.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"voice%1:07:00::","call_ids":["senseval3.d000.s114.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"voice%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1153,"cached_input_tokens":0,"output_tokens":127,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003576,"input_uncached_usd":0.002306,"input_cached_usd":0.0,"output_usd":0.00127,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.843459624913521}
{"item_id":"senseval3.d000.s115.t001","gold_sense_keys":["become%2:30:00::"],"candidates":[{"index":1,"sense_key":"become%2:30:00::","synset_id":"become.v.01"},{"index":2,"sense_key":"become%2:42:01::","synset_id":"become.v.02"},{"index":3,"sense_key":"become%2:42:00::","synset_id":"become.v.03"},{"index":4,"sense_key":"become%2:42:03::","synset_id":"become.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"become%2:30:00::","call_ids":["senseval3.d000.s115.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"become%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":668,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0021060000000000002,"input_uncached_usd":0.001336,"input_cached_usd":0.0,"output_usd":0.0007700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9109252919442952}
{"item_id":"senseval3.d000.s115.t002","gold_sense_keys":["sad%3:00:00::"],"candidates":[{"index":1,"sense_key":"sad%3:00:00::","synset_id":"sad.a.01"},{"index":2,"sense_key":"sad%5:00:00:sorrowful:00","synset_id":"sad.s.01"},{"index":3,"sense_key":"sad%5:00:00:bad:00","synset_id":"deplorable.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"sad%3:00:00::","call_ids":["senseval3.d000.s115.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"sad%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":689,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001498,"input_uncached_usd":0.0013779999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1686969170114025}
{"item_id":"senseval3.d000.s115.t003","gold_sense_keys":["moody%5:00:00:ill-natured:00"],"candidates":[{"index":1,"sense_key":"moody%5:00:00:ill-natured:00","synset_id":"dark.s.04"},{"index":2,"sense_key":"moody%5:00:00:emotional:00","synset_id":"moody.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"moody%5:00:00:ill-natured:00","call_ids":["senseval3.d000.s115.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"moody%5:00:00:ill-natured:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":610,"cached_input_tokens":0,"output_tokens":99,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00221,"input_uncached_usd":0.00122,"input_cached_usd":0.0,"output_usd":0.00099,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4408403750276193}
{"item_id":"senseval3.d000.s116.t000","gold_sense_keys":["want%2:37:00::"],"candidates":[{"index":1,"sense_key":"want%2:37:00::","synset_id":"desire.v.01"},{"index":2,"sense_key":"want%2:34:00::","synset_id":"want.v.02"},{"index":3,"sense_key":"want%2:35:00::","synset_id":"want.v.03"},{"index":4,"sense_key":"want%2:31:01::","synset_id":"want.v.04"},{"index":5,"sense_key":"want%2:42:01::","synset_id":"want.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"want%2:37:00::","call_ids":["senseval3.d000.s116.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"want%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":684,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001488,"input_uncached_usd":0.0013679999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0268062500981614}
{"item_id":"senseval3.d000.s116.t002","gold_sense_keys":["company%1:26:00::"],"candidates":[{"index":1,"sense_key":"company%1:14:01::","synset_id":"company.n.01"},{"index":2,"sense_key":"company%1:14:03::","synset_id":"company.n.02"},{"index":3,"sense_key":"company%1:26:00::","synset_id":"company.n.03"},{"index":4,"sense_key":"company%1:14:02::","synset_id":"company.n.04"},{"index":5,"sense_key":"company%1:18:00::","synset_id":"caller.n.01"},{"index":6,"sense_key":"company%1:14:00::","synset_id":"company.n.06"},{"index":7,"sense_key":"company%1:14:06::","synset_id":"party.n.03"},{"index":8,"sense_key":"company%1:14:04::","synset_id":"ship's_company.n.01"},{"index":9,"sense_key":"company%1:14:05::","synset_id":"company.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"company%1:26:00::","call_ids":["senseval3.d000.s116.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"company%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":913,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002456,"input_uncached_usd":0.001826,"input_cached_usd":0.0,"output_usd":0.00063,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3280616250121966}
{"item_id":"senseval3.d000.s116.t003","gold_sense_keys":["feel%2:31:00::"],"candidates":[{"index":1,"sense_key":"feel%2:37:00::","synset_id":"feel.v.01"},{"index":2,"sense_key":"feel%2:31:00::","synset_id":"find.v.05"},{"index":3,"sense_key":"feel%2:39:00::","synset_id":"feel.v.03"},{"index":4,"sense_key":"feel%2:29:00::","synset_id":"feel.v.04"},{"index":5,"sense_key":"feel%2:31:01::","synset_id":"feel.v.05"},{"index":6,"sense_key":"feel%2:39:01::","synset_id":"feel.v.06"},{"index":7,"sense_key":"feel%2:42:00::","synset_id":"feel.v.07"},{"index":8,"sense_key":"feel%2:39:10::","synset_id":"feel.v.08"},{"index":9,"sense_key":"feel%2:35:00::","synset_id":"feel.v.09"},{"index":10,"sense_key":"feel%2:35:01::","synset_id":"palpate.v.01"},{"index":11,"sense_key":"feel%2:40:00::","synset_id":"feel.v.11"},{"index":12,"sense_key":"feel%2:39:09::","synset_id":"feel.v.12"},{"index":13,"sense_key":"feel%2:35:11::","synset_id":"feel.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"feel%2:31:00::","call_ids":["senseval3.d000.s116.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"feel%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1208,"cached_input_tokens":0,"output_tokens":123,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003646,"input_uncached_usd":0.0024159999999999997,"input_cached_usd":0.0,"output_usd":0.0012300000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1212161669973284}
{"item_id":"senseval3.d000.s116.t004","gold_sense_keys":["buy%2:40:00::"],"candidates":[{"index":1,"sense_key":"buy%2:40:00::","synset_id":"buy.v.01"},{"index":2,"sense_key":"buy%2:40:02::","synset_id":"bribe.v.01"},{"index":3,"sense_key":"buy%2:42:00::","synset_id":"buy.v.03"},{"index":4,"sense_key":"buy%2:40:03::","synset_id":"buy.v.04"},{"index":5,"sense_key":"buy%2:31:09::","synset_id":"buy.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"buy%2:40:00::","call_ids":["senseval3.d000.s116.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"buy%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":712,"cached_input_tokens":0,"output_tokens":109,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0025139999999999997,"input_uncached_usd":0.001424,"input_cached_usd":0.0,"output_usd":0.00109,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7685953750042245}
{"item_id":"senseval3.d000.s116.t005","gold_sense_keys":["drink%1:13:04::"],"candidates":[{"index":1,"sense_key":"drink%1:13:04::","synset_id":"drink.n.01"},{"index":2,"sense_key":"drink%1:04:00::","synset_id":"drink.n.02"},{"index":3,"sense_key":"drink%1:13:00::","synset_id":"beverage.n.01"},{"index":4,"sense_key":"drink%1:17:00::","synset_id":"drink.n.04"},{"index":5,"sense_key":"drink%1:04:01::","synset_id":"swallow.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"drink%1:13:04::","call_ids":["senseval3.d000.s116.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"drink%1:13:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":733,"cached_input_tokens":0,"output_tokens":48,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0019459999999999998,"input_uncached_usd":0.0014659999999999999,"input_cached_usd":0.0,"output_usd":0.00048000000000000007,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3330226669786498}
{"item_id":"senseval3.d000.s116.t006","gold_sense_keys":["prevent%2:41:01::"],"candidates":[{"index":1,"sense_key":"prevent%2:41:01::","synset_id":"prevent.v.01"},{"index":2,"sense_key":"prevent%2:41:00::","synset_id":"prevent.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"prevent%2:41:01::","call_ids":["senseval3.d000.s116.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"prevent%2:41:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":598,"cached_input_tokens":0,"output_tokens":103,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002226,"input_uncached_usd":0.001196,"input_cached_usd":0.0,"output_usd":0.00103,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.185308499960229}
{"item_id":"senseval3.d000.s116.t007","gold_sense_keys":["possible%3:00:04::"],"candidates":[{"index":1,"sense_key":"possible%3:00:00::","synset_id":"possible.a.01"},{"index":2,"sense_key":"possible%3:00:04::","synset_id":"potential.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"possible%3:00:04::","call_ids":["senseval3.d000.s116.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"possible%3:00:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":526,"cached_input_tokens":0,"output_tokens":69,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001742,"input_uncached_usd":0.001052,"input_cached_usd":0.0,"output_usd":0.0006900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5279974580043927}
{"item_id":"senseval3.d000.s116.t008","gold_sense_keys":["trouble%1:04:02::","trouble%1:09:00::","trouble%1:11:00::","trouble%1:11:01::"],"candidates":[{"index":1,"sense_key":"trouble%1:09:00::","synset_id":"trouble.n.01"},{"index":2,"sense_key":"trouble%1:11:01::","synset_id":"fuss.n.02"},{"index":3,"sense_key":"trouble%1:11:00::","synset_id":"trouble.n.03"},{"index":4,"sense_key":"trouble%1:04:02::","synset_id":"trouble.n.04"},{"index":5,"sense_key":"trouble%1:12:03::","synset_id":"worry.n.02"},{"index":6,"sense_key":"trouble%1:26:02::","synset_id":"trouble.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"trouble%1:11:01::","call_ids":["senseval3.d000.s116.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"trouble%1:11:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":778,"cached_input_tokens":0,"output_tokens":85,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002406,"input_uncached_usd":0.0015559999999999999,"input_cached_usd":0.0,"output_usd":0.0008500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.088896124972962}
{"item_id":"senseval3.d000.s117.t000","gold_sense_keys":["trouble%1:04:02::","trouble%1:09:00::","trouble%1:11:00::","trouble%1:11:01::"],"candidates":[{"index":1,"sense_key":"trouble%1:09:00::","synset_id":"trouble.n.01"},{"index":2,"sense_key":"trouble%1:11:01::","synset_id":"fuss.n.02"},{"index":3,"sense_key":"trouble%1:11:00::","synset_id":"trouble.n.03"},{"index":4,"sense_key":"trouble%1:04:02::","synset_id":"trouble.n.04"},{"index":5,"sense_key":"trouble%1:12:03::","synset_id":"worry.n.02"},{"index":6,"sense_key":"trouble%1:26:02::","synset_id":"trouble.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"trouble%1:11:01::","call_ids":["senseval3.d000.s117.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"trouble%1:11:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":782,"cached_input_tokens":0,"output_tokens":94,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002504,"input_uncached_usd":0.0015639999999999999,"input_cached_usd":0.0,"output_usd":0.0009400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.649403459043242}
{"item_id":"senseval3.d000.s118.t000","gold_sense_keys":["guy%1:18:00::"],"candidates":[{"index":1,"sense_key":"guy%1:18:00::","synset_id":"guy.n.01"},{"index":2,"sense_key":"guy%1:06:01::","synset_id":"guy.n.02"},{"index":3,"sense_key":"guy%1:06:00::","synset_id":"guy.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"guy%1:18:00::","call_ids":["senseval3.d000.s118.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"guy%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":619,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001358,"input_uncached_usd":0.001238,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.022050666040741}
{"item_id":"senseval3.d000.s118.t002","gold_sense_keys":["drink%1:13:04::"],"candidates":[{"index":1,"sense_key":"drink%1:13:04::","synset_id":"drink.n.01"},{"index":2,"sense_key":"drink%1:04:00::","synset_id":"drink.n.02"},{"index":3,"sense_key":"drink%1:13:00::","synset_id":"beverage.n.01"},{"index":4,"sense_key":"drink%1:17:00::","synset_id":"drink.n.04"},{"index":5,"sense_key":"drink%1:04:01::","synset_id":"swallow.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"drink%1:13:04::","call_ids":["senseval3.d000.s118.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"drink%1:13:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":742,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002094,"input_uncached_usd":0.0014839999999999999,"input_cached_usd":0.0,"output_usd":0.0006100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.891211959067732}
{"item_id":"senseval3.d000.s118.t003","gold_sense_keys":["upset%2:37:00::","upset%2:37:01::"],"candidates":[{"index":1,"sense_key":"upset%2:30:00::","synset_id":"upset.v.01"},{"index":2,"sense_key":"upset%2:37:01::","synset_id":"upset.v.02"},{"index":3,"sense_key":"upset%2:37:00::","synset_id":"disturb.v.01"},{"index":4,"sense_key":"upset%2:38:00::","synset_id":"overturn.v.02"},{"index":5,"sense_key":"upset%2:35:00::","synset_id":"swage.v.01"},{"index":6,"sense_key":"upset%2:33:00::","synset_id":"upset.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"upset%2:37:00::","call_ids":["senseval3.d000.s118.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"upset%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":774,"cached_input_tokens":0,"output_tokens":189,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003438,"input_uncached_usd":0.001548,"input_cached_usd":0.0,"output_usd":0.0018900000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0778107499936596}
{"item_id":"senseval3.d000.s118.t005","gold_sense_keys":["appreciation%1:10:00::"],"candidates":[{"index":1,"sense_key":"appreciation%1:09:00::","synset_id":"appreciation.n.01"},{"index":2,"sense_key":"appreciation%1:09:01::","synset_id":"taste.n.03"},{"index":3,"sense_key":"appreciation%1:10:00::","synset_id":"appreciation.n.03"},{"index":4,"sense_key":"appreciation%1:04:00::","synset_id":"admiration.n.03"},{"index":5,"sense_key":"appreciation%1:04:01::","synset_id":"appreciation.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"appreciation%1:10:00::","call_ids":["senseval3.d000.s118.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"appreciation%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":763,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002246,"input_uncached_usd":0.001526,"input_cached_usd":0.0,"output_usd":0.00072,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0188695000251755}
{"item_id":"senseval3.d000.s119.t000","gold_sense_keys":["break%2:30:03::","break%2:42:05::"],"candidates":[{"index":1,"sense_key":"break%2:30:03::","synset_id":"interrupt.v.04"},{"index":2,"sense_key":"break%2:30:00::","synset_id":"break.v.02"},{"index":3,"sense_key":"break%2:30:06::","synset_id":"break.v.03"},{"index":4,"sense_key":"break%2:35:01::","synset_id":"break.v.04"},{"index":5,"sense_key":"break%2:30:10::","synset_id":"break.v.05"},{"index":6,"sense_key":"break%2:41:00::","synset_id":"transgress.v.01"},{"index":7,"sense_key":"break%2:38:02::","synset_id":"break.v.07"},{"index":8,"sense_key":"break%2:38:08::","synset_id":"break.v.08"},{"index":9,"sense_key":"break%2:37:00::","synset_id":"break.v.09"},{"index":10,"sense_key":"break%2:30:02::","synset_id":"break.v.10"},{"index":11,"sense_key":"break%2:41:11::","synset_id":"break_in.v.01"},{"index":12,"sense_key":"break%2:30:11::","synset_id":"break_in.v.06"},{"index":13,"sense_key":"break%2:42:00::","synset_id":"violate.v.01"},{"index":14,"sense_key":"break%2:33:00::","synset_id":"better.v.01"},{"index":15,"sense_key":"break%2:32:00::","synset_id":"unwrap.v.02"},{"index":16,"sense_key":"break%2:30:04::","synset_id":"break.v.16"},{"index":17,"sense_key":"break%2:30:05::","synset_id":"fail.v.04"},{"index":18,"sense_key":"break%2:41:01::","synset_id":"break.v.18"},{"index":19,"sense_key":"break%2:38:10::","synset_id":"break.v.19"},{"index":20,"sense_key":"break%2:38:01::","synset_id":"break.v.20"},{"index":21,"sense_key":"break%2:30:09::","synset_id":"dampen.v.07"},{"index":22,"sense_key":"break%2:30:12::","synset_id":"break.v.22"},{"index":23,"sense_key":"break%2:42:15::","synset_id":"break.v.23"},{"index":24,"sense_key":"break%2:42:05::","synset_id":"break.v.24"},{"index":25,"sense_key":"break%2:42:04::","synset_id":"break.v.25"},{"index":26,"sense_key":"break%2:42:03::","synset_id":"break.v.26"},{"index":27,"sense_key":"break%2:42:14::","synset_id":"break.v.27"},{"index":28,"sense_key":"break%2:41:15::","synset_id":"break.v.28"},{"index":29,"sense_key":"break%2:41:09::","synset_id":"break.v.29"},{"index":30,"sense_key":"break%2:41:10::","synset_id":"break.v.30"},{"index":31,"sense_key":"break%2:41:13::","synset_id":"separate.v.08"},{"index":32,"sense_key":"break%2:41:06::","synset_id":"demote.v.01"},{"index":33,"sense_key":"break%2:40:07::","synset_id":"bankrupt.v.01"},{"index":34,"sense_key":"break%2:38:15::","synset_id":"break.v.34"},{"index":35,"sense_key":"break%2:38:09::","synset_id":"break.v.35"},{"index":36,"sense_key":"break%2:38:11::","synset_id":"collapse.v.01"},{"index":37,"sense_key":"break%2:38:00::","synset_id":"break_dance.v.01"},{"index":38,"sense_key":"break%2:35:11::","synset_id":"break.v.38"},{"index":39,"sense_key":"break%2:35:10::","synset_id":"break.v.39"},{"index":40,"sense_key":"break%2:35:14::","synset_id":"break.v.40"},{"index":41,"sense_key":"break%2:35:04::","synset_id":"break.v.41"},{"index":42,"sense_key":"break%2:35:00::","synset_id":"break.v.42"},{"index":43,"sense_key":"break%2:35:05::","synset_id":"break.v.43"},{"index":44,"sense_key":"break%2:35:13::","synset_id":"break.v.44"},{"index":45,"sense_key":"break%2:35:09::","synset_id":"break.v.45"},{"index":46,"sense_key":"break%2:32:05::","synset_id":"break.v.46"},{"index":47,"sense_key":"break%2:32:04::","synset_id":"pause.v.02"},{"index":48,"sense_key":"break%2:32:13::","synset_id":"break.v.48"},{"index":49,"sense_key":"break%2:32:07::","synset_id":"break.v.49"},{"index":50,"sense_key":"break%2:31:06::","synset_id":"break.v.50"},{"index":51,"sense_key":"break%2:31:05::","synset_id":"break.v.51"},{"index":52,"sense_key":"break%2:30:14::","synset_id":"break.v.52"},{"index":53,"sense_key":"break%2:30:01::","synset_id":"break.v.53"},{"index":54,"sense_key":"break%2:30:15::","synset_id":"crack.v.01"},{"index":55,"sense_key":"break%2:30:07::","synset_id":"break.v.55"},{"index":56,"sense_key":"break%2:30:08::","synset_id":"break.v.56"},{"index":57,"sense_key":"break%2:29:04::","synset_id":"fracture.v.06"},{"index":58,"sense_key":"break%2:29:11::","synset_id":"break.v.58"},{"index":59,"sense_key":"break%2:29:10::","synset_id":"break.v.59"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":24,"chosen_sense_key":"break%2:42:05::","call_ids":["senseval3.d000.s119.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":24,"predicted_sense_key":"break%2:42:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":3786,"cached_input_tokens":0,"output_tokens":210,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.009672,"input_uncached_usd":0.007572,"input_cached_usd":0.0,"output_usd":0.0021000000000000003,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.3220759170362726}
{"item_id":"senseval3.d000.s119.t001","gold_sense_keys":["uncomfortable%3:00:01::"],"candidates":[{"index":1,"sense_key":"uncomfortable%3:00:01::","synset_id":"uncomfortable.a.01"},{"index":2,"sense_key":"uncomfortable%3:00:00::","synset_id":"uncomfortable.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"uncomfortable%3:00:01::","call_ids":["senseval3.d000.s119.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"uncomfortable%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":610,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00175,"input_uncached_usd":0.00122,"input_cached_usd":0.0,"output_usd":0.0005300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2729087920160964}
{"item_id":"senseval3.d000.s119.t002","gold_sense_keys":["silence%1:26:00::"],"candidates":[{"index":1,"sense_key":"silence%1:26:00::","synset_id":"silence.n.01"},{"index":2,"sense_key":"silence%1:07:00::","synset_id":"silence.n.02"},{"index":3,"sense_key":"silence%1:07:01::","synset_id":"muteness.n.02"},{"index":4,"sense_key":"silence%1:07:02::","synset_id":"secrecy.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"silence%1:26:00::","call_ids":["senseval3.d000.s119.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"silence%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":684,"cached_input_tokens":0,"output_tokens":138,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002748,"input_uncached_usd":0.0013679999999999999,"input_cached_usd":0.0,"output_usd":0.0013800000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.97355533298105}
{"item_id":"senseval3.d000.s119.t003","gold_sense_keys":["begin%2:30:00::"],"candidates":[{"index":1,"sense_key":"begin%2:30:00::","synset_id":"get_down.v.07"},{"index":2,"sense_key":"begin%2:42:00::","synset_id":"begin.v.02"},{"index":3,"sense_key":"begin%2:30:01::","synset_id":"begin.v.03"},{"index":4,"sense_key":"begin%2:32:03::","synset_id":"begin.v.04"},{"index":5,"sense_key":"begin%2:42:02::","synset_id":"begin.v.05"},{"index":6,"sense_key":"begin%2:42:04::","synset_id":"begin.v.06"},{"index":7,"sense_key":"begin%2:42:03::","synset_id":"begin.v.07"},{"index":8,"sense_key":"begin%2:41:04::","synset_id":"begin.v.08"},{"index":9,"sense_key":"begin%2:41:00::","synset_id":"begin.v.09"},{"index":10,"sense_key":"begin%2:32:04::","synset_id":"begin.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"begin%2:30:00::","call_ids":["senseval3.d000.s119.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"begin%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1174,"cached_input_tokens":0,"output_tokens":88,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003228,"input_uncached_usd":0.002348,"input_cached_usd":0.0,"output_usd":0.00088,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6352808329975232}
{"item_id":"senseval3.d000.s120.t000","gold_sense_keys":["drink%1:13:04::"],"candidates":[{"index":1,"sense_key":"drink%1:13:04::","synset_id":"drink.n.01"},{"index":2,"sense_key":"drink%1:04:00::","synset_id":"drink.n.02"},{"index":3,"sense_key":"drink%1:13:00::","synset_id":"beverage.n.01"},{"index":4,"sense_key":"drink%1:17:00::","synset_id":"drink.n.04"},{"index":5,"sense_key":"drink%1:04:01::","synset_id":"swallow.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"drink%1:13:04::","call_ids":["senseval3.d000.s120.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"drink%1:13:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":759,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002088,"input_uncached_usd":0.001518,"input_cached_usd":0.0,"output_usd":0.0005700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3193594999611378}
{"item_id":"senseval3.d000.s120.t001","gold_sense_keys":["complain%2:32:00::"],"candidates":[{"index":1,"sense_key":"complain%2:32:00::","synset_id":"complain.v.01"},{"index":2,"sense_key":"complain%2:32:01::","synset_id":"complain.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"complain%2:32:00::","call_ids":["senseval3.d000.s120.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"complain%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":597,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001314,"input_uncached_usd":0.001194,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0595421670004725}
{"item_id":"senseval3.d000.s120.t003","gold_sense_keys":["subject%1:09:01::","subject%1:10:00::"],"candidates":[{"index":1,"sense_key":"subject%1:10:00::","synset_id":"subject.n.01"},{"index":2,"sense_key":"subject%1:06:00::","synset_id":"subject.n.02"},{"index":3,"sense_key":"subject%1:09:00::","synset_id":"discipline.n.01"},{"index":4,"sense_key":"subject%1:09:01::","synset_id":"topic.n.02"},{"index":5,"sense_key":"subject%1:10:01::","synset_id":"subject.n.05"},{"index":6,"sense_key":"subject%1:18:01::","synset_id":"subject.n.06"},{"index":7,"sense_key":"subject%1:18:00::","synset_id":"national.n.01"},{"index":8,"sense_key":"subject%1:10:02::","synset_id":"subject.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"subject%1:10:00::","call_ids":["senseval3.d000.s120.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"subject%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1012,"cached_input_tokens":0,"output_tokens":101,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003034,"input_uncached_usd":0.0020239999999999998,"input_cached_usd":0.0,"output_usd":0.00101,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4827955409418792}
{"item_id":"senseval3.d000.s120.t004","gold_sense_keys":["minute%1:28:00::"],"candidates":[{"index":1,"sense_key":"minute%1:28:00::","synset_id":"minute.n.01"},{"index":2,"sense_key":"minute%1:28:01::","synset_id":"moment.n.02"},{"index":3,"sense_key":"minute%1:28:02::","synset_id":"moment.n.01"},{"index":4,"sense_key":"minute%1:23:00::","synset_id":"minute.n.04"},{"index":5,"sense_key":"minute%1:10:00::","synset_id":"minute.n.05"},{"index":6,"sense_key":"minute%1:07:00::","synset_id":"hour.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"minute%1:28:00::","call_ids":["senseval3.d000.s120.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"minute%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":796,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001712,"input_uncached_usd":0.0015919999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.200827541993931}
{"item_id":"senseval3.d000.s120.t005","gold_sense_keys":["notice%2:39:00::"],"candidates":[{"index":1,"sense_key":"notice%2:39:04::","synset_id":"detect.v.01"},{"index":2,"sense_key":"notice%2:39:00::","synset_id":"notice.v.02"},{"index":3,"sense_key":"notice%2:32:00::","synset_id":"comment.v.01"},{"index":4,"sense_key":"notice%2:32:01::","synset_id":"notice.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"notice%2:39:00::","call_ids":["senseval3.d000.s120.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"notice%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":736,"cached_input_tokens":0,"output_tokens":126,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002732,"input_uncached_usd":0.001472,"input_cached_usd":0.0,"output_usd":0.00126,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2026424169307575}
{"item_id":"senseval3.d000.s120.t007","gold_sense_keys":["interest%1:09:00::"],"candidates":[{"index":1,"sense_key":"interest%1:09:00::","synset_id":"interest.n.01"},{"index":2,"sense_key":"interest%1:07:01::","synset_id":"sake.n.01"},{"index":3,"sense_key":"interest%1:07:02::","synset_id":"interest.n.03"},{"index":4,"sense_key":"interest%1:21:00::","synset_id":"interest.n.04"},{"index":5,"sense_key":"interest%1:21:03::","synset_id":"interest.n.05"},{"index":6,"sense_key":"interest%1:14:00::","synset_id":"interest.n.06"},{"index":7,"sense_key":"interest%1:04:01::","synset_id":"pastime.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"interest%1:09:00::","call_ids":["senseval3.d000.s120.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"interest%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":935,"cached_input_tokens":0,"output_tokens":69,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0025599999999999998,"input_uncached_usd":0.00187,"input_cached_usd":0.0,"output_usd":0.0006900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5407292080344632}
{"item_id":"senseval3.d000.s120.t009","gold_sense_keys":["show%2:39:01::"],"candidates":[{"index":1,"sense_key":"show%2:39:02::","synset_id":"show.v.01"},{"index":2,"sense_key":"show%2:31:00::","synset_id":"prove.v.02"},{"index":3,"sense_key":"show%2:32:00::","synset_id":"testify.v.02"},{"index":4,"sense_key":"show%2:39:00::","synset_id":"show.v.04"},{"index":5,"sense_key":"show%2:36:00::","synset_id":"picture.v.02"},{"index":6,"sense_key":"show%2:32:02::","synset_id":"express.v.01"},{"index":7,"sense_key":"show%2:32:01::","synset_id":"indicate.v.02"},{"index":8,"sense_key":"show%2:39:01::","synset_id":"show.v.08"},{"index":9,"sense_key":"show%2:32:04::","synset_id":"read.v.08"},{"index":10,"sense_key":"show%2:32:03::","synset_id":"show.v.10"},{"index":11,"sense_key":"show%2:38:00::","synset_id":"usher.v.01"},{"index":12,"sense_key":"show%2:33:00::","synset_id":"show.v.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"show%2:39:01::","call_ids":["senseval3.d000.s120.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"show%2:39:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1256,"cached_input_tokens":0,"output_tokens":82,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003332,"input_uncached_usd":0.002512,"input_cached_usd":0.0,"output_usd":0.0008200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3932581660337746}
{"item_id":"senseval3.d000.s120.t010","gold_sense_keys":["alert%3:00:00::","alert%5:00:00:aware:00"],"candidates":[{"index":1,"sense_key":"alert%3:00:00::","synset_id":"alert.a.01"},{"index":2,"sense_key":"alert%5:00:00:energetic:00","synset_id":"alert.s.01"},{"index":3,"sense_key":"alert%5:00:00:aware:00","synset_id":"alert.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"alert%5:00:00:aware:00","call_ids":["senseval3.d000.s120.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"alert%5:00:00:aware:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":661,"cached_input_tokens":0,"output_tokens":114,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002462,"input_uncached_usd":0.001322,"input_cached_usd":0.0,"output_usd":0.0011400000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.584816792048514}
{"item_id":"senseval3.d000.s120.t011","gold_sense_keys":["face%1:07:03::"],"candidates":[{"index":1,"sense_key":"face%1:08:00::","synset_id":"face.n.01"},{"index":2,"sense_key":"face%1:07:03::","synset_id":"expression.n.01"},{"index":3,"sense_key":"face%1:07:00::","synset_id":"face.n.03"},{"index":4,"sense_key":"face%1:06:02::","synset_id":"face.n.04"},{"index":5,"sense_key":"face%1:18:00::","synset_id":"face.n.05"},{"index":6,"sense_key":"face%1:15:00::","synset_id":"side.n.04"},{"index":7,"sense_key":"face%1:08:01::","synset_id":"face.n.07"},{"index":8,"sense_key":"face%1:06:01::","synset_id":"face.n.08"},{"index":9,"sense_key":"face%1:10:01::","synset_id":"grimace.n.01"},{"index":10,"sense_key":"face%1:10:00::","synset_id":"font.n.01"},{"index":11,"sense_key":"face%1:07:01::","synset_id":"face.n.11"},{"index":12,"sense_key":"face%1:07:02::","synset_id":"boldness.n.02"},{"index":13,"sense_key":"face%1:06:00::","synset_id":"face.n.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"face%1:08:00::","call_ids":["senseval3.d000.s120.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"face%1:08:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1142,"cached_input_tokens":0,"output_tokens":95,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003234,"input_uncached_usd":0.002284,"input_cached_usd":0.0,"output_usd":0.0009500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2798579999944195}
{"item_id":"senseval3.d000.s120.t012","gold_sense_keys":["bright%5:00:00:happy:00"],"candidates":[{"index":1,"sense_key":"bright%3:00:00::","synset_id":"bright.a.01"},{"index":2,"sense_key":"bright%5:00:00:colorful:00","synset_id":"bright.s.01"},{"index":3,"sense_key":"bright%5:00:00:intelligent:00","synset_id":"bright.s.02"},{"index":4,"sense_key":"bright%5:00:00:light:06","synset_id":"bright.s.03"},{"index":5,"sense_key":"bright%5:00:00:polished:00","synset_id":"bright.s.04"},{"index":6,"sense_key":"bright%5:00:00:glorious:00","synset_id":"bright.s.05"},{"index":7,"sense_key":"bright%3:00:02::","synset_id":"undimmed.a.01"},{"index":8,"sense_key":"bright%5:00:00:reverberant:00","synset_id":"bright.s.06"},{"index":9,"sense_key":"bright%5:00:00:happy:00","synset_id":"bright.s.07"},{"index":10,"sense_key":"bright%5:00:00:auspicious:00","synset_id":"bright.s.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bright%3:00:00::","call_ids":["senseval3.d000.s120.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bright%3:00:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1142,"cached_input_tokens":0,"output_tokens":278,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.005064000000000001,"input_uncached_usd":0.002284,"input_cached_usd":0.0,"output_usd":0.0027800000000000004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.846685291035101}
{"item_id":"senseval3.d000.s120.t013","gold_sense_keys":["eye%1:08:00::"],"candidates":[{"index":1,"sense_key":"eye%1:08:00::","synset_id":"eye.n.01"},{"index":2,"sense_key":"eye%1:09:00::","synset_id":"eye.n.02"},{"index":3,"sense_key":"eye%1:09:01::","synset_id":"eye.n.03"},{"index":4,"sense_key":"eye%1:15:00::","synset_id":"center.n.01"},{"index":5,"sense_key":"eye%1:06:00::","synset_id":"eye.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"eye%1:08:00::","call_ids":["senseval3.d000.s120.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"eye%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":716,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001552,"input_uncached_usd":0.001432,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5682543329894543}
{"item_id":"senseval3.d000.s121.t000","gold_sense_keys":["pause%2:42:00::"],"candidates":[{"index":1,"sense_key":"pause%2:42:00::","synset_id":"hesitate.v.02"},{"index":2,"sense_key":"pause%2:32:01::","synset_id":"pause.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"pause%2:42:00::","call_ids":["senseval3.d000.s121.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"pause%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":551,"cached_input_tokens":0,"output_tokens":116,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002262,"input_uncached_usd":0.001102,"input_cached_usd":0.0,"output_usd":0.00116,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6536554999183863}
{"item_id":"senseval3.d000.s121.t001","gold_sense_keys":["moisten%2:30:00::"],"candidates":[{"index":1,"sense_key":"moisten%2:30:00::","synset_id":"moisten.v.01"},{"index":2,"sense_key":"moisten%2:35:00::","synset_id":"drizzle.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"moisten%2:30:00::","call_ids":["senseval3.d000.s121.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"moisten%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":545,"cached_input_tokens":0,"output_tokens":91,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002,"input_uncached_usd":0.00109,"input_cached_usd":0.0,"output_usd":0.0009100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6412949170917273}
{"item_id":"senseval3.d000.s121.t002","gold_sense_keys":["throat%1:08:00::"],"candidates":[{"index":1,"sense_key":"throat%1:08:00::","synset_id":"throat.n.01"},{"index":2,"sense_key":"throat%1:06:01::","synset_id":"throat.n.02"},{"index":3,"sense_key":"throat%1:06:00::","synset_id":"throat.n.03"},{"index":4,"sense_key":"throat%1:05:00::","synset_id":"throat.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"throat%1:08:00::","call_ids":["senseval3.d000.s121.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"throat%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":634,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0013880000000000001,"input_uncached_usd":0.001268,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.156584583921358}
{"item_id":"senseval3.d000.s121.t003","gold_sense_keys":["stranger%1:18:01::"],"candidates":[{"index":1,"sense_key":"stranger%1:18:00::","synset_id":"stranger.n.01"},{"index":2,"sense_key":"stranger%1:18:01::","synset_id":"stranger.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"stranger%1:18:01::","call_ids":["senseval3.d000.s121.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"stranger%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":523,"cached_input_tokens":0,"output_tokens":95,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0019960000000000004,"input_uncached_usd":0.001046,"input_cached_usd":0.0,"output_usd":0.0009500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0986859999829903}
{"item_id":"senseval3.d000.s121.t004","gold_sense_keys":["break_in%2:32:00::"],"candidates":[{"index":1,"sense_key":"break_in%2:41:00::","synset_id":"break_in.v.01"},{"index":2,"sense_key":"break_in%2:32:00::","synset_id":"chime_in.v.01"},{"index":3,"sense_key":"break_in%2:30:03::","synset_id":"break_in.v.03"},{"index":4,"sense_key":"break_in%2:35:00::","synset_id":"break_in.v.04"},{"index":5,"sense_key":"break_in%2:30:01::","synset_id":"break_in.v.05"},{"index":6,"sense_key":"break_in%2:30:00::","synset_id":"break_in.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"break_in%2:32:00::","call_ids":["senseval3.d000.s121.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"break_in%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":810,"cached_input_tokens":0,"output_tokens":86,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00248,"input_uncached_usd":0.00162,"input_cached_usd":0.0,"output_usd":0.0008600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0664994579274207}
{"item_id":"senseval3.d000.s122.t000","gold_sense_keys":["pay%2:40:14::"],"candidates":[{"index":1,"sense_key":"pay%2:40:00::","synset_id":"pay.v.01"},{"index":2,"sense_key":"pay%2:32:00::","synset_id":"give.v.05"},{"index":3,"sense_key":"pay%2:40:04::","synset_id":"pay_up.v.01"},{"index":4,"sense_key":"pay%2:40:01::","synset_id":"yield.v.10"},{"index":5,"sense_key":"pay%2:40:02::","synset_id":"pay.v.05"},{"index":6,"sense_key":"pay%2:31:00::","synset_id":"give.v.10"},{"index":7,"sense_key":"pay%2:42:00::","synset_id":"pay.v.07"},{"index":8,"sense_key":"pay%2:41:00::","synset_id":"pay.v.08"},{"index":9,"sense_key":"pay%2:31:03::","synset_id":"pay.v.09"},{"index":10,"sense_key":"pay%2:40:13::","synset_id":"pay.v.10"},{"index":11,"sense_key":"pay%2:40:14::","synset_id":"pay.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"pay%2:40:14::","call_ids":["senseval3.d000.s122.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"pay%2:40:14::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1021,"cached_input_tokens":0,"output_tokens":117,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003212,"input_uncached_usd":0.002042,"input_cached_usd":0.0,"output_usd":0.00117,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.346599292010069}
{"item_id":"senseval3.d000.s122.t001","gold_sense_keys":["bill%1:10:01::"],"candidates":[{"index":1,"sense_key":"bill%1:10:04::","synset_id":"bill.n.01"},{"index":2,"sense_key":"bill%1:10:01::","synset_id":"bill.n.02"},{"index":3,"sense_key":"bill%1:21:00::","synset_id":"bill.n.03"},{"index":4,"sense_key":"bill%1:04:00::","synset_id":"bill.n.04"},{"index":5,"sense_key":"bill%1:10:02::","synset_id":"circular.n.01"},{"index":6,"sense_key":"bill%1:10:00::","synset_id":"poster.n.01"},{"index":7,"sense_key":"bill%1:10:06::","synset_id":"bill.n.07"},{"index":8,"sense_key":"bill%1:06:01::","synset_id":"bill.n.08"},{"index":9,"sense_key":"bill%1:06:00::","synset_id":"bill.n.09"},{"index":10,"sense_key":"bill%1:05:00::","synset_id":"beak.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bill%1:10:01::","call_ids":["senseval3.d000.s122.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bill%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1091,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002302,"input_uncached_usd":0.002182,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6371045000851154}
{"item_id":"senseval3.d000.s123.t001","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":2,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":3,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"other%3:00:00::","call_ids":["senseval3.d000.s123.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":652,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0014240000000000001,"input_uncached_usd":0.001304,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0840575840556994}
{"item_id":"senseval3.d000.s123.t002","gold_sense_keys":["man%1:18:00::"],"candidates":[{"index":1,"sense_key":"man%1:18:00::","synset_id":"man.n.01"},{"index":2,"sense_key":"man%1:18:03::","synset_id":"serviceman.n.01"},{"index":3,"sense_key":"man%1:18:04::","synset_id":"man.n.03"},{"index":4,"sense_key":"man%1:05:01::","synset_id":"homo.n.02"},{"index":5,"sense_key":"man%1:18:08::","synset_id":"man.n.05"},{"index":6,"sense_key":"man%1:18:06::","synset_id":"man.n.06"},{"index":7,"sense_key":"man%1:18:05::","synset_id":"valet.n.01"},{"index":8,"sense_key":"man%1:18:07::","synset_id":"man.n.08"},{"index":9,"sense_key":"man%1:15:00::","synset_id":"man.n.09"},{"index":10,"sense_key":"man%1:06:00::","synset_id":"man.n.10"},{"index":11,"sense_key":"man%1:05:00::","synset_id":"world.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"man%1:18:00::","call_ids":["senseval3.d000.s123.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"man%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1158,"cached_input_tokens":0,"output_tokens":106,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003376,"input_uncached_usd":0.002316,"input_cached_usd":0.0,"output_usd":0.0010600000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2674849170725793}
{"item_id":"senseval3.d000.s123.t004","gold_sense_keys":["say%2:32:01::","say%2:32:13::"],"candidates":[{"index":1,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":6,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":9,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":10,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":11,"sense_key":"say%2:32:09::","synset_id":"say.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"say%2:32:01::","call_ids":["senseval3.d000.s123.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"say%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1076,"cached_input_tokens":0,"output_tokens":112,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0032719999999999997,"input_uncached_usd":0.002152,"input_cached_usd":0.0,"output_usd":0.0011200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.814747624914162}
{"item_id":"senseval3.d000.s123.t005","gold_sense_keys":["divorce%2:41:00::"],"candidates":[{"index":1,"sense_key":"divorce%2:41:01::","synset_id":"disassociate.v.01"},{"index":2,"sense_key":"divorce%2:41:00::","synset_id":"divorce.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"divorce%2:41:00::","call_ids":["senseval3.d000.s123.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"divorce%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":563,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0012460000000000001,"input_uncached_usd":0.001126,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1718014579964802}
{"item_id":"senseval3.d000.s125.t000","gold_sense_keys":["bitch%1:18:00::"],"candidates":[{"index":1,"sense_key":"bitch%1:26:00::","synset_id":"bitch.n.01"},{"index":2,"sense_key":"bitch%1:18:00::","synset_id":"cunt.n.01"},{"index":3,"sense_key":"bitch%1:10:00::","synset_id":"gripe.n.01"},{"index":4,"sense_key":"bitch%1:05:00::","synset_id":"bitch.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bitch%1:18:00::","call_ids":["senseval3.d000.s125.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bitch%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":621,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0013620000000000001,"input_uncached_usd":0.001242,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4605270409956574}
{"item_id":"senseval3.d000.s125.t001","gold_sense_keys":["love%2:37:02::"],"candidates":[{"index":1,"sense_key":"love%2:37:00::","synset_id":"love.v.01"},{"index":2,"sense_key":"love%2:37:02::","synset_id":"love.v.02"},{"index":3,"sense_key":"love%2:37:01::","synset_id":"love.v.03"},{"index":4,"sense_key":"love%2:35:00::","synset_id":"sleep_together.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"love%2:37:02::","call_ids":["senseval3.d000.s125.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"love%2:37:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":628,"cached_input_tokens":0,"output_tokens":136,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0026160000000000003,"input_uncached_usd":0.001256,"input_cached_usd":0.0,"output_usd":0.00136,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2958418340422213}
{"item_id":"senseval3.d000.s126.t000","gold_sense_keys":["get%2:40:00::"],"candidates":[{"index":1,"sense_key":"get%2:40:00::","synset_id":"get.v.01"},{"index":2,"sense_key":"get%2:30:00::","synset_id":"become.v.01"},{"index":3,"sense_key":"get%2:30:02::","synset_id":"get.v.03"},{"index":4,"sense_key":"get%2:30:03::","synset_id":"receive.v.02"},{"index":5,"sense_key":"get%2:38:00::","synset_id":"arrive.v.01"},{"index":6,"sense_key":"get%2:35:03::","synset_id":"bring.v.04"},{"index":7,"sense_key":"get%2:39:00::","synset_id":"experience.v.03"},{"index":8,"sense_key":"get%2:33:00::","synset_id":"pay_back.v.02"},{"index":9,"sense_key":"get%2:40:01::","synset_id":"have.v.17"},{"index":10,"sense_key":"get%2:32:00::","synset_id":"induce.v.02"},{"index":11,"sense_key":"get%2:35:09::","synset_id":"get.v.11"},{"index":12,"sense_key":"get%2:29:01::","synset_id":"grow.v.08"},{"index":13,"sense_key":"get%2:29:00::","synset_id":"contract.v.04"},{"index":14,"sense_key":"get%2:32:03::","synset_id":"get.v.14"},{"index":15,"sense_key":"get%2:30:01::","synset_id":"make.v.02"},{"index":16,"sense_key":"get%2:32:08::","synset_id":"drive.v.11"},{"index":17,"sense_key":"get%2:31:09::","synset_id":"catch.v.18"},{"index":18,"sense_key":"get%2:35:00::","synset_id":"catch.v.07"},{"index":19,"sense_key":"get%2:35:10::","synset_id":"get.v.19"},{"index":20,"sense_key":"get%2:31:03::","synset_id":"get.v.20"},{"index":21,"sense_key":"get%2:40:14::","synset_id":"get.v.21"},{"index":22,"sense_key":"get%2:40:13::","synset_id":"get.v.22"},{"index":23,"sense_key":"get%2:39:13::","synset_id":"catch.v.21"},{"index":24,"sense_key":"get%2:39:09::","synset_id":"catch.v.22"},{"index":25,"sense_key":"get%2:39:14::","synset_id":"get.v.25"},{"index":26,"sense_key":"get%2:38:15::","synset_id":"scram.v.01"},{"index":27,"sense_key":"get%2:38:12::","synset_id":"get.v.27"},{"index":28,"sense_key":"get%2:37:12::","synset_id":"get.v.28"},{"index":29,"sense_key":"get%2:37:11::","synset_id":"get.v.29"},{"index":30,"sense_key":"get%2:36:14::","synset_id":"catch.v.24"},{"index":31,"sense_key":"get%2:36:00::","synset_id":"draw.v.15"},{"index":32,"sense_key":"get%2:35:12::","synset_id":"get.v.32"},{"index":33,"sense_key":"get%2:31:10::","synset_id":"perplex.v.01"},{"index":34,"sense_key":"get%2:30:12::","synset_id":"get_down.v.07"},{"index":35,"sense_key":"get%2:29:11::","synset_id":"suffer.v.02"},{"index":36,"sense_key":"get%2:29:08::","synset_id":"beget.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"get%2:40:00::","call_ids":["senseval3.d000.s126.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"get%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2677,"cached_input_tokens":0,"output_tokens":97,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.006324,"input_uncached_usd":0.005353999999999999,"input_cached_usd":0.0,"output_usd":0.00097,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6375068329507485}
{"item_id":"senseval3.d000.s126.t001","gold_sense_keys":["half%1:23:00::"],"candidates":[{"index":1,"sense_key":"half%1:23:00::","synset_id":"one-half.n.01"},{"index":2,"sense_key":"half%1:28:00::","synset_id":"half.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"half%1:23:00::","call_ids":["senseval3.d000.s126.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"half%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":442,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001004,"input_uncached_usd":0.0008839999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.563144415966235}
{"item_id":"senseval3.d000.s126.t002","gold_sense_keys":["have%2:40:00::","have%2:40:04::"],"candidates":[{"index":1,"sense_key":"have%2:40:00::","synset_id":"have.v.01"},{"index":2,"sense_key":"have%2:42:00::","synset_id":"have.v.02"},{"index":3,"sense_key":"have%2:39:06::","synset_id":"experience.v.03"},{"index":4,"sense_key":"have%2:40:04::","synset_id":"own.v.01"},{"index":5,"sense_key":"have%2:30:00::","synset_id":"get.v.03"},{"index":6,"sense_key":"have%2:34:00::","synset_id":"consume.v.02"},{"index":7,"sense_key":"have%2:41:00::","synset_id":"have.v.07"},{"index":8,"sense_key":"have%2:36:00::","synset_id":"hold.v.03"},{"index":9,"sense_key":"have%2:40:01::","synset_id":"have.v.09"},{"index":10,"sense_key":"have%2:42:12::","synset_id":"have.v.10"},{"index":11,"sense_key":"have%2:30:01::","synset_id":"have.v.11"},{"index":12,"sense_key":"have%2:29:02::","synset_id":"have.v.12"},{"index":13,"sense_key":"have%2:32:00::","synset_id":"induce.v.02"},{"index":14,"sense_key":"have%2:40:05::","synset_id":"accept.v.02"},{"index":15,"sense_key":"have%2:40:03::","synset_id":"receive.v.01"},{"index":16,"sense_key":"have%2:29:05::","synset_id":"suffer.v.02"},{"index":17,"sense_key":"have%2:40:02::","synset_id":"have.v.17"},{"index":18,"sense_key":"have%2:29:00::","synset_id":"give_birth.v.01"},{"index":19,"sense_key":"have%2:35:00::","synset_id":"take.v.35"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"have%2:40:00::","call_ids":["senseval3.d000.s126.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"have%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1503,"cached_input_tokens":0,"output_tokens":88,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003886,"input_uncached_usd":0.003006,"input_cached_usd":0.0,"output_usd":0.00088,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.706306540989317}
{"item_id":"senseval3.d000.s127.t001","gold_sense_keys":["deal%1:10:00::"],"candidates":[{"index":1,"sense_key":"deal%1:04:02::","synset_id":"deal.n.01"},{"index":2,"sense_key":"deal%1:10:00::","synset_id":"bargain.n.01"},{"index":3,"sense_key":"deal%1:23:00::","synset_id":"batch.n.02"},{"index":4,"sense_key":"deal%1:27:00::","synset_id":"deal.n.04"},{"index":5,"sense_key":"deal%1:27:01::","synset_id":"softwood.n.01"},{"index":6,"sense_key":"deal%1:14:00::","synset_id":"hand.n.06"},{"index":7,"sense_key":"deal%1:11:00::","synset_id":"deal.n.07"},{"index":8,"sense_key":"deal%1:04:00::","synset_id":"deal.n.08"},{"index":9,"sense_key":"deal%1:04:01::","synset_id":"deal.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"deal%1:10:00::","call_ids":["senseval3.d000.s127.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"deal%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":891,"cached_input_tokens":0,"output_tokens":112,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002902,"input_uncached_usd":0.001782,"input_cached_usd":0.0,"output_usd":0.0011200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6247357910033315}
{"item_id":"senseval3.d000.s128.t001","gold_sense_keys":["business%1:14:00::"],"candidates":[{"index":1,"sense_key":"business%1:14:00::","synset_id":"business.n.01"},{"index":2,"sense_key":"business%1:04:01::","synset_id":"commercial_enterprise.n.02"},{"index":3,"sense_key":"business%1:04:00::","synset_id":"occupation.n.01"},{"index":4,"sense_key":"business%1:09:01::","synset_id":"business.n.04"},{"index":5,"sense_key":"business%1:09:00::","synset_id":"business.n.05"},{"index":6,"sense_key":"business%1:04:02::","synset_id":"business.n.06"},{"index":7,"sense_key":"business%1:14:02::","synset_id":"business.n.07"},{"index":8,"sense_key":"business%1:14:01::","synset_id":"clientele.n.01"},{"index":9,"sense_key":"business%1:04:03::","synset_id":"business.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"business%1:14:00::","call_ids":["senseval3.d000.s128.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"business%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":865,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00246,"input_uncached_usd":0.00173,"input_cached_usd":0.0,"output_usd":0.0007300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.549579459009692}
{"item_id":"senseval3.d000.s128.t003","gold_sense_keys":["share%1:21:00::"],"candidates":[{"index":1,"sense_key":"share%1:21:00::","synset_id":"share.n.01"},{"index":2,"sense_key":"share%1:21:01::","synset_id":"share.n.02"},{"index":3,"sense_key":"share%1:04:00::","synset_id":"parcel.n.02"},{"index":4,"sense_key":"share%1:04:01::","synset_id":"contribution.n.01"},{"index":5,"sense_key":"share%1:06:00::","synset_id":"plowshare.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"share%1:21:00::","call_ids":["senseval3.d000.s128.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"share%1:21:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":678,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002006,"input_uncached_usd":0.001356,"input_cached_usd":0.0,"output_usd":0.0006500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2928422089898959}
{"item_id":"senseval3.d000.s130.t000","gold_sense_keys":["nod%2:32:00::"],"candidates":[{"index":1,"sense_key":"nod%2:32:01::","synset_id":"nod.v.01"},{"index":2,"sense_key":"nod%2:32:00::","synset_id":"nod.v.02"},{"index":3,"sense_key":"nod%2:29:03::","synset_id":"nod.v.03"},{"index":4,"sense_key":"nod%2:38:00::","synset_id":"nod.v.04"},{"index":5,"sense_key":"nod%2:29:02::","synset_id":"nod.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"nod%2:32:00::","call_ids":["senseval3.d000.s130.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"nod%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":578,"cached_input_tokens":0,"output_tokens":80,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001956,"input_uncached_usd":0.001156,"input_cached_usd":0.0,"output_usd":0.0008,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5800389159703627}
{"item_id":"senseval3.d000.s132.t000","gold_sense_keys":["see%2:31:01::"],"candidates":[{"index":1,"sense_key":"see%2:39:00::","synset_id":"see.v.01"},{"index":2,"sense_key":"see%2:31:01::","synset_id":"understand.v.02"},{"index":3,"sense_key":"see%2:39:02::","synset_id":"witness.v.02"},{"index":4,"sense_key":"see%2:36:00::","synset_id":"visualize.v.01"},{"index":5,"sense_key":"see%2:31:00::","synset_id":"see.v.05"},{"index":6,"sense_key":"see%2:31:02::","synset_id":"learn.v.02"},{"index":7,"sense_key":"see%2:39:03::","synset_id":"watch.v.03"},{"index":8,"sense_key":"see%2:38:00::","synset_id":"meet.v.01"},{"index":9,"sense_key":"see%2:32:00::","synset_id":"determine.v.08"},{"index":10,"sense_key":"see%2:31:03::","synset_id":"see.v.10"},{"index":11,"sense_key":"see%2:41:11::","synset_id":"see.v.11"},{"index":12,"sense_key":"see%2:41:12::","synset_id":"see.v.12"},{"index":13,"sense_key":"see%2:41:10::","synset_id":"visit.v.01"},{"index":14,"sense_key":"see%2:41:14::","synset_id":"attend.v.02"},{"index":15,"sense_key":"see%2:35:09::","synset_id":"see.v.15"},{"index":16,"sense_key":"see%2:41:00::","synset_id":"go_steady.v.01"},{"index":17,"sense_key":"see%2:39:05::","synset_id":"see.v.17"},{"index":18,"sense_key":"see%2:31:08::","synset_id":"see.v.18"},{"index":19,"sense_key":"see%2:39:11::","synset_id":"see.v.19"},{"index":20,"sense_key":"see%2:39:13::","synset_id":"examine.v.02"},{"index":21,"sense_key":"see%2:39:12::","synset_id":"experience.v.01"},{"index":22,"sense_key":"see%2:38:03::","synset_id":"see.v.22"},{"index":23,"sense_key":"see%2:33:00::","synset_id":"see.v.23"},{"index":24,"sense_key":"see%2:31:13::","synset_id":"interpret.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"see%2:31:01::","call_ids":["senseval3.d000.s132.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"see%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1861,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003842,"input_uncached_usd":0.003722,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2563200409058481}
{"item_id":"senseval3.d000.s133.t000","gold_sense_keys":["understand%2:31:01::"],"candidates":[{"index":1,"sense_key":"understand%2:31:00::","synset_id":"understand.v.01"},{"index":2,"sense_key":"understand%2:31:01::","synset_id":"understand.v.02"},{"index":3,"sense_key":"understand%2:31:03::","synset_id":"understand.v.03"},{"index":4,"sense_key":"understand%2:31:02::","synset_id":"understand.v.04"},{"index":5,"sense_key":"understand%2:31:04::","synset_id":"sympathize.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"understand%2:31:04::","call_ids":["senseval3.d000.s133.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"understand%2:31:04::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":603,"cached_input_tokens":0,"output_tokens":94,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0021460000000000003,"input_uncached_usd":0.001206,"input_cached_usd":0.0,"output_usd":0.0009400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4763710830593482}
{"item_id":"senseval3.d000.s133.t001","gold_sense_keys":["state%1:15:01::"],"candidates":[{"index":1,"sense_key":"state%1:15:01::","synset_id":"state.n.01"},{"index":2,"sense_key":"state%1:03:00::","synset_id":"state.n.02"},{"index":3,"sense_key":"state%1:14:01::","synset_id":"state.n.03"},{"index":4,"sense_key":"state%1:14:00::","synset_id":"state.n.04"},{"index":5,"sense_key":"state%1:26:02::","synset_id":"state_of_matter.n.01"},{"index":6,"sense_key":"state%1:26:01::","synset_id":"state.n.06"},{"index":7,"sense_key":"state%1:15:00::","synset_id":"country.n.02"},{"index":8,"sense_key":"state%1:14:02::","synset_id":"department_of_state.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"state%1:03:00::","call_ids":["senseval3.d000.s133.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"state%1:03:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":836,"cached_input_tokens":0,"output_tokens":217,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003842,"input_uncached_usd":0.0016719999999999999,"input_cached_usd":0.0,"output_usd":0.00217,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.686037083971314}
{"item_id":"senseval3.d000.s134.t000","gold_sense_keys":["come%2:38:00::","come%2:38:04::"],"candidates":[{"index":1,"sense_key":"come%2:38:00::","synset_id":"come.v.01"},{"index":2,"sense_key":"come%2:38:04::","synset_id":"arrive.v.01"},{"index":3,"sense_key":"come%2:30:01::","synset_id":"come.v.03"},{"index":4,"sense_key":"come%2:30:03::","synset_id":"come.v.04"},{"index":5,"sense_key":"come%2:42:04::","synset_id":"come.v.05"},{"index":6,"sense_key":"come%2:30:02::","synset_id":"come.v.06"},{"index":7,"sense_key":"come%2:42:13::","synset_id":"issue_forth.v.01"},{"index":8,"sense_key":"come%2:42:09::","synset_id":"hail.v.02"},{"index":9,"sense_key":"come%2:42:03::","synset_id":"come.v.09"},{"index":10,"sense_key":"come%2:42:05::","synset_id":"come.v.10"},{"index":11,"sense_key":"come%2:38:03::","synset_id":"come.v.11"},{"index":12,"sense_key":"come%2:42:00::","synset_id":"fall.v.04"},{"index":13,"sense_key":"come%2:30:13::","synset_id":"come.v.13"},{"index":14,"sense_key":"come%2:42:12::","synset_id":"total.v.01"},{"index":15,"sense_key":"come%2:42:11::","synset_id":"come.v.15"},{"index":16,"sense_key":"come%2:38:02::","synset_id":"come.v.16"},{"index":17,"sense_key":"come%2:31:14::","synset_id":"occur.v.02"},{"index":18,"sense_key":"come%2:42:15::","synset_id":"derive.v.05"},{"index":19,"sense_key":"come%2:42:02::","synset_id":"do.v.04"},{"index":20,"sense_key":"come%2:39:00::","synset_id":"come.v.20"},{"index":21,"sense_key":"come%2:31:13::","synset_id":"come.v.21"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"come%2:38:04::","call_ids":["senseval3.d000.s134.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"come%2:38:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1476,"cached_input_tokens":0,"output_tokens":235,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.005301999999999999,"input_uncached_usd":0.0029519999999999998,"input_cached_usd":0.0,"output_usd":0.00235,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.8983676669886336}
{"item_id":"senseval3.d000.s134.t001","gold_sense_keys":["retire%2:41:01::"],"candidates":[{"index":1,"sense_key":"retire%2:41:01::","synset_id":"retire.v.01"},{"index":2,"sense_key":"retire%2:41:02::","synset_id":"retire.v.02"},{"index":3,"sense_key":"retire%2:38:00::","synset_id":"withdraw.v.01"},{"index":4,"sense_key":"retire%2:41:04::","synset_id":"retire.v.04"},{"index":5,"sense_key":"retire%2:41:00::","synset_id":"adjourn.v.02"},{"index":6,"sense_key":"retire%2:41:03::","synset_id":"retire.v.06"},{"index":7,"sense_key":"retire%2:40:00::","synset_id":"retire.v.07"},{"index":8,"sense_key":"retire%2:37:00::","synset_id":"retire.v.08"},{"index":9,"sense_key":"retire%2:35:00::","synset_id":"put_out.v.07"},{"index":10,"sense_key":"retire%2:33:00::","synset_id":"retire.v.10"},{"index":11,"sense_key":"retire%2:29:00::","synset_id":"go_to_bed.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"retire%2:41:01::","call_ids":["senseval3.d000.s134.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"retire%2:41:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":941,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002452,"input_uncached_usd":0.001882,"input_cached_usd":0.0,"output_usd":0.0005700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.439087999984622}
{"item_id":"senseval3.d000.s135.t000","gold_sense_keys":["do%2:36:01::"],"candidates":[{"index":1,"sense_key":"do%2:41:01::","synset_id":"make.v.01"},{"index":2,"sense_key":"do%2:36:01::","synset_id":"perform.v.01"},{"index":3,"sense_key":"do%2:41:04::","synset_id":"do.v.03"},{"index":4,"sense_key":"do%2:42:00::","synset_id":"do.v.04"},{"index":5,"sense_key":"do%2:36:02::","synset_id":"cause.v.01"},{"index":6,"sense_key":"do%2:41:02::","synset_id":"practice.v.01"},{"index":7,"sense_key":"do%2:42:01::","synset_id":"suffice.v.01"},{"index":8,"sense_key":"do%2:36:00::","synset_id":"do.v.08"},{"index":9,"sense_key":"do%2:29:09::","synset_id":"act.v.02"},{"index":10,"sense_key":"do%2:42:02::","synset_id":"serve.v.09"},{"index":11,"sense_key":"do%2:41:03::","synset_id":"do.v.11"},{"index":12,"sense_key":"do%2:29:04::","synset_id":"dress.v.16"},{"index":13,"sense_key":"do%2:38:00::","synset_id":"do.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"do%2:41:04::","call_ids":["senseval3.d000.s135.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"do%2:41:04::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1113,"cached_input_tokens":0,"output_tokens":198,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004206,"input_uncached_usd":0.002226,"input_cached_usd":0.0,"output_usd":0.00198,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.412145624984987}
{"item_id":"senseval3.d000.s135.t001","gold_sense_keys":["free%5:00:00:unpaid:00"],"candidates":[{"index":1,"sense_key":"free%3:00:00::","synset_id":"free.a.01"},{"index":2,"sense_key":"free%3:00:02::","synset_id":"free.a.02"},{"index":3,"sense_key":"free%5:00:00:unpaid:00","synset_id":"complimentary.s.01"},{"index":4,"sense_key":"free%5:00:02:unoccupied:00","synset_id":"free.s.02"},{"index":5,"sense_key":"free%5:00:00:unfixed:00","synset_id":"detached.s.05"},{"index":6,"sense_key":"free%3:00:01::","synset_id":"free.a.06"},{"index":7,"sense_key":"free%5:00:01:unoccupied:00","synset_id":"spare.s.03"},{"index":8,"sense_key":"free%5:00:00:nonexistent:00","synset_id":"barren.s.03"},{"index":9,"sense_key":"free%5:00:00:inexact:00","synset_id":"free.s.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"free%5:00:00:unpaid:00","call_ids":["senseval3.d000.s135.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"free%5:00:00:unpaid:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":867,"cached_input_tokens":0,"output_tokens":44,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0021739999999999997,"input_uncached_usd":0.0017339999999999999,"input_cached_usd":0.0,"output_usd":0.00044,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.780271124909632}
{"item_id":"senseval3.d000.s135.t002","gold_sense_keys":["job%1:04:01::"],"candidates":[{"index":1,"sense_key":"job%1:04:00::","synset_id":"occupation.n.01"},{"index":2,"sense_key":"job%1:04:01::","synset_id":"job.n.02"},{"index":3,"sense_key":"job%1:06:00::","synset_id":"job.n.03"},{"index":4,"sense_key":"job%1:06:01::","synset_id":"job.n.04"},{"index":5,"sense_key":"job%1:04:04::","synset_id":"job.n.05"},{"index":6,"sense_key":"job%1:04:05::","synset_id":"job.n.06"},{"index":7,"sense_key":"job%1:04:03::","synset_id":"job.n.07"},{"index":8,"sense_key":"job%1:26:00::","synset_id":"problem.n.01"},{"index":9,"sense_key":"job%1:18:00::","synset_id":"job.n.09"},{"index":10,"sense_key":"job%1:18:01::","synset_id":"job.n.10"},{"index":11,"sense_key":"job%1:10:01::","synset_id":"job.n.11"},{"index":12,"sense_key":"job%1:10:00::","synset_id":"job.n.12"},{"index":13,"sense_key":"job%1:04:02::","synset_id":"caper.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"job%1:04:01::","call_ids":["senseval3.d000.s135.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"job%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1043,"cached_input_tokens":0,"output_tokens":69,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002776,"input_uncached_usd":0.0020859999999999997,"input_cached_usd":0.0,"output_usd":0.0006900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8376544169150293}
{"item_id":"senseval3.d000.s136.t000","gold_sense_keys":["tell%2:32:00::","tell%2:32:04::"],"candidates":[{"index":1,"sense_key":"tell%2:32:04::","synset_id":"state.v.01"},{"index":2,"sense_key":"tell%2:32:00::","synset_id":"tell.v.02"},{"index":3,"sense_key":"tell%2:32:01::","synset_id":"tell.v.03"},{"index":4,"sense_key":"tell%2:32:02::","synset_id":"order.v.01"},{"index":5,"sense_key":"tell%2:31:00::","synset_id":"tell.v.05"},{"index":6,"sense_key":"tell%2:32:08::","synset_id":"assure.v.02"},{"index":7,"sense_key":"tell%2:32:03::","synset_id":"tell.v.07"},{"index":8,"sense_key":"tell%2:31:01::","synset_id":"distinguish.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"tell%2:32:00::","call_ids":["senseval3.d000.s136.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"tell%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":782,"cached_input_tokens":0,"output_tokens":81,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002374,"input_uncached_usd":0.0015639999999999999,"input_cached_usd":0.0,"output_usd":0.0008100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6748325830558315}
{"item_id":"senseval3.d000.s137.t000","gold_sense_keys":["feel%2:29:00::","feel%2:37:00::"],"candidates":[{"index":1,"sense_key":"feel%2:37:00::","synset_id":"feel.v.01"},{"index":2,"sense_key":"feel%2:31:00::","synset_id":"find.v.05"},{"index":3,"sense_key":"feel%2:39:00::","synset_id":"feel.v.03"},{"index":4,"sense_key":"feel%2:29:00::","synset_id":"feel.v.04"},{"index":5,"sense_key":"feel%2:31:01::","synset_id":"feel.v.05"},{"index":6,"sense_key":"feel%2:39:01::","synset_id":"feel.v.06"},{"index":7,"sense_key":"feel%2:42:00::","synset_id":"feel.v.07"},{"index":8,"sense_key":"feel%2:39:10::","synset_id":"feel.v.08"},{"index":9,"sense_key":"feel%2:35:00::","synset_id":"feel.v.09"},{"index":10,"sense_key":"feel%2:35:01::","synset_id":"palpate.v.01"},{"index":11,"sense_key":"feel%2:40:00::","synset_id":"feel.v.11"},{"index":12,"sense_key":"feel%2:39:09::","synset_id":"feel.v.12"},{"index":13,"sense_key":"feel%2:35:11::","synset_id":"feel.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"feel%2:37:00::","call_ids":["senseval3.d000.s137.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"feel%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1121,"cached_input_tokens":0,"output_tokens":142,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0036620000000000003,"input_uncached_usd":0.002242,"input_cached_usd":0.0,"output_usd":0.00142,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.843840540968813}
{"item_id":"senseval3.d000.s137.t001","gold_sense_keys":["twinge%1:12:00::"],"candidates":[{"index":1,"sense_key":"twinge%1:12:00::","synset_id":"pang.n.01"},{"index":2,"sense_key":"twinge%1:09:00::","synset_id":"twinge.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"twinge%1:12:00::","call_ids":["senseval3.d000.s137.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"twinge%1:12:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":430,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00098,"input_uncached_usd":0.00086,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.001458915998228}
{"item_id":"senseval3.d000.s137.t002","gold_sense_keys":["annoyance%1:12:00::"],"candidates":[{"index":1,"sense_key":"annoyance%1:26:00::","synset_id":"irritation.n.01"},{"index":2,"sense_key":"annoyance%1:12:00::","synset_id":"annoyance.n.02"},{"index":3,"sense_key":"annoyance%1:18:00::","synset_id":"aggravator.n.01"},{"index":4,"sense_key":"annoyance%1:09:00::","synset_id":"annoyance.n.04"},{"index":5,"sense_key":"annoyance%1:04:00::","synset_id":"annoyance.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"annoyance%1:26:00::","call_ids":["senseval3.d000.s137.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"annoyance%1:26:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":651,"cached_input_tokens":0,"output_tokens":107,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002372,"input_uncached_usd":0.001302,"input_cached_usd":0.0,"output_usd":0.00107,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.709154250100255}
{"item_id":"senseval3.d000.s137.t003","gold_sense_keys":["hear%2:39:00::"],"candidates":[{"index":1,"sense_key":"hear%2:39:00::","synset_id":"hear.v.01"},{"index":2,"sense_key":"hear%2:31:00::","synset_id":"learn.v.02"},{"index":3,"sense_key":"hear%2:41:00::","synset_id":"hear.v.03"},{"index":4,"sense_key":"hear%2:39:01::","synset_id":"hear.v.04"},{"index":5,"sense_key":"hear%2:39:05::","synset_id":"listen.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"hear%2:39:00::","call_ids":["senseval3.d000.s137.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"hear%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":638,"cached_input_tokens":0,"output_tokens":108,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002356,"input_uncached_usd":0.001276,"input_cached_usd":0.0,"output_usd":0.00108,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":6.332075332989916}
{"item_id":"senseval3.d000.s137.t004","gold_sense_keys":["familiar%3:00:00::"],"candidates":[{"index":1,"sense_key":"familiar%3:00:00::","synset_id":"familiar.a.01"},{"index":2,"sense_key":"familiar%3:00:02::","synset_id":"familiar.a.02"},{"index":3,"sense_key":"familiar%5:00:00:informed:00","synset_id":"conversant.s.01"},{"index":4,"sense_key":"familiar%5:00:00:close:02","synset_id":"familiar.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"familiar%3:00:00::","call_ids":["senseval3.d000.s137.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"familiar%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":608,"cached_input_tokens":0,"output_tokens":79,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002006,"input_uncached_usd":0.0012159999999999999,"input_cached_usd":0.0,"output_usd":0.00079,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.513797583989799}
{"item_id":"senseval3.d000.s138.t000","gold_sense_keys":["wild%5:00:00:unrealistic:00"],"candidates":[{"index":1,"sense_key":"wild%3:00:02::","synset_id":"wild.a.01"},{"index":2,"sense_key":"wild%3:00:01::","synset_id":"wild.a.02"},{"index":3,"sense_key":"wild%5:00:00:passionate:00","synset_id":"wild.s.01"},{"index":4,"sense_key":"wild%5:00:01:uncontrolled:00","synset_id":"wild.s.02"},{"index":5,"sense_key":"wild%5:00:00:intense:00","synset_id":"violent.s.02"},{"index":6,"sense_key":"wild%5:00:00:unsupported:02","synset_id":"baseless.s.01"},{"index":7,"sense_key":"wild%5:00:00:insane:00","synset_id":"raving_mad.s.01"},{"index":8,"sense_key":"wild%5:00:00:dangerous:00","synset_id":"hazardous.s.01"},{"index":9,"sense_key":"wild%5:00:00:unrealistic:00","synset_id":"fantastic.s.03"},{"index":10,"sense_key":"wild%5:00:00:inhospitable:00","synset_id":"godforsaken.s.01"},{"index":11,"sense_key":"wild%5:00:00:enthusiastic:00","synset_id":"crazy.s.05"},{"index":12,"sense_key":"wild%5:00:00:noncivilized:00","synset_id":"barbarian.s.01"},{"index":13,"sense_key":"wild%5:00:00:stormy:00","synset_id":"angry.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"wild%5:00:00:unrealistic:00","call_ids":["senseval3.d000.s138.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"wild%5:00:00:unrealistic:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1282,"cached_input_tokens":0,"output_tokens":122,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003784,"input_uncached_usd":0.002564,"input_cached_usd":0.0,"output_usd":0.0012200000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9978362920228392}
{"item_id":"senseval3.d000.s138.t001","gold_sense_keys":["thought%1:09:01::"],"candidates":[{"index":1,"sense_key":"thought%1:09:01::","synset_id":"idea.n.01"},{"index":2,"sense_key":"thought%1:09:00::","synset_id":"thinking.n.01"},{"index":3,"sense_key":"thought%1:09:02::","synset_id":"thought.n.03"},{"index":4,"sense_key":"thought%1:09:03::","synset_id":"opinion.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"thought%1:09:01::","call_ids":["senseval3.d000.s138.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"thought%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":639,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001398,"input_uncached_usd":0.0012779999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5669154160423204}
{"item_id":"senseval3.d000.s138.t004","gold_sense_keys":["clouded%5:00:00:confused:00"],"candidates":[{"index":1,"sense_key":"clouded%5:00:00:troubled:00","synset_id":"clouded.s.01"},{"index":2,"sense_key":"clouded%5:00:00:cloudy:00","synset_id":"cloud-covered.s.01"},{"index":3,"sense_key":"clouded%5:00:00:confused:00","synset_id":"clouded.s.03"},{"index":4,"sense_key":"clouded%5:00:00:unclear:00","synset_id":"blurred.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"clouded%5:00:00:confused:00","call_ids":["senseval3.d000.s138.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"clouded%5:00:00:confused:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":613,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001346,"input_uncached_usd":0.0012259999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1196841669734567}
{"item_id":"senseval3.d000.s138.t005","gold_sense_keys":["brain%1:09:01::"],"candidates":[{"index":1,"sense_key":"brain%1:08:00::","synset_id":"brain.n.01"},{"index":2,"sense_key":"brain%1:09:00::","synset_id":"brain.n.02"},{"index":3,"sense_key":"brain%1:09:01::","synset_id":"mind.n.01"},{"index":4,"sense_key":"brain%1:18:00::","synset_id":"genius.n.01"},{"index":5,"sense_key":"brain%1:13:00::","synset_id":"brain.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"brain%1:09:01::","call_ids":["senseval3.d000.s138.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"brain%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":707,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002034,"input_uncached_usd":0.001414,"input_cached_usd":0.0,"output_usd":0.00062,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.203461250057444}
{"item_id":"senseval3.d000.s139.t000","gold_sense_keys":["suppose%2:32:00::"],"candidates":[{"index":1,"sense_key":"suppose%2:32:00::","synset_id":"suppose.v.01"},{"index":2,"sense_key":"suppose%2:31:00::","synset_id":"think.v.02"},{"index":3,"sense_key":"suppose%2:31:01::","synset_id":"speculate.v.01"},{"index":4,"sense_key":"suppose%2:31:06::","synset_id":"presuppose.v.01"},{"index":5,"sense_key":"suppose%2:31:05::","synset_id":"presuppose.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"suppose%2:32:00::","call_ids":["senseval3.d000.s139.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"suppose%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":725,"cached_input_tokens":0,"output_tokens":103,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00248,"input_uncached_usd":0.00145,"input_cached_usd":0.0,"output_usd":0.00103,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8048256250331178}
{"item_id":"senseval3.d000.s139.t001","gold_sense_keys":["suppose%2:32:00::"],"candidates":[{"index":1,"sense_key":"suppose%2:32:00::","synset_id":"suppose.v.01"},{"index":2,"sense_key":"suppose%2:31:00::","synset_id":"think.v.02"},{"index":3,"sense_key":"suppose%2:31:01::","synset_id":"speculate.v.01"},{"index":4,"sense_key":"suppose%2:31:06::","synset_id":"presuppose.v.01"},{"index":5,"sense_key":"suppose%2:31:05::","synset_id":"presuppose.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"suppose%2:32:00::","call_ids":["senseval3.d000.s139.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"suppose%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":724,"cached_input_tokens":0,"output_tokens":82,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002268,"input_uncached_usd":0.0014479999999999999,"input_cached_usd":0.0,"output_usd":0.0008200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3246343749342486}
{"item_id":"senseval3.d000.s139.t002","gold_sense_keys":["guy%1:18:00::"],"candidates":[{"index":1,"sense_key":"guy%1:18:00::","synset_id":"guy.n.01"},{"index":2,"sense_key":"guy%1:06:01::","synset_id":"guy.n.02"},{"index":3,"sense_key":"guy%1:06:00::","synset_id":"guy.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"guy%1:18:00::","call_ids":["senseval3.d000.s139.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"guy%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":532,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0011840000000000002,"input_uncached_usd":0.001064,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4628449579467997}
{"item_id":"senseval3.d000.s139.t003","gold_sense_keys":["say%2:32:01::"],"candidates":[{"index":1,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":6,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":9,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":10,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":11,"sense_key":"say%2:32:09::","synset_id":"say.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"say%2:32:01::","call_ids":["senseval3.d000.s139.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"say%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":997,"cached_input_tokens":0,"output_tokens":115,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003144,"input_uncached_usd":0.001994,"input_cached_usd":0.0,"output_usd":0.0011500000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8738342080032453}
{"item_id":"senseval3.d000.s140.t001","gold_sense_keys":["professional%3:00:01::"],"candidates":[{"index":1,"sense_key":"professional%3:00:01::","synset_id":"professional.a.01"},{"index":2,"sense_key":"professional%3:01:01::","synset_id":"professional.a.02"},{"index":3,"sense_key":"professional%3:00:02::","synset_id":"professional.a.03"},{"index":4,"sense_key":"professional%3:01:00::","synset_id":"professional.a.04"},{"index":5,"sense_key":"professional%5:00:00:white-collar:00","synset_id":"professional.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"professional%3:00:01::","call_ids":["senseval3.d000.s140.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"professional%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":637,"cached_input_tokens":0,"output_tokens":95,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0022240000000000003,"input_uncached_usd":0.001274,"input_cached_usd":0.0,"output_usd":0.0009500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1101300000445917}
{"item_id":"senseval3.d000.s140.t002","gold_sense_keys":["killer%1:18:00::"],"candidates":[{"index":1,"sense_key":"killer%1:18:00::","synset_id":"killer.n.01"},{"index":2,"sense_key":"killer%1:26:00::","synset_id":"cause_of_death.n.01"},{"index":3,"sense_key":"killer%1:09:00::","synset_id":"killer.n.03"},{"index":4,"sense_key":"killer%1:05:00::","synset_id":"killer_whale.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"killer%1:18:00::","call_ids":["senseval3.d000.s140.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"killer%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":579,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001278,"input_uncached_usd":0.001158,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5552277920069173}
{"item_id":"senseval3.d000.s141.t000","gold_sense_keys":["nut%1:18:02::"],"candidates":[{"index":1,"sense_key":"nut%1:20:00::","synset_id":"nut.n.01"},{"index":2,"sense_key":"nut%1:18:00::","synset_id":"nut.n.02"},{"index":3,"sense_key":"nut%1:06:00::","synset_id":"nut.n.03"},{"index":4,"sense_key":"nut%1:23:00::","synset_id":"en.n.01"},{"index":5,"sense_key":"nut%1:18:02::","synset_id":"crackpot.n.01"},{"index":6,"sense_key":"nut%1:18:01::","synset_id":"addict.n.01"},{"index":7,"sense_key":"nut%1:08:00::","synset_id":"testis.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"nut%1:18:02::","call_ids":["senseval3.d000.s141.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"nut%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":734,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001588,"input_uncached_usd":0.001468,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1300966669805348}
{"item_id":"senseval3.d000.s141.t001","gold_sense_keys":["harm%1:11:01::"],"candidates":[{"index":1,"sense_key":"harm%1:26:00::","synset_id":"injury.n.01"},{"index":2,"sense_key":"harm%1:11:01::","synset_id":"damage.n.01"},{"index":3,"sense_key":"harm%1:04:00::","synset_id":"damage.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"harm%1:04:00::","call_ids":["senseval3.d000.s141.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"harm%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":504,"cached_input_tokens":0,"output_tokens":174,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0027480000000000004,"input_uncached_usd":0.001008,"input_cached_usd":0.0,"output_usd":0.0017400000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.123198291985318}
{"item_id":"senseval3.d000.s141.t002","gold_sense_keys":["do%2:36:02::"],"candidates":[{"index":1,"sense_key":"do%2:41:01::","synset_id":"make.v.01"},{"index":2,"sense_key":"do%2:36:01::","synset_id":"perform.v.01"},{"index":3,"sense_key":"do%2:41:04::","synset_id":"do.v.03"},{"index":4,"sense_key":"do%2:42:00::","synset_id":"do.v.04"},{"index":5,"sense_key":"do%2:36:02::","synset_id":"cause.v.01"},{"index":6,"sense_key":"do%2:41:02::","synset_id":"practice.v.01"},{"index":7,"sense_key":"do%2:42:01::","synset_id":"suffice.v.01"},{"index":8,"sense_key":"do%2:36:00::","synset_id":"do.v.08"},{"index":9,"sense_key":"do%2:29:09::","synset_id":"act.v.02"},{"index":10,"sense_key":"do%2:42:02::","synset_id":"serve.v.09"},{"index":11,"sense_key":"do%2:41:03::","synset_id":"do.v.11"},{"index":12,"sense_key":"do%2:29:04::","synset_id":"dress.v.16"},{"index":13,"sense_key":"do%2:38:00::","synset_id":"do.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"do%2:36:02::","call_ids":["senseval3.d000.s141.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"do%2:36:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1158,"cached_input_tokens":0,"output_tokens":149,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0038060000000000004,"input_uncached_usd":0.002316,"input_cached_usd":0.0,"output_usd":0.0014900000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8336726250126958}
{"item_id":"senseval3.d000.s142.t001","gold_sense_keys":["do%2:36:01::","do%2:41:01::","do%2:41:04::"],"candidates":[{"index":1,"sense_key":"do%2:41:01::","synset_id":"make.v.01"},{"index":2,"sense_key":"do%2:36:01::","synset_id":"perform.v.01"},{"index":3,"sense_key":"do%2:41:04::","synset_id":"do.v.03"},{"index":4,"sense_key":"do%2:42:00::","synset_id":"do.v.04"},{"index":5,"sense_key":"do%2:36:02::","synset_id":"cause.v.01"},{"index":6,"sense_key":"do%2:41:02::","synset_id":"practice.v.01"},{"index":7,"sense_key":"do%2:42:01::","synset_id":"suffice.v.01"},{"index":8,"sense_key":"do%2:36:00::","synset_id":"do.v.08"},{"index":9,"sense_key":"do%2:29:09::","synset_id":"act.v.02"},{"index":10,"sense_key":"do%2:42:02::","synset_id":"serve.v.09"},{"index":11,"sense_key":"do%2:41:03::","synset_id":"do.v.11"},{"index":12,"sense_key":"do%2:29:04::","synset_id":"dress.v.16"},{"index":13,"sense_key":"do%2:38:00::","synset_id":"do.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"do%2:36:01::","call_ids":["senseval3.d000.s142.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"do%2:36:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1175,"cached_input_tokens":0,"output_tokens":40,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0027500000000000003,"input_uncached_usd":0.00235,"input_cached_usd":0.0,"output_usd":0.0004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.775788249913603}
{"item_id":"senseval3.d000.s143.t000","gold_sense_keys":["feel%2:29:00::","feel%2:37:00::"],"candidates":[{"index":1,"sense_key":"feel%2:37:00::","synset_id":"feel.v.01"},{"index":2,"sense_key":"feel%2:31:00::","synset_id":"find.v.05"},{"index":3,"sense_key":"feel%2:39:00::","synset_id":"feel.v.03"},{"index":4,"sense_key":"feel%2:29:00::","synset_id":"feel.v.04"},{"index":5,"sense_key":"feel%2:31:01::","synset_id":"feel.v.05"},{"index":6,"sense_key":"feel%2:39:01::","synset_id":"feel.v.06"},{"index":7,"sense_key":"feel%2:42:00::","synset_id":"feel.v.07"},{"index":8,"sense_key":"feel%2:39:10::","synset_id":"feel.v.08"},{"index":9,"sense_key":"feel%2:35:00::","synset_id":"feel.v.09"},{"index":10,"sense_key":"feel%2:35:01::","synset_id":"palpate.v.01"},{"index":11,"sense_key":"feel%2:40:00::","synset_id":"feel.v.11"},{"index":12,"sense_key":"feel%2:39:09::","synset_id":"feel.v.12"},{"index":13,"sense_key":"feel%2:35:11::","synset_id":"feel.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"feel%2:37:00::","call_ids":["senseval3.d000.s143.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"feel%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1143,"cached_input_tokens":0,"output_tokens":201,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0042959999999999995,"input_uncached_usd":0.002286,"input_cached_usd":0.0,"output_usd":0.00201,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2851993748918176}
{"item_id":"senseval3.d000.s143.t001","gold_sense_keys":["cunning%5:00:00:adroit:00","cunning%5:00:00:artful:00"],"candidates":[{"index":1,"sense_key":"cunning%5:00:00:attractive:01","synset_id":"cunning.s.01"},{"index":2,"sense_key":"cunning%5:00:00:artful:00","synset_id":"crafty.s.01"},{"index":3,"sense_key":"cunning%5:00:00:adroit:00","synset_id":"clever.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cunning%5:00:00:artful:00","call_ids":["senseval3.d000.s143.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cunning%5:00:00:artful:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":595,"cached_input_tokens":0,"output_tokens":78,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00197,"input_uncached_usd":0.0011899999999999999,"input_cached_usd":0.0,"output_usd":0.0007800000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6326482080621645}
{"item_id":"senseval3.d000.s143.t002","gold_sense_keys":["proud%3:00:00::"],"candidates":[{"index":1,"sense_key":"proud%3:00:00::","synset_id":"proud.a.01"},{"index":2,"sense_key":"proud%5:00:00:impressive:00","synset_id":"gallant.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"proud%3:00:00::","call_ids":["senseval3.d000.s143.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"proud%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":500,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0011200000000000001,"input_uncached_usd":0.001,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1418860000558197}
{"item_id":"senseval3.d000.s143.t003","gold_sense_keys":["play%2:34:13::"],"candidates":[{"index":1,"sense_key":"play%2:33:00::","synset_id":"play.v.01"},{"index":2,"sense_key":"play%2:41:12::","synset_id":"play.v.02"},{"index":3,"sense_key":"play%2:36:12::","synset_id":"play.v.03"},{"index":4,"sense_key":"play%2:36:02::","synset_id":"act.v.03"},{"index":5,"sense_key":"play%2:41:03::","synset_id":"play.v.05"},{"index":6,"sense_key":"play%2:36:00::","synset_id":"play.v.06"},{"index":7,"sense_key":"play%2:36:01::","synset_id":"play.v.07"},{"index":8,"sense_key":"play%2:29:01::","synset_id":"act.v.05"},{"index":9,"sense_key":"play%2:38:00::","synset_id":"play.v.09"},{"index":10,"sense_key":"play%2:33:01::","synset_id":"play.v.10"},{"index":11,"sense_key":"play%2:41:00::","synset_id":"play.v.11"},{"index":12,"sense_key":"play%2:36:06::","synset_id":"play.v.12"},{"index":13,"sense_key":"play%2:36:11::","synset_id":"play.v.13"},{"index":14,"sense_key":"play%2:36:03::","synset_id":"play.v.14"},{"index":15,"sense_key":"play%2:33:03::","synset_id":"play.v.15"},{"index":16,"sense_key":"play%2:29:00::","synset_id":"play.v.16"},{"index":17,"sense_key":"play%2:41:02::","synset_id":"play.v.17"},{"index":18,"sense_key":"play%2:36:05::","synset_id":"play.v.18"},{"index":19,"sense_key":"play%2:35:00::","synset_id":"toy.v.02"},{"index":20,"sense_key":"play%2:34:13::","synset_id":"play.v.20"},{"index":21,"sense_key":"play%2:31:01::","synset_id":"dally.v.04"},{"index":22,"sense_key":"play%2:42:00::","synset_id":"play.v.22"},{"index":23,"sense_key":"play%2:41:13::","synset_id":"dally.v.01"},{"index":24,"sense_key":"play%2:38:03::","synset_id":"play.v.24"},{"index":25,"sense_key":"play%2:36:10::","synset_id":"act.v.10"},{"index":26,"sense_key":"play%2:36:04::","synset_id":"play.v.26"},{"index":27,"sense_key":"play%2:36:13::","synset_id":"bring.v.03"},{"index":28,"sense_key":"play%2:35:07::","synset_id":"play.v.28"},{"index":29,"sense_key":"play%2:33:07::","synset_id":"play.v.29"},{"index":30,"sense_key":"play%2:33:08::","synset_id":"bet.v.02"},{"index":31,"sense_key":"play%2:33:09::","synset_id":"play.v.31"},{"index":32,"sense_key":"play%2:33:13::","synset_id":"play.v.32"},{"index":33,"sense_key":"play%2:33:12::","synset_id":"play.v.33"},{"index":34,"sense_key":"play%2:33:02::","synset_id":"meet.v.10"},{"index":35,"sense_key":"play%2:29:04::","synset_id":"play.v.35"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":20,"chosen_sense_key":"play%2:34:13::","call_ids":["senseval3.d000.s143.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":20,"predicted_sense_key":"play%2:34:13::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2479,"cached_input_tokens":0,"output_tokens":76,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.005718,"input_uncached_usd":0.004958,"input_cached_usd":0.0,"output_usd":0.00076,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3958857089746743}
{"item_id":"senseval3.d000.s143.t004","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":2,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":3,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"other%3:00:00::","call_ids":["senseval3.d000.s143.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":586,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001292,"input_uncached_usd":0.0011719999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1429623340955004}
{"item_id":"senseval3.d000.s143.t005","gold_sense_keys":["man%1:18:00::"],"candidates":[{"index":1,"sense_key":"man%1:18:00::","synset_id":"man.n.01"},{"index":2,"sense_key":"man%1:18:03::","synset_id":"serviceman.n.01"},{"index":3,"sense_key":"man%1:18:04::","synset_id":"man.n.03"},{"index":4,"sense_key":"man%1:05:01::","synset_id":"homo.n.02"},{"index":5,"sense_key":"man%1:18:08::","synset_id":"man.n.05"},{"index":6,"sense_key":"man%1:18:06::","synset_id":"man.n.06"},{"index":7,"sense_key":"man%1:18:05::","synset_id":"valet.n.01"},{"index":8,"sense_key":"man%1:18:07::","synset_id":"man.n.08"},{"index":9,"sense_key":"man%1:15:00::","synset_id":"man.n.09"},{"index":10,"sense_key":"man%1:06:00::","synset_id":"man.n.10"},{"index":11,"sense_key":"man%1:05:00::","synset_id":"world.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"man%1:18:00::","call_ids":["senseval3.d000.s143.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"man%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1093,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002736,"input_uncached_usd":0.002186,"input_cached_usd":0.0,"output_usd":0.00055,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.201970374910161}
{"item_id":"senseval3.d000.s143.t006","gold_sense_keys":["soft_spot%1:07:00::"],"candidates":[{"index":1,"sense_key":"soft_spot%1:07:00::","synset_id":"weak_part.n.01"},{"index":2,"sense_key":"soft_spot%1:12:00::","synset_id":"soft_spot.n.02"},{"index":3,"sense_key":"soft_spot%1:08:00::","synset_id":"fontanelle.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"soft_spot%1:07:00::","call_ids":["senseval3.d000.s143.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"soft_spot%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":520,"cached_input_tokens":0,"output_tokens":305,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00409,"input_uncached_usd":0.00104,"input_cached_usd":0.0,"output_usd":0.00305,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.45514145900961}
{"item_id":"senseval3.d000.s144.t000","gold_sense_keys":["offense%1:04:02::"],"candidates":[{"index":1,"sense_key":"offense%1:04:02::","synset_id":"discourtesy.n.03"},{"index":2,"sense_key":"offense%1:12:00::","synset_id":"umbrage.n.01"},{"index":3,"sense_key":"offense%1:04:01::","synset_id":"crime.n.01"},{"index":4,"sense_key":"offense%1:14:00::","synset_id":"offense.n.04"},{"index":5,"sense_key":"offense%1:04:00::","synset_id":"offense.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"offense%1:04:02::","call_ids":["senseval3.d000.s144.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"offense%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":651,"cached_input_tokens":0,"output_tokens":188,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003182,"input_uncached_usd":0.001302,"input_cached_usd":0.0,"output_usd":0.0018800000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.3340990830911323}
{"item_id":"senseval3.d000.s144.t001","gold_sense_keys":["intend%2:31:00::"],"candidates":[{"index":1,"sense_key":"intend%2:31:00::","synset_id":"intend.v.01"},{"index":2,"sense_key":"intend%2:31:01::","synset_id":"intend.v.02"},{"index":3,"sense_key":"intend%2:32:00::","synset_id":"mean.v.01"},{"index":4,"sense_key":"intend%2:32:02::","synset_id":"mean.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"intend%2:31:00::","call_ids":["senseval3.d000.s144.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"intend%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":587,"cached_input_tokens":0,"output_tokens":69,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001864,"input_uncached_usd":0.001174,"input_cached_usd":0.0,"output_usd":0.0006900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.125373125076294}
{"item_id":"senseval3.d000.s144.t002","gold_sense_keys":["say%2:32:15::"],"candidates":[{"index":1,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":6,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":9,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":10,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":11,"sense_key":"say%2:32:09::","synset_id":"say.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"say%2:32:00::","call_ids":["senseval3.d000.s144.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"say%2:32:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1004,"cached_input_tokens":0,"output_tokens":234,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0043479999999999994,"input_uncached_usd":0.002008,"input_cached_usd":0.0,"output_usd":0.00234,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.8476471670437604}
{"item_id":"senseval3.d000.s146.t001","gold_sense_keys":["suspect%2:32:00::"],"candidates":[{"index":1,"sense_key":"suspect%2:32:00::","synset_id":"suspect.v.01"},{"index":2,"sense_key":"suspect%2:31:00::","synset_id":"distrust.v.01"},{"index":3,"sense_key":"suspect%2:32:02::","synset_id":"suspect.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"suspect%2:32:00::","call_ids":["senseval3.d000.s146.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"suspect%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":564,"cached_input_tokens":0,"output_tokens":90,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002028,"input_uncached_usd":0.0011279999999999999,"input_cached_usd":0.0,"output_usd":0.0009000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.980386917013675}
{"item_id":"senseval3.d000.s146.t002","gold_sense_keys":["thing%1:06:01::"],"candidates":[{"index":1,"sense_key":"thing%1:26:00::","synset_id":"thing.n.01"},{"index":2,"sense_key":"thing%1:04:00::","synset_id":"thing.n.02"},{"index":3,"sense_key":"thing%1:09:00::","synset_id":"thing.n.03"},{"index":4,"sense_key":"thing%1:06:00::","synset_id":"thing.n.04"},{"index":5,"sense_key":"thing%1:11:00::","synset_id":"thing.n.05"},{"index":6,"sense_key":"thing%1:09:02::","synset_id":"matter.n.01"},{"index":7,"sense_key":"thing%1:10:00::","synset_id":"thing.n.07"},{"index":8,"sense_key":"thing%1:06:01::","synset_id":"thing.n.08"},{"index":9,"sense_key":"thing%1:07:00::","synset_id":"thing.n.09"},{"index":10,"sense_key":"thing%1:09:01::","synset_id":"thing.n.10"},{"index":11,"sense_key":"thing%1:12:00::","synset_id":"thing.n.11"},{"index":12,"sense_key":"thing%1:03:00::","synset_id":"thing.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"thing%1:06:01::","call_ids":["senseval3.d000.s146.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"thing%1:06:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":912,"cached_input_tokens":0,"output_tokens":147,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003294,"input_uncached_usd":0.0018239999999999999,"input_cached_usd":0.0,"output_usd":0.0014700000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8439913330366835}
{"item_id":"senseval3.d000.s146.t003","gold_sense_keys":["think%2:31:03::"],"candidates":[{"index":1,"sense_key":"think%2:31:01::","synset_id":"think.v.01"},{"index":2,"sense_key":"think%2:31:03::","synset_id":"think.v.02"},{"index":3,"sense_key":"think%2:31:00::","synset_id":"think.v.03"},{"index":4,"sense_key":"think%2:31:02::","synset_id":"remember.v.01"},{"index":5,"sense_key":"think%2:36:00::","synset_id":"think.v.05"},{"index":6,"sense_key":"think%2:31:09::","synset_id":"think.v.06"},{"index":7,"sense_key":"think%2:31:06::","synset_id":"intend.v.01"},{"index":8,"sense_key":"think%2:31:12::","synset_id":"think.v.08"},{"index":9,"sense_key":"think%2:31:13::","synset_id":"think.v.09"},{"index":10,"sense_key":"think%2:31:15::","synset_id":"think.v.10"},{"index":11,"sense_key":"think%2:31:14::","synset_id":"think.v.11"},{"index":12,"sense_key":"think%2:31:08::","synset_id":"think.v.12"},{"index":13,"sense_key":"think%2:30:00::","synset_id":"think.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"think%2:31:03::","call_ids":["senseval3.d000.s146.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"think%2:31:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1145,"cached_input_tokens":0,"output_tokens":98,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0032700000000000003,"input_uncached_usd":0.00229,"input_cached_usd":0.0,"output_usd":0.0009800000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1088244589045644}
{"item_id":"senseval3.d000.s146.t004","gold_sense_keys":["coincidence%1:11:00::"],"candidates":[{"index":1,"sense_key":"coincidence%1:11:00::","synset_id":"coincidence.n.01"},{"index":2,"sense_key":"coincidence%1:07:02::","synset_id":"coincidence.n.02"},{"index":3,"sense_key":"coincidence%1:07:01::","synset_id":"concurrence.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"coincidence%1:11:00::","call_ids":["senseval3.d000.s146.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"coincidence%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":552,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001224,"input_uncached_usd":0.001104,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0819654170190915}
{"item_id":"senseval3.d000.s147.t000","gold_sense_keys":["know%2:31:00::"],"candidates":[{"index":1,"sense_key":"know%2:31:01::","synset_id":"know.v.01"},{"index":2,"sense_key":"know%2:31:03::","synset_id":"know.v.02"},{"index":3,"sense_key":"know%2:31:02::","synset_id":"know.v.03"},{"index":4,"sense_key":"know%2:31:00::","synset_id":"know.v.04"},{"index":5,"sense_key":"know%2:31:04::","synset_id":"know.v.05"},{"index":6,"sense_key":"know%2:31:06::","synset_id":"acknowledge.v.06"},{"index":7,"sense_key":"know%2:31:15::","synset_id":"know.v.07"},{"index":8,"sense_key":"know%2:35:00::","synset_id":"sleep_together.v.01"},{"index":9,"sense_key":"know%2:31:11::","synset_id":"know.v.09"},{"index":10,"sense_key":"know%2:31:12::","synset_id":"know.v.10"},{"index":11,"sense_key":"know%2:31:14::","synset_id":"know.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"know%2:31:00::","call_ids":["senseval3.d000.s147.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"know%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1071,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0027619999999999997,"input_uncached_usd":0.002142,"input_cached_usd":0.0,"output_usd":0.00062,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7174270000541583}
{"item_id":"senseval3.d000.s150.t000","gold_sense_keys":["say%2:32:01::"],"candidates":[{"index":1,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":6,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":9,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":10,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":11,"sense_key":"say%2:32:09::","synset_id":"say.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"say%2:32:01::","call_ids":["senseval3.d000.s150.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"say%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":976,"cached_input_tokens":0,"output_tokens":89,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002842,"input_uncached_usd":0.001952,"input_cached_usd":0.0,"output_usd":0.0008900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1616176249226555}
{"item_id":"senseval3.d000.s150.t001","gold_sense_keys":["manage%2:41:09::"],"candidates":[{"index":1,"sense_key":"manage%2:41:09::","synset_id":"pull_off.v.03"},{"index":2,"sense_key":"manage%2:41:00::","synset_id":"manage.v.02"},{"index":3,"sense_key":"manage%2:41:03::","synset_id":"cope.v.01"},{"index":4,"sense_key":"manage%2:41:01::","synset_id":"oversee.v.01"},{"index":5,"sense_key":"manage%2:41:08::","synset_id":"wangle.v.01"},{"index":6,"sense_key":"manage%2:41:02::","synset_id":"do.v.11"},{"index":7,"sense_key":"manage%2:35:03::","synset_id":"wield.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"manage%2:41:09::","call_ids":["senseval3.d000.s150.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"manage%2:41:09::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":781,"cached_input_tokens":0,"output_tokens":137,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002932,"input_uncached_usd":0.001562,"input_cached_usd":0.0,"output_usd":0.0013700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.954435916035436}
{"item_id":"senseval3.d000.s151.t000","gold_sense_keys":["stranger%1:18:01::"],"candidates":[{"index":1,"sense_key":"stranger%1:18:00::","synset_id":"stranger.n.01"},{"index":2,"sense_key":"stranger%1:18:01::","synset_id":"stranger.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"stranger%1:18:01::","call_ids":["senseval3.d000.s151.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"stranger%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":406,"cached_input_tokens":0,"output_tokens":71,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001522,"input_uncached_usd":0.000812,"input_cached_usd":0.0,"output_usd":0.00071,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3176292499992996}
{"item_id":"senseval3.d000.s151.t001","gold_sense_keys":["hook%2:32:09::"],"candidates":[{"index":1,"sense_key":"hook%2:35:00::","synset_id":"hook.v.01"},{"index":2,"sense_key":"hook%2:40:02::","synset_id":"overcharge.v.01"},{"index":3,"sense_key":"hook%2:36:00::","synset_id":"crochet.v.02"},{"index":4,"sense_key":"hook%2:35:03::","synset_id":"hook.v.04"},{"index":5,"sense_key":"hook%2:40:01::","synset_id":"hook.v.05"},{"index":6,"sense_key":"hook%2:40:00::","synset_id":"pilfer.v.01"},{"index":7,"sense_key":"hook%2:35:02::","synset_id":"hook.v.07"},{"index":8,"sense_key":"hook%2:35:01::","synset_id":"hook.v.08"},{"index":9,"sense_key":"hook%2:34:00::","synset_id":"addict.v.01"},{"index":10,"sense_key":"hook%2:33:00::","synset_id":"hook.v.10"},{"index":11,"sense_key":"hook%2:32:09::","synset_id":"hook.v.11"},{"index":12,"sense_key":"hook%2:32:00::","synset_id":"hook.v.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"hook%2:32:09::","call_ids":["senseval3.d000.s151.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"hook%2:32:09::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":951,"cached_input_tokens":0,"output_tokens":123,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0031320000000000002,"input_uncached_usd":0.0019019999999999998,"input_cached_usd":0.0,"output_usd":0.0012300000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7428907919675112}
{"item_id":"senseval3.d000.s152.t000","gold_sense_keys":["eye%1:08:00::"],"candidates":[{"index":1,"sense_key":"eye%1:08:00::","synset_id":"eye.n.01"},{"index":2,"sense_key":"eye%1:09:00::","synset_id":"eye.n.02"},{"index":3,"sense_key":"eye%1:09:01::","synset_id":"eye.n.03"},{"index":4,"sense_key":"eye%1:15:00::","synset_id":"center.n.01"},{"index":5,"sense_key":"eye%1:06:00::","synset_id":"eye.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"eye%1:08:00::","call_ids":["senseval3.d000.s152.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"eye%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":583,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001286,"input_uncached_usd":0.001166,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.043751584016718}
{"item_id":"senseval3.d000.s152.t001","gold_sense_keys":["burn%2:43:01::"],"candidates":[{"index":1,"sense_key":"burn%2:30:02::","synset_id":"burn.v.01"},{"index":2,"sense_key":"burn%2:43:01::","synset_id":"burn.v.02"},{"index":3,"sense_key":"burn%2:30:00::","synset_id":"burn.v.03"},{"index":4,"sense_key":"burn%2:39:00::","synset_id":"bite.v.02"},{"index":5,"sense_key":"burn%2:43:02::","synset_id":"burn.v.05"},{"index":6,"sense_key":"burn%2:37:00::","synset_id":"burn.v.06"},{"index":7,"sense_key":"burn%2:30:01::","synset_id":"burn.v.07"},{"index":8,"sense_key":"burn%2:41:00::","synset_id":"burn.v.08"},{"index":9,"sense_key":"burn%2:40:01::","synset_id":"burn.v.09"},{"index":10,"sense_key":"burn%2:39:04::","synset_id":"burn.v.10"},{"index":11,"sense_key":"burn%2:30:08::","synset_id":"cauterize.v.01"},{"index":12,"sense_key":"burn%2:29:00::","synset_id":"sunburn.v.01"},{"index":13,"sense_key":"burn%2:36:05::","synset_id":"cut.v.21"},{"index":14,"sense_key":"burn%2:34:00::","synset_id":"burn_off.v.01"},{"index":15,"sense_key":"burn%2:30:03::","synset_id":"burn.v.15"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"burn%2:43:01::","call_ids":["senseval3.d000.s152.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"burn%2:43:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1142,"cached_input_tokens":0,"output_tokens":183,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0041140000000000005,"input_uncached_usd":0.002284,"input_cached_usd":0.0,"output_usd":0.0018300000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.20308137498796}
{"item_id":"senseval3.d000.s153.t000","gold_sense_keys":["mutter%2:32:01::"],"candidates":[{"index":1,"sense_key":"mutter%2:32:01::","synset_id":"mumble.v.01"},{"index":2,"sense_key":"mutter%2:32:00::","synset_id":"murmur.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"mutter%2:32:01::","call_ids":["senseval3.d000.s153.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"mutter%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":441,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0016120000000000002,"input_uncached_usd":0.000882,"input_cached_usd":0.0,"output_usd":0.0007300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4992602500133216}
{"item_id":"senseval3.d000.s154.t001","gold_sense_keys":["look%2:39:01::"],"candidates":[{"index":1,"sense_key":"look%2:39:00::","synset_id":"look.v.01"},{"index":2,"sense_key":"look%2:39:01::","synset_id":"look.v.02"},{"index":3,"sense_key":"look%2:29:00::","synset_id":"look.v.03"},{"index":4,"sense_key":"look%2:39:02::","synset_id":"search.v.02"},{"index":5,"sense_key":"look%2:42:00::","synset_id":"front.v.01"},{"index":6,"sense_key":"look%2:41:13::","synset_id":"attend.v.02"},{"index":7,"sense_key":"look%2:32:00::","synset_id":"look.v.07"},{"index":8,"sense_key":"look%2:31:00::","synset_id":"expect.v.03"},{"index":9,"sense_key":"look%2:42:02::","synset_id":"look.v.09"},{"index":10,"sense_key":"look%2:31:02::","synset_id":"count.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"look%2:39:01::","call_ids":["senseval3.d000.s154.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"look%2:39:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":936,"cached_input_tokens":0,"output_tokens":74,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002612,"input_uncached_usd":0.001872,"input_cached_usd":0.0,"output_usd":0.0007400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.697250833036378}
{"item_id":"senseval3.d000.s155.t001","gold_sense_keys":["way%1:04:01::","way%1:07:01::"],"candidates":[{"index":1,"sense_key":"way%1:07:01::","synset_id":"manner.n.01"},{"index":2,"sense_key":"way%1:04:01::","synset_id":"means.n.01"},{"index":3,"sense_key":"way%1:15:00::","synset_id":"direction.n.01"},{"index":4,"sense_key":"way%1:26:00::","synset_id":"way.n.04"},{"index":5,"sense_key":"way%1:04:00::","synset_id":"way.n.05"},{"index":6,"sense_key":"way%1:06:00::","synset_id":"way.n.06"},{"index":7,"sense_key":"way%1:04:02::","synset_id":"way.n.07"},{"index":8,"sense_key":"way%1:23:00::","synset_id":"room.n.02"},{"index":9,"sense_key":"way%1:07:02::","synset_id":"way.n.09"},{"index":10,"sense_key":"way%1:09:00::","synset_id":"way.n.10"},{"index":11,"sense_key":"way%1:09:01::","synset_id":"way.n.11"},{"index":12,"sense_key":"way%1:21:00::","synset_id":"way.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"way%1:07:01::","call_ids":["senseval3.d000.s155.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"way%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":942,"cached_input_tokens":0,"output_tokens":128,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003164,"input_uncached_usd":0.0018839999999999998,"input_cached_usd":0.0,"output_usd":0.00128,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.007745457929559}
{"item_id":"senseval3.d000.s155.t002","gold_sense_keys":["time%1:28:06::"],"candidates":[{"index":1,"sense_key":"time%1:11:00::","synset_id":"time.n.01"},{"index":2,"sense_key":"time%1:28:05::","synset_id":"time.n.02"},{"index":3,"sense_key":"time%1:28:00::","synset_id":"time.n.03"},{"index":4,"sense_key":"time%1:28:06::","synset_id":"time.n.04"},{"index":5,"sense_key":"time%1:03:00::","synset_id":"time.n.05"},{"index":6,"sense_key":"time%1:11:01::","synset_id":"time.n.06"},{"index":7,"sense_key":"time%1:28:03::","synset_id":"clock_time.n.01"},{"index":8,"sense_key":"time%1:28:04::","synset_id":"fourth_dimension.n.01"},{"index":9,"sense_key":"time%1:07:01::","synset_id":"meter.n.04"},{"index":10,"sense_key":"time%1:28:01::","synset_id":"prison_term.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"time%1:28:00::","call_ids":["senseval3.d000.s155.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"time%1:28:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":894,"cached_input_tokens":0,"output_tokens":139,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0031780000000000003,"input_uncached_usd":0.0017879999999999999,"input_cached_usd":0.0,"output_usd":0.0013900000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.751240042038262}
{"item_id":"senseval3.d000.s155.t004","gold_sense_keys":["have%2:40:00::"],"candidates":[{"index":1,"sense_key":"have%2:40:00::","synset_id":"have.v.01"},{"index":2,"sense_key":"have%2:42:00::","synset_id":"have.v.02"},{"index":3,"sense_key":"have%2:39:06::","synset_id":"experience.v.03"},{"index":4,"sense_key":"have%2:40:04::","synset_id":"own.v.01"},{"index":5,"sense_key":"have%2:30:00::","synset_id":"get.v.03"},{"index":6,"sense_key":"have%2:34:00::","synset_id":"consume.v.02"},{"index":7,"sense_key":"have%2:41:00::","synset_id":"have.v.07"},{"index":8,"sense_key":"have%2:36:00::","synset_id":"hold.v.03"},{"index":9,"sense_key":"have%2:40:01::","synset_id":"have.v.09"},{"index":10,"sense_key":"have%2:42:12::","synset_id":"have.v.10"},{"index":11,"sense_key":"have%2:30:01::","synset_id":"have.v.11"},{"index":12,"sense_key":"have%2:29:02::","synset_id":"have.v.12"},{"index":13,"sense_key":"have%2:32:00::","synset_id":"induce.v.02"},{"index":14,"sense_key":"have%2:40:05::","synset_id":"accept.v.02"},{"index":15,"sense_key":"have%2:40:03::","synset_id":"receive.v.01"},{"index":16,"sense_key":"have%2:29:05::","synset_id":"suffer.v.02"},{"index":17,"sense_key":"have%2:40:02::","synset_id":"have.v.17"},{"index":18,"sense_key":"have%2:29:00::","synset_id":"give_birth.v.01"},{"index":19,"sense_key":"have%2:35:00::","synset_id":"take.v.35"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"have%2:40:00::","call_ids":["senseval3.d000.s155.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"have%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1492,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0035139999999999998,"input_uncached_usd":0.0029839999999999997,"input_cached_usd":0.0,"output_usd":0.0005300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4167757909744978}
{"item_id":"senseval3.d000.s155.t005","gold_sense_keys":["alibi%1:10:00::"],"candidates":[{"index":1,"sense_key":"alibi%1:10:00::","synset_id":"alibi.n.01"},{"index":2,"sense_key":"alibi%1:10:01::","synset_id":"excuse.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"alibi%1:10:00::","call_ids":["senseval3.d000.s155.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"alibi%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":477,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0016740000000000001,"input_uncached_usd":0.000954,"input_cached_usd":0.0,"output_usd":0.00072,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3734166659414768}
{"item_id":"senseval3.d000.s156.t000","gold_sense_keys":["let%2:32:02::","let%2:41:00::"],"candidates":[{"index":1,"sense_key":"let%2:41:00::","synset_id":"let.v.01"},{"index":2,"sense_key":"let%2:32:00::","synset_id":"let.v.02"},{"index":3,"sense_key":"let%2:32:02::","synset_id":"permit.v.01"},{"index":4,"sense_key":"let%2:30:01::","synset_id":"get.v.03"},{"index":5,"sense_key":"let%2:30:00::","synset_id":"let.v.05"},{"index":6,"sense_key":"let%2:40:00::","synset_id":"lease.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"let%2:32:02::","call_ids":["senseval3.d000.s156.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"let%2:32:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":696,"cached_input_tokens":0,"output_tokens":128,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002672,"input_uncached_usd":0.001392,"input_cached_usd":0.0,"output_usd":0.00128,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.680265709059313}
{"item_id":"senseval3.d000.s156.t001","gold_sense_keys":["prove%2:31:00::"],"candidates":[{"index":1,"sense_key":"prove%2:42:00::","synset_id":"prove.v.01"},{"index":2,"sense_key":"prove%2:31:00::","synset_id":"prove.v.02"},{"index":3,"sense_key":"prove%2:32:00::","synset_id":"testify.v.02"},{"index":4,"sense_key":"prove%2:31:02::","synset_id":"prove.v.04"},{"index":5,"sense_key":"prove%2:41:03::","synset_id":"test.v.01"},{"index":6,"sense_key":"prove%2:38:01::","synset_id":"rise.v.15"},{"index":7,"sense_key":"prove%2:38:00::","synset_id":"raise.v.15"},{"index":8,"sense_key":"prove%2:36:00::","synset_id":"prove.v.08"},{"index":9,"sense_key":"prove%2:31:01::","synset_id":"prove.v.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"prove%2:31:00::","call_ids":["senseval3.d000.s156.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"prove%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":862,"cached_input_tokens":0,"output_tokens":95,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002674,"input_uncached_usd":0.0017239999999999998,"input_cached_usd":0.0,"output_usd":0.0009500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8023552499944344}
{"item_id":"senseval3.d000.s157.t000","gold_sense_keys":["think%2:31:03::"],"candidates":[{"index":1,"sense_key":"think%2:31:01::","synset_id":"think.v.01"},{"index":2,"sense_key":"think%2:31:03::","synset_id":"think.v.02"},{"index":3,"sense_key":"think%2:31:00::","synset_id":"think.v.03"},{"index":4,"sense_key":"think%2:31:02::","synset_id":"remember.v.01"},{"index":5,"sense_key":"think%2:36:00::","synset_id":"think.v.05"},{"index":6,"sense_key":"think%2:31:09::","synset_id":"think.v.06"},{"index":7,"sense_key":"think%2:31:06::","synset_id":"intend.v.01"},{"index":8,"sense_key":"think%2:31:12::","synset_id":"think.v.08"},{"index":9,"sense_key":"think%2:31:13::","synset_id":"think.v.09"},{"index":10,"sense_key":"think%2:31:15::","synset_id":"think.v.10"},{"index":11,"sense_key":"think%2:31:14::","synset_id":"think.v.11"},{"index":12,"sense_key":"think%2:31:08::","synset_id":"think.v.12"},{"index":13,"sense_key":"think%2:30:00::","synset_id":"think.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"think%2:31:03::","call_ids":["senseval3.d000.s157.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"think%2:31:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1116,"cached_input_tokens":0,"output_tokens":172,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003952000000000001,"input_uncached_usd":0.002232,"input_cached_usd":0.0,"output_usd":0.0017200000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.262418458936736}
{"item_id":"senseval3.d000.s157.t002","gold_sense_keys":["favor%1:04:00::"],"candidates":[{"index":1,"sense_key":"favor%1:04:00::","synset_id":"favor.n.01"},{"index":2,"sense_key":"favor%1:07:00::","synset_id":"favor.n.02"},{"index":3,"sense_key":"favor%1:09:00::","synset_id":"favor.n.03"},{"index":4,"sense_key":"favor%1:12:00::","synset_id":"favor.n.04"},{"index":5,"sense_key":"favor%1:06:00::","synset_id":"party_favor.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"favor%1:04:00::","call_ids":["senseval3.d000.s157.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"favor%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":571,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001262,"input_uncached_usd":0.001142,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.55842254194431}
{"item_id":"senseval3.d000.s158.t000","gold_sense_keys":["wait%2:42:00::"],"candidates":[{"index":1,"sense_key":"wait%2:42:00::","synset_id":"wait.v.01"},{"index":2,"sense_key":"wait%2:42:01::","synset_id":"wait.v.02"},{"index":3,"sense_key":"wait%2:31:00::","synset_id":"expect.v.03"},{"index":4,"sense_key":"wait%2:41:00::","synset_id":"wait.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"wait%2:42:00::","call_ids":["senseval3.d000.s158.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"wait%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":597,"cached_input_tokens":0,"output_tokens":94,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002134,"input_uncached_usd":0.001194,"input_cached_usd":0.0,"output_usd":0.0009400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4918693749932572}
{"item_id":"senseval3.d000.s159.t001","gold_sense_keys":["liquor%1:13:00::"],"candidates":[{"index":1,"sense_key":"liquor%1:13:00::","synset_id":"liquor.n.01"},{"index":2,"sense_key":"liquor%1:27:00::","synset_id":"liquor.n.02"},{"index":3,"sense_key":"liquor%1:13:01::","synset_id":"liquor.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"liquor%1:13:00::","call_ids":["senseval3.d000.s159.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"liquor%1:13:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":521,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001162,"input_uncached_usd":0.001042,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4258759579388425}
{"item_id":"senseval3.d000.s159.t002","gold_sense_keys":["bottom%1:15:01::"],"candidates":[{"index":1,"sense_key":"bottom%1:15:00::","synset_id":"bottom.n.01"},{"index":2,"sense_key":"bottom%1:15:01::","synset_id":"bottom.n.02"},{"index":3,"sense_key":"bottom%1:08:00::","synset_id":"buttocks.n.01"},{"index":4,"sense_key":"bottom%1:28:00::","synset_id":"bottom.n.04"},{"index":5,"sense_key":"bottom%1:17:00::","synset_id":"bed.n.03"},{"index":6,"sense_key":"bottom%1:17:01::","synset_id":"bottomland.n.01"},{"index":7,"sense_key":"bottom%1:06:00::","synset_id":"bottom.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bottom%1:15:01::","call_ids":["senseval3.d000.s159.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bottom%1:15:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":736,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002062,"input_uncached_usd":0.001472,"input_cached_usd":0.0,"output_usd":0.00059,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.34130070800893}
{"item_id":"senseval3.d000.s159.t003","gold_sense_keys":["glass%1:06:00::"],"candidates":[{"index":1,"sense_key":"glass%1:27:00::","synset_id":"glass.n.01"},{"index":2,"sense_key":"glass%1:06:00::","synset_id":"glass.n.02"},{"index":3,"sense_key":"glass%1:23:00::","synset_id":"glass.n.03"},{"index":4,"sense_key":"glass%1:06:03::","synset_id":"field_glass.n.01"},{"index":5,"sense_key":"glass%1:06:01::","synset_id":"methamphetamine.n.01"},{"index":6,"sense_key":"glass%1:06:02::","synset_id":"looking_glass.n.01"},{"index":7,"sense_key":"glass%1:06:04::","synset_id":"glass.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"glass%1:06:00::","call_ids":["senseval3.d000.s159.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"glass%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":704,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001528,"input_uncached_usd":0.001408,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0927203750470653}
{"item_id":"senseval3.d001.s000.t000","gold_sense_keys":["rout%1:11:00::"],"candidates":[{"index":1,"sense_key":"rout%1:14:00::","synset_id":"mob.n.01"},{"index":2,"sense_key":"rout%1:11:00::","synset_id":"rout.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"rout%1:11:00::","call_ids":["senseval3.d001.s000.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"rout%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":452,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001024,"input_uncached_usd":0.000904,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1120107090100646}
{"item_id":"senseval3.d001.s000.t004","gold_sense_keys":["back%2:41:00::"],"candidates":[{"index":1,"sense_key":"back%2:41:00::","synset_id":"back.v.01"},{"index":2,"sense_key":"back%2:38:00::","synset_id":"back.v.02"},{"index":3,"sense_key":"back%2:41:01::","synset_id":"second.v.01"},{"index":4,"sense_key":"back%2:38:01::","synset_id":"back.v.04"},{"index":5,"sense_key":"back%2:40:00::","synset_id":"back.v.05"},{"index":6,"sense_key":"back%2:42:00::","synset_id":"back.v.06"},{"index":7,"sense_key":"back%2:33:00::","synset_id":"bet_on.v.01"},{"index":8,"sense_key":"back%2:30:03::","synset_id":"back.v.08"},{"index":9,"sense_key":"back%2:30:00::","synset_id":"back.v.09"},{"index":10,"sense_key":"back%2:30:01::","synset_id":"back.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"back%2:41:00::","call_ids":["senseval3.d001.s000.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"back%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":878,"cached_input_tokens":0,"output_tokens":134,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003096,"input_uncached_usd":0.001756,"input_cached_usd":0.0,"output_usd":0.00134,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7795295000541955}
{"item_id":"senseval3.d001.s000.t005","gold_sense_keys":["democratic%3:01:00::"],"candidates":[{"index":1,"sense_key":"democratic%3:00:00::","synset_id":"democratic.a.01"},{"index":2,"sense_key":"democratic%3:01:00::","synset_id":"democratic.a.02"},{"index":3,"sense_key":"democratic%5:00:00:common:01","synset_id":"democratic.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"democratic%3:01:00::","call_ids":["senseval3.d001.s000.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"democratic%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":574,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001268,"input_uncached_usd":0.001148,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.157849541050382}
{"item_id":"senseval3.d001.s000.t006","gold_sense_keys":["presidential%3:01:00::"],"candidates":[{"index":1,"sense_key":"presidential%3:01:00::","synset_id":"presidential.a.01"},{"index":2,"sense_key":"presidential%3:00:00::","synset_id":"presidential.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"presidential%3:01:00::","call_ids":["senseval3.d001.s000.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"presidential%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":472,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001064,"input_uncached_usd":0.000944,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9977067919680849}
{"item_id":"senseval3.d001.s000.t007","gold_sense_keys":["candidate%1:18:01::"],"candidates":[{"index":1,"sense_key":"candidate%1:18:01::","synset_id":"campaigner.n.01"},{"index":2,"sense_key":"candidate%1:18:00::","synset_id":"candidate.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"candidate%1:18:01::","call_ids":["senseval3.d001.s000.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"candidate%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":479,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001078,"input_uncached_usd":0.000958,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1335170000093058}
{"item_id":"senseval3.d001.s000.t008","gold_sense_keys":["reminder%1:09:00::"],"candidates":[{"index":1,"sense_key":"reminder%1:10:00::","synset_id":"reminder.n.01"},{"index":2,"sense_key":"reminder%1:09:00::","synset_id":"reminder.n.02"},{"index":3,"sense_key":"reminder%1:18:00::","synset_id":"admonisher.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"reminder%1:09:00::","call_ids":["senseval3.d001.s000.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"reminder%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":516,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001152,"input_uncached_usd":0.001032,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3629660409642383}
{"item_id":"senseval3.d001.s000.t009","gold_sense_keys":["federal%5:00:00:national:01"],"candidates":[{"index":1,"sense_key":"federal%5:00:00:national:01","synset_id":"federal.s.01"},{"index":2,"sense_key":"federal%3:01:02::","synset_id":"federal.a.02"},{"index":3,"sense_key":"federal%5:00:00:northern:02","synset_id":"union.s.01"},{"index":4,"sense_key":"federal%3:00:00::","synset_id":"federal.a.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"federal%5:00:00:national:01","call_ids":["senseval3.d001.s000.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"federal%5:00:00:national:01","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":659,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002088,"input_uncached_usd":0.001318,"input_cached_usd":0.0,"output_usd":0.0007700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5410621670307592}
{"item_id":"senseval3.d001.s000.t010","gold_sense_keys":["level%1:26:01::"],"candidates":[{"index":1,"sense_key":"level%1:07:00::","synset_id":"degree.n.01"},{"index":2,"sense_key":"level%1:26:01::","synset_id":"grade.n.02"},{"index":3,"sense_key":"level%1:26:00::","synset_id":"degree.n.02"},{"index":4,"sense_key":"level%1:07:01::","synset_id":"level.n.04"},{"index":5,"sense_key":"level%1:06:00::","synset_id":"level.n.05"},{"index":6,"sense_key":"level%1:06:03::","synset_id":"horizontal_surface.n.01"},{"index":7,"sense_key":"level%1:09:00::","synset_id":"level.n.07"},{"index":8,"sense_key":"level%1:06:01::","synset_id":"floor.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"level%1:26:01::","call_ids":["senseval3.d001.s000.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"level%1:26:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":898,"cached_input_tokens":0,"output_tokens":144,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003236,"input_uncached_usd":0.0017959999999999999,"input_cached_usd":0.0,"output_usd":0.00144,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.464503666968085}
{"item_id":"senseval3.d001.s000.t011","gold_sense_keys":["political%3:00:00::"],"candidates":[{"index":1,"sense_key":"political%3:00:00::","synset_id":"political.a.01"},{"index":2,"sense_key":"political%3:01:01::","synset_id":"political.a.02"},{"index":3,"sense_key":"political%3:01:02::","synset_id":"political.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"political%3:00:00::","call_ids":["senseval3.d001.s000.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"political%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":530,"cached_input_tokens":0,"output_tokens":104,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0021000000000000003,"input_uncached_usd":0.00106,"input_cached_usd":0.0,"output_usd":0.0010400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4804897499270737}
{"item_id":"senseval3.d001.s000.t013","gold_sense_keys":["past%3:00:00::"],"candidates":[{"index":1,"sense_key":"past%3:00:00::","synset_id":"past.a.01"},{"index":2,"sense_key":"past%5:00:00:outgoing:00","synset_id":"past.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"past%3:00:00::","call_ids":["senseval3.d001.s000.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"past%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":487,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0017439999999999999,"input_uncached_usd":0.0009739999999999999,"input_cached_usd":0.0,"output_usd":0.0007700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.451954166055657}
{"item_id":"senseval3.d001.s001.t000","gold_sense_keys":["presidential%3:01:00::"],"candidates":[{"index":1,"sense_key":"presidential%3:01:00::","synset_id":"presidential.a.01"},{"index":2,"sense_key":"presidential%3:00:00::","synset_id":"presidential.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"presidential%3:01:00::","call_ids":["senseval3.d001.s001.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"presidential%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":495,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00111,"input_uncached_usd":0.00099,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.123695750022307}
{"item_id":"senseval3.d001.s001.t001","gold_sense_keys":["election%1:04:01::"],"candidates":[{"index":1,"sense_key":"election%1:04:01::","synset_id":"election.n.01"},{"index":2,"sense_key":"election%1:04:00::","synset_id":"election.n.02"},{"index":3,"sense_key":"election%1:26:00::","synset_id":"election.n.03"},{"index":4,"sense_key":"election%1:09:00::","synset_id":"election.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"election%1:04:01::","call_ids":["senseval3.d001.s001.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"election%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":600,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00132,"input_uncached_usd":0.0012,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.049510084092617}
{"item_id":"senseval3.d001.s001.t002","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["senseval3.d001.s001.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":658,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001436,"input_uncached_usd":0.001316,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0617726249620318}
{"item_id":"senseval3.d001.s001.t004","gold_sense_keys":["nation%1:14:00::"],"candidates":[{"index":1,"sense_key":"nation%1:14:00::","synset_id":"state.n.04"},{"index":2,"sense_key":"nation%1:14:01::","synset_id":"nation.n.02"},{"index":3,"sense_key":"nation%1:18:00::","synset_id":"nation.n.03"},{"index":4,"sense_key":"nation%1:14:02::","synset_id":"nation.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"nation%1:14:00::","call_ids":["senseval3.d001.s001.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"nation%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":682,"cached_input_tokens":0,"output_tokens":108,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002444,"input_uncached_usd":0.001364,"input_cached_usd":0.0,"output_usd":0.00108,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8583331250119954}
{"item_id":"senseval3.d001.s001.t007","gold_sense_keys":["choose%2:31:00::"],"candidates":[{"index":1,"sense_key":"choose%2:31:00::","synset_id":"choose.v.01"},{"index":2,"sense_key":"choose%2:31:02::","synset_id":"choose.v.02"},{"index":3,"sense_key":"choose%2:31:01::","synset_id":"choose.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"choose%2:31:00::","call_ids":["senseval3.d001.s001.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"choose%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":620,"cached_input_tokens":0,"output_tokens":104,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00228,"input_uncached_usd":0.00124,"input_cached_usd":0.0,"output_usd":0.0010400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9465758330188692}
{"item_id":"senseval3.d001.s001.t009","gold_sense_keys":["party%1:14:01::"],"candidates":[{"index":1,"sense_key":"party%1:14:01::","synset_id":"party.n.01"},{"index":2,"sense_key":"party%1:14:00::","synset_id":"party.n.02"},{"index":3,"sense_key":"party%1:14:02::","synset_id":"party.n.03"},{"index":4,"sense_key":"party%1:11:00::","synset_id":"party.n.04"},{"index":5,"sense_key":"party%1:18:00::","synset_id":"party.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"party%1:14:01::","call_ids":["senseval3.d001.s001.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"party%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":681,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001482,"input_uncached_usd":0.001362,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1494705840013921}
{"item_id":"senseval3.d001.s001.t010","gold_sense_keys":["candidate%1:18:01::"],"candidates":[{"index":1,"sense_key":"candidate%1:18:01::","synset_id":"campaigner.n.01"},{"index":2,"sense_key":"candidate%1:18:00::","synset_id":"candidate.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"candidate%1:18:01::","call_ids":["senseval3.d001.s001.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"candidate%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":502,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001124,"input_uncached_usd":0.001004,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5254115830175579}
{"item_id":"senseval3.d001.s002.t000","gold_sense_keys":["percentage%1:24:00::"],"candidates":[{"index":1,"sense_key":"percentage%1:24:00::","synset_id":"percentage.n.01"},{"index":2,"sense_key":"percentage%1:21:00::","synset_id":"share.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"percentage%1:24:00::","call_ids":["senseval3.d001.s002.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"percentage%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":560,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00124,"input_uncached_usd":0.00112,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8833968749968335}
{"item_id":"senseval3.d001.s002.t001","gold_sense_keys":["equal%2:42:00::"],"candidates":[{"index":1,"sense_key":"equal%2:42:00::","synset_id":"equal.v.01"},{"index":2,"sense_key":"equal%2:42:01::","synset_id":"equal.v.02"},{"index":3,"sense_key":"equal%2:30:00::","synset_id":"equal.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"equal%2:42:00::","call_ids":["senseval3.d001.s002.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"equal%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":658,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001436,"input_uncached_usd":0.001316,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4764412080403417}
{"item_id":"senseval3.d001.s003.t000","gold_sense_keys":["know%2:31:01::"],"candidates":[{"index":1,"sense_key":"know%2:31:01::","synset_id":"know.v.01"},{"index":2,"sense_key":"know%2:31:03::","synset_id":"know.v.02"},{"index":3,"sense_key":"know%2:31:02::","synset_id":"know.v.03"},{"index":4,"sense_key":"know%2:31:00::","synset_id":"know.v.04"},{"index":5,"sense_key":"know%2:31:04::","synset_id":"know.v.05"},{"index":6,"sense_key":"know%2:31:06::","synset_id":"acknowledge.v.06"},{"index":7,"sense_key":"know%2:31:15::","synset_id":"know.v.07"},{"index":8,"sense_key":"know%2:35:00::","synset_id":"sleep_together.v.01"},{"index":9,"sense_key":"know%2:31:11::","synset_id":"know.v.09"},{"index":10,"sense_key":"know%2:31:12::","synset_id":"know.v.10"},{"index":11,"sense_key":"know%2:31:14::","synset_id":"know.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"know%2:31:01::","call_ids":["senseval3.d001.s003.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"know%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1220,"cached_input_tokens":0,"output_tokens":101,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00345,"input_uncached_usd":0.00244,"input_cached_usd":0.0,"output_usd":0.00101,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6722987500252202}
{"item_id":"senseval3.d001.s003.t002","gold_sense_keys":["tend%2:42:01::"],"candidates":[{"index":1,"sense_key":"tend%2:42:01::","synset_id":"tend.v.01"},{"index":2,"sense_key":"tend%2:41:00::","synset_id":"tend.v.02"},{"index":3,"sense_key":"tend%2:41:01::","synset_id":"tend.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"tend%2:42:01::","call_ids":["senseval3.d001.s003.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"tend%2:42:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":653,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001426,"input_uncached_usd":0.0013059999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1235547080868855}
{"item_id":"senseval3.d001.s003.t003","gold_sense_keys":["favor%2:41:00::"],"candidates":[{"index":1,"sense_key":"favor%2:41:00::","synset_id":"prefer.v.03"},{"index":2,"sense_key":"favor%2:31:00::","synset_id":"favor.v.02"},{"index":3,"sense_key":"favor%2:41:01::","synset_id":"favor.v.03"},{"index":4,"sense_key":"favor%2:41:03::","synset_id":"privilege.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"favor%2:41:00::","call_ids":["senseval3.d001.s003.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"favor%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":678,"cached_input_tokens":0,"output_tokens":120,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002556,"input_uncached_usd":0.001356,"input_cached_usd":0.0,"output_usd":0.0012000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.207281875074841}
{"item_id":"senseval3.d001.s003.t004","gold_sense_keys":["republican%1:18:01::"],"candidates":[{"index":1,"sense_key":"republican%1:18:01::","synset_id":"republican.n.01"},{"index":2,"sense_key":"republican%1:18:00::","synset_id":"republican.n.02"},{"index":3,"sense_key":"republican%1:17:00::","synset_id":"republican.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"republican%1:18:01::","call_ids":["senseval3.d001.s003.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"republican%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":645,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00141,"input_uncached_usd":0.00129,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0417049169773236}
{"item_id":"senseval3.d001.s003.t005","gold_sense_keys":["race%1:11:01::"],"candidates":[{"index":1,"sense_key":"race%1:11:01::","synset_id":"race.n.01"},{"index":2,"sense_key":"race%1:11:00::","synset_id":"race.n.02"},{"index":3,"sense_key":"race%1:14:00::","synset_id":"race.n.03"},{"index":4,"sense_key":"race%1:14:01::","synset_id":"subspecies.n.01"},{"index":5,"sense_key":"race%1:19:00::","synset_id":"slipstream.n.01"},{"index":6,"sense_key":"race%1:06:00::","synset_id":"raceway.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"race%1:11:01::","call_ids":["senseval3.d001.s003.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"race%1:11:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":804,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001728,"input_uncached_usd":0.0016079999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4861422500107437}
{"item_id":"senseval3.d001.s003.t006","gold_sense_keys":["president%1:04:04::"],"candidates":[{"index":1,"sense_key":"president%1:18:01::","synset_id":"president.n.01"},{"index":2,"sense_key":"president%1:18:04::","synset_id":"president_of_the_united_states.n.01"},{"index":3,"sense_key":"president%1:18:00::","synset_id":"president.n.03"},{"index":4,"sense_key":"president%1:18:03::","synset_id":"president.n.04"},{"index":5,"sense_key":"president%1:18:02::","synset_id":"president.n.05"},{"index":6,"sense_key":"president%1:04:04::","synset_id":"president_of_the_united_states.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"president%1:04:04::","call_ids":["senseval3.d001.s003.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"president%1:04:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":811,"cached_input_tokens":0,"output_tokens":96,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002582,"input_uncached_usd":0.001622,"input_cached_usd":0.0,"output_usd":0.0009600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7482735000085086}
{"item_id":"senseval3.d001.s004.t000","gold_sense_keys":["presidential%3:01:00::"],"candidates":[{"index":1,"sense_key":"presidential%3:01:00::","synset_id":"presidential.a.01"},{"index":2,"sense_key":"presidential%3:00:00::","synset_id":"presidential.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"presidential%3:01:00::","call_ids":["senseval3.d001.s004.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"presidential%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":616,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0013520000000000001,"input_uncached_usd":0.001232,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9241427910747007}
{"item_id":"senseval3.d001.s004.t001","gold_sense_keys":["election%1:04:01::"],"candidates":[{"index":1,"sense_key":"election%1:04:01::","synset_id":"election.n.01"},{"index":2,"sense_key":"election%1:04:00::","synset_id":"election.n.02"},{"index":3,"sense_key":"election%1:26:00::","synset_id":"election.n.03"},{"index":4,"sense_key":"election%1:09:00::","synset_id":"election.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"election%1:04:01::","call_ids":["senseval3.d001.s004.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"election%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":721,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001562,"input_uncached_usd":0.001442,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8894275419879705}
{"item_id":"senseval3.d001.s004.t002","gold_sense_keys":["past%3:00:00::"],"candidates":[{"index":1,"sense_key":"past%3:00:00::","synset_id":"past.a.01"},{"index":2,"sense_key":"past%5:00:00:outgoing:00","synset_id":"past.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"past%3:00:00::","call_ids":["senseval3.d001.s004.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"past%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":631,"cached_input_tokens":0,"output_tokens":78,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002042,"input_uncached_usd":0.0012619999999999999,"input_cached_usd":0.0,"output_usd":0.0007800000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5421395000303164}
{"item_id":"senseval3.d001.s004.t004","gold_sense_keys":["presidential%3:01:00::"],"candidates":[{"index":1,"sense_key":"presidential%3:01:00::","synset_id":"presidential.a.01"},{"index":2,"sense_key":"presidential%3:00:00::","synset_id":"presidential.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"presidential%3:01:00::","call_ids":["senseval3.d001.s004.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"presidential%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":616,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0013520000000000001,"input_uncached_usd":0.001232,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.503653583000414}
{"item_id":"senseval3.d001.s004.t007","gold_sense_keys":["percentage%1:24:00::"],"candidates":[{"index":1,"sense_key":"percentage%1:24:00::","synset_id":"percentage.n.01"},{"index":2,"sense_key":"percentage%1:21:00::","synset_id":"share.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"percentage%1:24:00::","call_ids":["senseval3.d001.s004.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"percentage%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":653,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001426,"input_uncached_usd":0.0013059999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.7807992079760879}
{"item_id":"senseval3.d001.s004.t008","gold_sense_keys":["major%3:00:01::","major%3:00:06::"],"candidates":[{"index":1,"sense_key":"major%3:00:06::","synset_id":"major.a.01"},{"index":2,"sense_key":"major%3:00:02::","synset_id":"major.a.02"},{"index":3,"sense_key":"major%3:00:01::","synset_id":"major.a.03"},{"index":4,"sense_key":"major%3:00:05::","synset_id":"major.a.04"},{"index":5,"sense_key":"major%3:00:04::","synset_id":"major.a.05"},{"index":6,"sense_key":"major%3:00:07::","synset_id":"major.a.06"},{"index":7,"sense_key":"major%3:00:03::","synset_id":"major.a.07"},{"index":8,"sense_key":"major%5:00:00:senior:00","synset_id":"major.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"major%3:00:06::","call_ids":["senseval3.d001.s004.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"major%3:00:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":872,"cached_input_tokens":0,"output_tokens":104,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002784,"input_uncached_usd":0.0017439999999999999,"input_cached_usd":0.0,"output_usd":0.0010400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6143058750312775}
{"item_id":"senseval3.d001.s004.t009","gold_sense_keys":["party%1:14:01::"],"candidates":[{"index":1,"sense_key":"party%1:14:01::","synset_id":"party.n.01"},{"index":2,"sense_key":"party%1:14:00::","synset_id":"party.n.02"},{"index":3,"sense_key":"party%1:14:02::","synset_id":"party.n.03"},{"index":4,"sense_key":"party%1:11:00::","synset_id":"party.n.04"},{"index":5,"sense_key":"party%1:18:00::","synset_id":"party.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"party%1:14:01::","call_ids":["senseval3.d001.s004.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"party%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":801,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001722,"input_uncached_usd":0.001602,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.516278208931908}
{"item_id":"senseval3.d001.s004.t010","gold_sense_keys":["popular%5:00:00:democratic:00"],"candidates":[{"index":1,"sense_key":"popular%3:00:00::","synset_id":"popular.a.01"},{"index":2,"sense_key":"popular%5:00:00:democratic:00","synset_id":"popular.s.01"},{"index":3,"sense_key":"popular%5:00:01:common:01","synset_id":"democratic.s.01"},{"index":4,"sense_key":"popular%5:00:00:nonclassical:00","synset_id":"popular.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"popular%5:00:00:democratic:00","call_ids":["senseval3.d001.s004.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"popular%5:00:00:democratic:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":771,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001662,"input_uncached_usd":0.001542,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0174082500161603}
{"item_id":"senseval3.d001.s004.t011","gold_sense_keys":["vote%1:09:00::"],"candidates":[{"index":1,"sense_key":"vote%1:04:00::","synset_id":"vote.n.01"},{"index":2,"sense_key":"vote%1:04:01::","synset_id":"vote.n.02"},{"index":3,"sense_key":"vote%1:07:00::","synset_id":"right_to_vote.n.01"},{"index":4,"sense_key":"vote%1:14:00::","synset_id":"vote.n.04"},{"index":5,"sense_key":"vote%1:09:00::","synset_id":"vote.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"vote%1:09:00::","call_ids":["senseval3.d001.s004.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"vote%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":850,"cached_input_tokens":0,"output_tokens":127,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00297,"input_uncached_usd":0.0017,"input_cached_usd":0.0,"output_usd":0.00127,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.9508862500078976}
{"item_id":"senseval3.d001.s004.t012","gold_sense_keys":["president%1:04:04::"],"candidates":[{"index":1,"sense_key":"president%1:18:01::","synset_id":"president.n.01"},{"index":2,"sense_key":"president%1:18:04::","synset_id":"president_of_the_united_states.n.01"},{"index":3,"sense_key":"president%1:18:00::","synset_id":"president.n.03"},{"index":4,"sense_key":"president%1:18:03::","synset_id":"president.n.04"},{"index":5,"sense_key":"president%1:18:02::","synset_id":"president.n.05"},{"index":6,"sense_key":"president%1:04:04::","synset_id":"president_of_the_united_states.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"president%1:04:04::","call_ids":["senseval3.d001.s004.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"president%1:04:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":832,"cached_input_tokens":0,"output_tokens":116,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002824,"input_uncached_usd":0.001664,"input_cached_usd":0.0,"output_usd":0.00116,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":7.890476332977414}
{"item_id":"senseval3.d001.s004.t014","gold_sense_keys":["seat%1:07:00::"],"candidates":[{"index":1,"sense_key":"seat%1:15:01::","synset_id":"seat.n.01"},{"index":2,"sense_key":"seat%1:08:00::","synset_id":"buttocks.n.01"},{"index":3,"sense_key":"seat%1:06:01::","synset_id":"seat.n.03"},{"index":4,"sense_key":"seat%1:06:00::","synset_id":"seat.n.04"},{"index":5,"sense_key":"seat%1:15:00::","synset_id":"seat.n.05"},{"index":6,"sense_key":"seat%1:15:02::","synset_id":"seat.n.06"},{"index":7,"sense_key":"seat%1:07:00::","synset_id":"seat.n.07"},{"index":8,"sense_key":"seat%1:06:03::","synset_id":"seat.n.08"},{"index":9,"sense_key":"seat%1:06:02::","synset_id":"seat.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"seat%1:07:00::","call_ids":["senseval3.d001.s004.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"seat%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1048,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0022159999999999997,"input_uncached_usd":0.0020959999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.504184458986856}
{"item_id":"senseval3.d001.s004.t015","gold_sense_keys":["popular%5:00:00:democratic:00"],"candidates":[{"index":1,"sense_key":"popular%3:00:00::","synset_id":"popular.a.01"},{"index":2,"sense_key":"popular%5:00:00:democratic:00","synset_id":"popular.s.01"},{"index":3,"sense_key":"popular%5:00:01:common:01","synset_id":"democratic.s.01"},{"index":4,"sense_key":"popular%5:00:00:nonclassical:00","synset_id":"popular.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"popular%5:00:00:democratic:00","call_ids":["senseval3.d001.s004.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"popular%5:00:00:democratic:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":771,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001662,"input_uncached_usd":0.001542,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0528821249026805}
{"item_id":"senseval3.d001.s004.t016","gold_sense_keys":["vote%1:09:00::"],"candidates":[{"index":1,"sense_key":"vote%1:04:00::","synset_id":"vote.n.01"},{"index":2,"sense_key":"vote%1:04:01::","synset_id":"vote.n.02"},{"index":3,"sense_key":"vote%1:07:00::","synset_id":"right_to_vote.n.01"},{"index":4,"sense_key":"vote%1:14:00::","synset_id":"vote.n.04"},{"index":5,"sense_key":"vote%1:09:00::","synset_id":"vote.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"vote%1:09:00::","call_ids":["senseval3.d001.s004.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"vote%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":850,"cached_input_tokens":0,"output_tokens":129,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00299,"input_uncached_usd":0.0017,"input_cached_usd":0.0,"output_usd":0.0012900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2016909579979256}
{"item_id":"senseval3.d001.s005.t001","gold_sense_keys":["opposite%1:24:03::"],"candidates":[{"index":1,"sense_key":"opposite%1:10:00::","synset_id":"antonym.n.01"},{"index":2,"sense_key":"opposite%1:24:03::","synset_id":"reverse.n.01"},{"index":3,"sense_key":"opposite%1:18:00::","synset_id":"opposition.n.04"},{"index":4,"sense_key":"opposite%1:24:04::","synset_id":"inverse.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"opposite%1:24:03::","call_ids":["senseval3.d001.s005.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"opposite%1:24:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":809,"cached_input_tokens":0,"output_tokens":128,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002898,"input_uncached_usd":0.0016179999999999999,"input_cached_usd":0.0,"output_usd":0.00128,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6081070000072941}
{"item_id":"senseval3.d001.s006.t000","gold_sense_keys":["account_for%2:42:00::"],"candidates":[{"index":1,"sense_key":"account_for%2:42:00::","synset_id":"account_for.v.01"},{"index":2,"sense_key":"account_for%2:32:00::","synset_id":"account_for.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"account_for%2:42:00::","call_ids":["senseval3.d001.s006.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"account_for%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":573,"cached_input_tokens":0,"output_tokens":116,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002306,"input_uncached_usd":0.0011459999999999999,"input_cached_usd":0.0,"output_usd":0.00116,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0507420420181006}
{"item_id":"senseval3.d001.s006.t001","gold_sense_keys":["result%1:11:00::"],"candidates":[{"index":1,"sense_key":"result%1:19:00::","synset_id":"consequence.n.01"},{"index":2,"sense_key":"result%1:10:00::","synset_id":"solution.n.02"},{"index":3,"sense_key":"result%1:11:00::","synset_id":"result.n.03"},{"index":4,"sense_key":"result%1:10:01::","synset_id":"resultant_role.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"result%1:19:00::","call_ids":["senseval3.d001.s006.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"result%1:19:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":795,"cached_input_tokens":0,"output_tokens":150,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00309,"input_uncached_usd":0.0015899999999999998,"input_cached_usd":0.0,"output_usd":0.0015,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0188678330741823}
{"item_id":"senseval3.d001.s006.t002","gold_sense_keys":["recent%5:00:00:past:00"],"candidates":[{"index":1,"sense_key":"recent%5:00:00:new:00","synset_id":"recent.s.01"},{"index":2,"sense_key":"recent%5:00:00:past:00","synset_id":"late.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"recent%5:00:00:past:00","call_ids":["senseval3.d001.s006.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"recent%5:00:00:past:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":583,"cached_input_tokens":0,"output_tokens":102,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002186,"input_uncached_usd":0.001166,"input_cached_usd":0.0,"output_usd":0.00102,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4728686669841409}
{"item_id":"senseval3.d001.s006.t003","gold_sense_keys":["decade%1:28:00::"],"candidates":[{"index":1,"sense_key":"decade%1:28:00::","synset_id":"decade.n.01"},{"index":2,"sense_key":"decade%1:23:00::","synset_id":"ten.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"decade%1:28:00::","call_ids":["senseval3.d001.s006.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"decade%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":585,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0012900000000000001,"input_uncached_usd":0.00117,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9799425420351326}
{"item_id":"senseval3.d001.s007.t000","gold_sense_keys":["simple%3:00:02::","simple%5:00:00:easy:01"],"candidates":[{"index":1,"sense_key":"simple%3:00:02::","synset_id":"simple.a.01"},{"index":2,"sense_key":"simple%5:00:00:easy:01","synset_id":"elementary.s.01"},{"index":3,"sense_key":"simple%5:00:02:plain:01","synset_id":"bare.s.04"},{"index":4,"sense_key":"simple%5:00:00:naive:00","synset_id":"childlike.s.02"},{"index":5,"sense_key":"simple%5:00:00:retarded:00","synset_id":"dim-witted.s.01"},{"index":6,"sense_key":"simple%3:00:01::","synset_id":"simple.a.06"},{"index":7,"sense_key":"simple%5:00:00:plain:01","synset_id":"simple.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"simple%3:00:02::","call_ids":["senseval3.d001.s007.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"simple%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":911,"cached_input_tokens":0,"output_tokens":104,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002862,"input_uncached_usd":0.0018219999999999998,"input_cached_usd":0.0,"output_usd":0.0010400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.071308375103399}
{"item_id":"senseval3.d001.s007.t004","gold_sense_keys":["partial%5:00:00:incomplete:00"],"candidates":[{"index":1,"sense_key":"partial%5:00:00:incomplete:00","synset_id":"partial.s.01"},{"index":2,"sense_key":"partial%3:00:01::","synset_id":"partial.a.02"},{"index":3,"sense_key":"partial%5:00:00:inclined:02","synset_id":"fond.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"partial%5:00:00:incomplete:00","call_ids":["senseval3.d001.s007.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"partial%5:00:00:incomplete:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":598,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0013160000000000001,"input_uncached_usd":0.001196,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1960026660235599}
{"item_id":"senseval3.d001.s007.t007","gold_sense_keys":["display%2:39:00::"],"candidates":[{"index":1,"sense_key":"display%2:39:00::","synset_id":"expose.v.03"},{"index":2,"sense_key":"display%2:41:00::","synset_id":"display.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"display%2:39:00::","call_ids":["senseval3.d001.s007.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"display%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":569,"cached_input_tokens":0,"output_tokens":80,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001938,"input_uncached_usd":0.001138,"input_cached_usd":0.0,"output_usd":0.0008,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4120881659910083}
{"item_id":"senseval3.d001.s007.t008","gold_sense_keys":["american%1:18:00::"],"candidates":[{"index":1,"sense_key":"american%1:18:00::","synset_id":"american.n.01"},{"index":2,"sense_key":"american%1:10:00::","synset_id":"american_english.n.01"},{"index":3,"sense_key":"american%1:18:01::","synset_id":"american.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"american%1:18:00::","call_ids":["senseval3.d001.s007.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"american%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":552,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001224,"input_uncached_usd":0.001104,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9792301659472287}
{"item_id":"senseval3.d001.s008.t002","gold_sense_keys":["assumption%1:09:00::","assumption%1:10:00::"],"candidates":[{"index":1,"sense_key":"assumption%1:10:00::","synset_id":"premise.n.01"},{"index":2,"sense_key":"assumption%1:09:00::","synset_id":"assumption.n.02"},{"index":3,"sense_key":"assumption%1:04:00::","synset_id":"assumption.n.03"},{"index":4,"sense_key":"assumption%1:28:00::","synset_id":"assumption.n.04"},{"index":5,"sense_key":"assumption%1:11:00::","synset_id":"assumption.n.05"},{"index":6,"sense_key":"assumption%1:07:00::","synset_id":"presumption.n.03"},{"index":7,"sense_key":"assumption%1:04:01::","synset_id":"assumption.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"assumption%1:10:00::","call_ids":["senseval3.d001.s008.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"assumption%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1050,"cached_input_tokens":0,"output_tokens":245,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00455,"input_uncached_usd":0.0021,"input_cached_usd":0.0,"output_usd":0.0024500000000000004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.5082455000374466}
{"item_id":"senseval3.d001.s009.t002","gold_sense_keys":["brand%1:09:00::"],"candidates":[{"index":1,"sense_key":"brand%1:10:00::","synset_id":"trade_name.n.01"},{"index":2,"sense_key":"brand%1:09:00::","synset_id":"brand.n.02"},{"index":3,"sense_key":"brand%1:10:02::","synset_id":"brand.n.03"},{"index":4,"sense_key":"brand%1:27:00::","synset_id":"brand.n.04"},{"index":5,"sense_key":"brand%1:10:01::","synset_id":"mark.n.06"},{"index":6,"sense_key":"brand%1:06:00::","synset_id":"sword.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"brand%1:09:00::","call_ids":["senseval3.d001.s009.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"brand%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":936,"cached_input_tokens":0,"output_tokens":152,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003392,"input_uncached_usd":0.001872,"input_cached_usd":0.0,"output_usd":0.00152,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1865862500853837}
{"item_id":"senseval3.d001.s009.t004","gold_sense_keys":["political%3:00:00::","political%3:01:02::"],"candidates":[{"index":1,"sense_key":"political%3:00:00::","synset_id":"political.a.01"},{"index":2,"sense_key":"political%3:01:01::","synset_id":"political.a.02"},{"index":3,"sense_key":"political%3:01:02::","synset_id":"political.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"political%3:00:00::","call_ids":["senseval3.d001.s009.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"political%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":759,"cached_input_tokens":0,"output_tokens":129,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002808,"input_uncached_usd":0.001518,"input_cached_usd":0.0,"output_usd":0.0012900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9642414579866454}
{"item_id":"senseval3.d001.s009.t005","gold_sense_keys":["agent%1:18:02::"],"candidates":[{"index":1,"sense_key":"agent%1:17:00::","synset_id":"agent.n.01"},{"index":2,"sense_key":"agent%1:18:02::","synset_id":"agent.n.02"},{"index":3,"sense_key":"agent%1:27:00::","synset_id":"agent.n.03"},{"index":4,"sense_key":"agent%1:18:01::","synset_id":"agent.n.04"},{"index":5,"sense_key":"agent%1:18:03::","synset_id":"agent.n.05"},{"index":6,"sense_key":"agent%1:10:00::","synset_id":"agentive_role.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"agent%1:18:02::","call_ids":["senseval3.d001.s009.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"agent%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":901,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001922,"input_uncached_usd":0.001802,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0845825000433251}
{"item_id":"senseval3.d001.s009.t006","gold_sense_keys":["republican%3:01:01::"],"candidates":[{"index":1,"sense_key":"republican%3:01:01::","synset_id":"republican.a.01"},{"index":2,"sense_key":"republican%5:00:00:democratic:00","synset_id":"republican.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"republican%3:01:01::","call_ids":["senseval3.d001.s009.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"republican%3:01:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":793,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001706,"input_uncached_usd":0.001586,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0644697499228641}
{"item_id":"senseval3.d001.s009.t007","gold_sense_keys":["brand%1:09:00::"],"candidates":[{"index":1,"sense_key":"brand%1:10:00::","synset_id":"trade_name.n.01"},{"index":2,"sense_key":"brand%1:09:00::","synset_id":"brand.n.02"},{"index":3,"sense_key":"brand%1:10:02::","synset_id":"brand.n.03"},{"index":4,"sense_key":"brand%1:27:00::","synset_id":"brand.n.04"},{"index":5,"sense_key":"brand%1:10:01::","synset_id":"mark.n.06"},{"index":6,"sense_key":"brand%1:06:00::","synset_id":"sword.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"brand%1:09:00::","call_ids":["senseval3.d001.s009.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"brand%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":936,"cached_input_tokens":0,"output_tokens":128,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0031520000000000003,"input_uncached_usd":0.001872,"input_cached_usd":0.0,"output_usd":0.00128,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.801623999956064}
{"item_id":"senseval3.d001.s009.t009","gold_sense_keys":["minimalist%5:00:00:moderate:00"],"candidates":[{"index":1,"sense_key":"minimalist%3:01:00::","synset_id":"minimalist.a.01"},{"index":2,"sense_key":"minimalist%5:00:00:moderate:00","synset_id":"minimalist.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"minimalist%5:00:00:moderate:00","call_ids":["senseval3.d001.s009.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"minimalist%5:00:00:moderate:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":692,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0015040000000000001,"input_uncached_usd":0.001384,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3180219169007614}
{"item_id":"senseval3.d001.s009.t010","gold_sense_keys":["state%1:14:00::","state%1:14:01::"],"candidates":[{"index":1,"sense_key":"state%1:15:01::","synset_id":"state.n.01"},{"index":2,"sense_key":"state%1:03:00::","synset_id":"state.n.02"},{"index":3,"sense_key":"state%1:14:01::","synset_id":"state.n.03"},{"index":4,"sense_key":"state%1:14:00::","synset_id":"state.n.04"},{"index":5,"sense_key":"state%1:26:02::","synset_id":"state_of_matter.n.01"},{"index":6,"sense_key":"state%1:26:01::","synset_id":"state.n.06"},{"index":7,"sense_key":"state%1:15:00::","synset_id":"country.n.02"},{"index":8,"sense_key":"state%1:14:02::","synset_id":"department_of_state.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"state%1:14:01::","call_ids":["senseval3.d001.s009.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"state%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1136,"cached_input_tokens":0,"output_tokens":79,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0030619999999999996,"input_uncached_usd":0.0022719999999999997,"input_cached_usd":0.0,"output_usd":0.00079,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.449494207976386}
{"item_id":"senseval3.d001.s009.t011","gold_sense_keys":["virtue%1:07:03::"],"candidates":[{"index":1,"sense_key":"virtue%1:07:01::","synset_id":"virtue.n.01"},{"index":2,"sense_key":"virtue%1:07:03::","synset_id":"merit.n.01"},{"index":3,"sense_key":"virtue%1:07:02::","synset_id":"virtue.n.03"},{"index":4,"sense_key":"virtue%1:07:04::","synset_id":"virtue.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"virtue%1:07:03::","call_ids":["senseval3.d001.s009.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"virtue%1:07:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":797,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0023639999999999998,"input_uncached_usd":0.001594,"input_cached_usd":0.0,"output_usd":0.0007700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7542672079289332}
{"item_id":"senseval3.d001.s009.t012","gold_sense_keys":["private%3:00:00::"],"candidates":[{"index":1,"sense_key":"private%3:00:00::","synset_id":"private.a.01"},{"index":2,"sense_key":"private%5:00:02:personal:00","synset_id":"private.s.01"},{"index":3,"sense_key":"private%5:00:00:personal:00","synset_id":"individual.s.03"},{"index":4,"sense_key":"private%5:00:00:inward:00","synset_id":"secret.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"private%3:00:00::","call_ids":["senseval3.d001.s009.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"private%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":815,"cached_input_tokens":0,"output_tokens":146,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00309,"input_uncached_usd":0.00163,"input_cached_usd":0.0,"output_usd":0.0014600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.8876648329896852}
{"item_id":"senseval3.d001.s009.t013","gold_sense_keys":["market%1:04:00::"],"candidates":[{"index":1,"sense_key":"market%1:04:00::","synset_id":"market.n.01"},{"index":2,"sense_key":"market%1:14:00::","synset_id":"market.n.02"},{"index":3,"sense_key":"market%1:06:00::","synset_id":"grocery_store.n.01"},{"index":4,"sense_key":"market%1:14:01::","synset_id":"market.n.04"},{"index":5,"sense_key":"market%1:06:01::","synset_id":"marketplace.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"market%1:04:00::","call_ids":["senseval3.d001.s009.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"market%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":904,"cached_input_tokens":0,"output_tokens":51,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0023179999999999997,"input_uncached_usd":0.001808,"input_cached_usd":0.0,"output_usd":0.00051,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1518133329227567}
{"item_id":"senseval3.d001.s009.t014","gold_sense_keys":["vice%1:07:00::"],"candidates":[{"index":1,"sense_key":"vice%1:07:00::","synset_id":"frailty.n.02"},{"index":2,"sense_key":"vice%1:04:00::","synset_id":"vice.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"vice%1:04:00::","call_ids":["senseval3.d001.s009.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"vice%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":689,"cached_input_tokens":0,"output_tokens":120,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002578,"input_uncached_usd":0.0013779999999999999,"input_cached_usd":0.0,"output_usd":0.0012000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.5618933339137584}
{"item_id":"senseval3.d001.s009.t015","gold_sense_keys":["public%3:00:00::"],"candidates":[{"index":1,"sense_key":"public%3:00:00::","synset_id":"public.a.01"},{"index":2,"sense_key":"public%5:00:00:common:02","synset_id":"public.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"public%3:00:00::","call_ids":["senseval3.d001.s009.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"public%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":699,"cached_input_tokens":0,"output_tokens":111,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002508,"input_uncached_usd":0.001398,"input_cached_usd":0.0,"output_usd":0.00111,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2301799170672894}
{"item_id":"senseval3.d001.s009.t016","gold_sense_keys":["action%1:04:04::"],"candidates":[{"index":1,"sense_key":"action%1:04:02::","synset_id":"action.n.01"},{"index":2,"sense_key":"action%1:26:00::","synset_id":"action.n.02"},{"index":3,"sense_key":"action%1:04:03::","synset_id":"military_action.n.01"},{"index":4,"sense_key":"action%1:22:00::","synset_id":"natural_process.n.01"},{"index":5,"sense_key":"action%1:10:00::","synset_id":"action.n.05"},{"index":6,"sense_key":"action%1:07:00::","synset_id":"action.n.06"},{"index":7,"sense_key":"action%1:06:00::","synset_id":"action.n.07"},{"index":8,"sense_key":"action%1:04:01::","synset_id":"legal_action.n.01"},{"index":9,"sense_key":"action%1:04:04::","synset_id":"action.n.09"},{"index":10,"sense_key":"action%1:04:00::","synset_id":"action.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"action%1:04:04::","call_ids":["senseval3.d001.s009.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"action%1:04:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1228,"cached_input_tokens":0,"output_tokens":47,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0029259999999999998,"input_uncached_usd":0.002456,"input_cached_usd":0.0,"output_usd":0.00047000000000000004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3256483329460025}
{"item_id":"senseval3.d001.s009.t017","gold_sense_keys":["democratic%3:01:00::"],"candidates":[{"index":1,"sense_key":"democratic%3:00:00::","synset_id":"democratic.a.01"},{"index":2,"sense_key":"democratic%3:01:00::","synset_id":"democratic.a.02"},{"index":3,"sense_key":"democratic%5:00:00:common:01","synset_id":"democratic.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"democratic%3:01:00::","call_ids":["senseval3.d001.s009.t017__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"democratic%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":803,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001726,"input_uncached_usd":0.001606,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3968040419276804}
{"item_id":"senseval3.d001.s009.t018","gold_sense_keys":["brand%1:09:00::"],"candidates":[{"index":1,"sense_key":"brand%1:10:00::","synset_id":"trade_name.n.01"},{"index":2,"sense_key":"brand%1:09:00::","synset_id":"brand.n.02"},{"index":3,"sense_key":"brand%1:10:02::","synset_id":"brand.n.03"},{"index":4,"sense_key":"brand%1:27:00::","synset_id":"brand.n.04"},{"index":5,"sense_key":"brand%1:10:01::","synset_id":"mark.n.06"},{"index":6,"sense_key":"brand%1:06:00::","synset_id":"sword.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"brand%1:10:00::","call_ids":["senseval3.d001.s009.t018__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"brand%1:10:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":936,"cached_input_tokens":0,"output_tokens":124,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003112,"input_uncached_usd":0.001872,"input_cached_usd":0.0,"output_usd":0.00124,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8896500000264496}
{"item_id":"senseval3.d001.s009.t022","gold_sense_keys":["public%3:00:00::"],"candidates":[{"index":1,"sense_key":"public%3:00:00::","synset_id":"public.a.01"},{"index":2,"sense_key":"public%5:00:00:common:02","synset_id":"public.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"public%3:00:00::","call_ids":["senseval3.d001.s009.t022__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"public%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":699,"cached_input_tokens":0,"output_tokens":111,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002508,"input_uncached_usd":0.001398,"input_cached_usd":0.0,"output_usd":0.00111,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7973012910224497}
{"item_id":"senseval3.d001.s009.t023","gold_sense_keys":["intervention%1:04:00::"],"candidates":[{"index":1,"sense_key":"intervention%1:04:00::","synset_id":"intervention.n.01"},{"index":2,"sense_key":"intervention%1:10:00::","synset_id":"intervention.n.02"},{"index":3,"sense_key":"intervention%1:04:03::","synset_id":"interposition.n.02"},{"index":4,"sense_key":"intervention%1:04:01::","synset_id":"intervention.n.04"},{"index":5,"sense_key":"intervention%1:04:02::","synset_id":"treatment.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"intervention%1:04:00::","call_ids":["senseval3.d001.s009.t023__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"intervention%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":954,"cached_input_tokens":0,"output_tokens":170,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003608,"input_uncached_usd":0.001908,"input_cached_usd":0.0,"output_usd":0.0017000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.301419584080577}
{"item_id":"senseval3.d001.s009.t024","gold_sense_keys":["remedy%2:30:00::"],"candidates":[{"index":1,"sense_key":"remedy%2:30:00::","synset_id":"rectify.v.04"},{"index":2,"sense_key":"remedy%2:29:00::","synset_id":"remedy.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"remedy%2:30:00::","call_ids":["senseval3.d001.s009.t024__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"remedy%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":727,"cached_input_tokens":0,"output_tokens":85,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002304,"input_uncached_usd":0.001454,"input_cached_usd":0.0,"output_usd":0.0008500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5543696670792997}
{"item_id":"senseval3.d001.s009.t025","gold_sense_keys":["excess%1:07:00::"],"candidates":[{"index":1,"sense_key":"excess%1:07:02::","synset_id":"excess.n.01"},{"index":2,"sense_key":"excess%1:07:00::","synset_id":"excess.n.02"},{"index":3,"sense_key":"excess%1:26:00::","synset_id":"surfeit.n.01"},{"index":4,"sense_key":"excess%1:04:00::","synset_id":"overindulgence.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"excess%1:07:00::","call_ids":["senseval3.d001.s009.t025__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"excess%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":823,"cached_input_tokens":0,"output_tokens":137,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003016,"input_uncached_usd":0.001646,"input_cached_usd":0.0,"output_usd":0.0013700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.09406862501055}
{"item_id":"senseval3.d001.s009.t026","gold_sense_keys":["attendant%5:00:00:subsequent:00"],"candidates":[{"index":1,"sense_key":"attendant%5:00:00:present:02","synset_id":"attendant.s.01"},{"index":2,"sense_key":"attendant%5:00:00:subsequent:00","synset_id":"attendant.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"attendant%5:00:00:subsequent:00","call_ids":["senseval3.d001.s009.t026__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"attendant%5:00:00:subsequent:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":769,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001658,"input_uncached_usd":0.0015379999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.10273208306171}
{"item_id":"senseval3.d001.s009.t027","gold_sense_keys":["pursuit%1:09:00::"],"candidates":[{"index":1,"sense_key":"pursuit%1:04:00::","synset_id":"pursuit.n.01"},{"index":2,"sense_key":"pursuit%1:09:00::","synset_id":"pursuit.n.02"},{"index":3,"sense_key":"pursuit%1:04:02::","synset_id":"avocation.n.01"},{"index":4,"sense_key":"pursuit%1:04:01::","synset_id":"pastime.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"pursuit%1:09:00::","call_ids":["senseval3.d001.s009.t027__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"pursuit%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":912,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002484,"input_uncached_usd":0.0018239999999999999,"input_cached_usd":0.0,"output_usd":0.0006600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5398795830551535}
{"item_id":"senseval3.d001.s009.t028","gold_sense_keys":["private%3:00:00::"],"candidates":[{"index":1,"sense_key":"private%3:00:00::","synset_id":"private.a.01"},{"index":2,"sense_key":"private%5:00:02:personal:00","synset_id":"private.s.01"},{"index":3,"sense_key":"private%5:00:00:personal:00","synset_id":"individual.s.03"},{"index":4,"sense_key":"private%5:00:00:inward:00","synset_id":"secret.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"private%5:00:00:personal:00","call_ids":["senseval3.d001.s009.t028__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"private%5:00:00:personal:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":815,"cached_input_tokens":0,"output_tokens":119,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00282,"input_uncached_usd":0.00163,"input_cached_usd":0.0,"output_usd":0.00119,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.1170648749684915}
{"item_id":"senseval3.d001.s009.t029","gold_sense_keys":["interest%1:07:01::"],"candidates":[{"index":1,"sense_key":"interest%1:09:00::","synset_id":"interest.n.01"},{"index":2,"sense_key":"interest%1:07:01::","synset_id":"sake.n.01"},{"index":3,"sense_key":"interest%1:07:02::","synset_id":"interest.n.03"},{"index":4,"sense_key":"interest%1:21:00::","synset_id":"interest.n.04"},{"index":5,"sense_key":"interest%1:21:03::","synset_id":"interest.n.05"},{"index":6,"sense_key":"interest%1:14:00::","synset_id":"interest.n.06"},{"index":7,"sense_key":"interest%1:04:01::","synset_id":"pastime.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"interest%1:21:03::","call_ids":["senseval3.d001.s009.t029__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"interest%1:21:03::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1098,"cached_input_tokens":0,"output_tokens":299,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.005186,"input_uncached_usd":0.002196,"input_cached_usd":0.0,"output_usd":0.00299,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.0469992919825017}
{"item_id":"senseval3.d001.s010.t001","gold_sense_keys":["representative%1:18:01::"],"candidates":[{"index":1,"sense_key":"representative%1:18:00::","synset_id":"representative.n.01"},{"index":2,"sense_key":"representative%1:18:02::","synset_id":"spokesperson.n.01"},{"index":3,"sense_key":"representative%1:18:01::","synset_id":"congressman.n.01"},{"index":4,"sense_key":"representative%1:09:00::","synset_id":"example.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"representative%1:18:01::","call_ids":["senseval3.d001.s010.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"representative%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":831,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001782,"input_uncached_usd":0.0016619999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.134620958007872}
{"item_id":"senseval3.d001.s010.t002","gold_sense_keys":["have%2:40:00::","have%2:42:12::"],"candidates":[{"index":1,"sense_key":"have%2:40:00::","synset_id":"have.v.01"},{"index":2,"sense_key":"have%2:42:00::","synset_id":"have.v.02"},{"index":3,"sense_key":"have%2:39:06::","synset_id":"experience.v.03"},{"index":4,"sense_key":"have%2:40:04::","synset_id":"own.v.01"},{"index":5,"sense_key":"have%2:30:00::","synset_id":"get.v.03"},{"index":6,"sense_key":"have%2:34:00::","synset_id":"consume.v.02"},{"index":7,"sense_key":"have%2:41:00::","synset_id":"have.v.07"},{"index":8,"sense_key":"have%2:36:00::","synset_id":"hold.v.03"},{"index":9,"sense_key":"have%2:40:01::","synset_id":"have.v.09"},{"index":10,"sense_key":"have%2:42:12::","synset_id":"have.v.10"},{"index":11,"sense_key":"have%2:30:01::","synset_id":"have.v.11"},{"index":12,"sense_key":"have%2:29:02::","synset_id":"have.v.12"},{"index":13,"sense_key":"have%2:32:00::","synset_id":"induce.v.02"},{"index":14,"sense_key":"have%2:40:05::","synset_id":"accept.v.02"},{"index":15,"sense_key":"have%2:40:03::","synset_id":"receive.v.01"},{"index":16,"sense_key":"have%2:29:05::","synset_id":"suffer.v.02"},{"index":17,"sense_key":"have%2:40:02::","synset_id":"have.v.17"},{"index":18,"sense_key":"have%2:29:00::","synset_id":"give_birth.v.01"},{"index":19,"sense_key":"have%2:35:00::","synset_id":"take.v.35"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"have%2:40:00::","call_ids":["senseval3.d001.s010.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"have%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1760,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0042,"input_uncached_usd":0.0035199999999999997,"input_cached_usd":0.0,"output_usd":0.00068,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9548201670404524}
{"item_id":"senseval3.d001.s010.t004","gold_sense_keys":["responsibility%1:04:00::"],"candidates":[{"index":1,"sense_key":"responsibility%1:04:00::","synset_id":"duty.n.01"},{"index":2,"sense_key":"responsibility%1:26:00::","synset_id":"province.n.02"},{"index":3,"sense_key":"responsibility%1:07:00::","synset_id":"responsibility.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"responsibility%1:04:00::","call_ids":["senseval3.d001.s010.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"responsibility%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":857,"cached_input_tokens":0,"output_tokens":185,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0035640000000000003,"input_uncached_usd":0.001714,"input_cached_usd":0.0,"output_usd":0.00185,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0838692920515314}
{"item_id":"senseval3.d001.s010.t005","gold_sense_keys":["vote%2:41:10::","vote%2:41:11::","vote%2:41:12::","vote%2:41:13::"],"candidates":[{"index":1,"sense_key":"vote%2:41:13::","synset_id":"vote.v.01"},{"index":2,"sense_key":"vote%2:41:10::","synset_id":"vote.v.02"},{"index":3,"sense_key":"vote%2:41:11::","synset_id":"vote.v.03"},{"index":4,"sense_key":"vote%2:41:09::","synset_id":"vote.v.04"},{"index":5,"sense_key":"vote%2:41:12::","synset_id":"vote.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"vote%2:41:13::","call_ids":["senseval3.d001.s010.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"vote%2:41:13::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":864,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002368,"input_uncached_usd":0.001728,"input_cached_usd":0.0,"output_usd":0.00064,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.388114915927872}
{"item_id":"senseval3.d001.s010.t006","gold_sense_keys":["office%1:26:00::"],"candidates":[{"index":1,"sense_key":"office%1:06:00::","synset_id":"office.n.01"},{"index":2,"sense_key":"office%1:14:01::","synset_id":"agency.n.01"},{"index":3,"sense_key":"office%1:04:01::","synset_id":"function.n.03"},{"index":4,"sense_key":"office%1:26:00::","synset_id":"office.n.04"},{"index":5,"sense_key":"office%1:14:00::","synset_id":"office.n.05"},{"index":6,"sense_key":"office%1:04:00::","synset_id":"office.n.06"},{"index":7,"sense_key":"office%1:04:02::","synset_id":"position.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"office%1:26:00::","call_ids":["senseval3.d001.s010.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"office%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1040,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00263,"input_uncached_usd":0.00208,"input_cached_usd":0.0,"output_usd":0.00055,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.235015208949335}
{"item_id":"senseval3.d001.s010.t007","gold_sense_keys":["deal%2:41:09::"],"candidates":[{"index":1,"sense_key":"deal%2:32:08::","synset_id":"cover.v.05"},{"index":2,"sense_key":"deal%2:31:10::","synset_id":"consider.v.03"},{"index":3,"sense_key":"deal%2:41:09::","synset_id":"deal.v.03"},{"index":4,"sense_key":"deal%2:41:00::","synset_id":"cope.v.01"},{"index":5,"sense_key":"deal%2:40:01::","synset_id":"distribute.v.01"},{"index":6,"sense_key":"deal%2:40:00::","synset_id":"deal.v.06"},{"index":7,"sense_key":"deal%2:41:13::","synset_id":"manage.v.02"},{"index":8,"sense_key":"deal%2:41:04::","synset_id":"deal.v.08"},{"index":9,"sense_key":"deal%2:40:02::","synset_id":"deal.v.09"},{"index":10,"sense_key":"deal%2:41:02::","synset_id":"conduct.v.01"},{"index":11,"sense_key":"deal%2:40:07::","synset_id":"share.v.04"},{"index":12,"sense_key":"deal%2:40:06::","synset_id":"deal.v.12"},{"index":13,"sense_key":"deal%2:40:11::","synset_id":"deal.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"deal%2:41:09::","call_ids":["senseval3.d001.s010.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"deal%2:41:09::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1399,"cached_input_tokens":0,"output_tokens":139,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004188,"input_uncached_usd":0.0027979999999999997,"input_cached_usd":0.0,"output_usd":0.0013900000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.920140041038394}
{"item_id":"senseval3.d001.s010.t008","gold_sense_keys":["national%3:00:00::","national%3:00:01::","national%3:01:00::","national%5:00:00:domestic:00"],"candidates":[{"index":1,"sense_key":"national%3:01:00::","synset_id":"national.a.01"},{"index":2,"sense_key":"national%3:00:00::","synset_id":"national.a.02"},{"index":3,"sense_key":"national%3:00:01::","synset_id":"national.a.03"},{"index":4,"sense_key":"national%5:00:00:public:00","synset_id":"national.s.01"},{"index":5,"sense_key":"national%5:00:00:domestic:00","synset_id":"home.s.01"},{"index":6,"sense_key":"national%3:01:02::","synset_id":"national.a.06"},{"index":7,"sense_key":"national%3:01:01::","synset_id":"national.a.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"national%3:00:01::","call_ids":["senseval3.d001.s010.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"national%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":954,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002528,"input_uncached_usd":0.001908,"input_cached_usd":0.0,"output_usd":0.00062,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4381505000637844}
{"item_id":"senseval3.d001.s010.t009","gold_sense_keys":["issue%1:09:01::"],"candidates":[{"index":1,"sense_key":"issue%1:09:01::","synset_id":"issue.n.01"},{"index":2,"sense_key":"issue%1:10:00::","synset_id":"issue.n.02"},{"index":3,"sense_key":"issue%1:09:00::","synset_id":"topic.n.02"},{"index":4,"sense_key":"issue%1:04:01::","synset_id":"issue.n.04"},{"index":5,"sense_key":"issue%1:21:01::","synset_id":"issue.n.05"},{"index":6,"sense_key":"issue%1:21:00::","synset_id":"return.n.06"},{"index":7,"sense_key":"issue%1:19:00::","synset_id":"consequence.n.01"},{"index":8,"sense_key":"issue%1:18:00::","synset_id":"offspring.n.01"},{"index":9,"sense_key":"issue%1:11:00::","synset_id":"emergence.n.02"},{"index":10,"sense_key":"issue%1:06:00::","synset_id":"exit.n.01"},{"index":11,"sense_key":"issue%1:04:00::","synset_id":"issue.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"issue%1:09:01::","call_ids":["senseval3.d001.s010.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"issue%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1370,"cached_input_tokens":0,"output_tokens":133,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00407,"input_uncached_usd":0.00274,"input_cached_usd":0.0,"output_usd":0.00133,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.794146207976155}
{"item_id":"senseval3.d001.s010.t010","gold_sense_keys":["action%1:04:04::"],"candidates":[{"index":1,"sense_key":"action%1:04:02::","synset_id":"action.n.01"},{"index":2,"sense_key":"action%1:26:00::","synset_id":"action.n.02"},{"index":3,"sense_key":"action%1:04:03::","synset_id":"military_action.n.01"},{"index":4,"sense_key":"action%1:22:00::","synset_id":"natural_process.n.01"},{"index":5,"sense_key":"action%1:10:00::","synset_id":"action.n.05"},{"index":6,"sense_key":"action%1:07:00::","synset_id":"action.n.06"},{"index":7,"sense_key":"action%1:06:00::","synset_id":"action.n.07"},{"index":8,"sense_key":"action%1:04:01::","synset_id":"legal_action.n.01"},{"index":9,"sense_key":"action%1:04:04::","synset_id":"action.n.09"},{"index":10,"sense_key":"action%1:04:00::","synset_id":"action.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"action%1:04:02::","call_ids":["senseval3.d001.s010.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"action%1:04:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1216,"cached_input_tokens":0,"output_tokens":126,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003692,"input_uncached_usd":0.0024319999999999997,"input_cached_usd":0.0,"output_usd":0.00126,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7851637919666246}
{"item_id":"senseval3.d001.s010.t011","gold_sense_keys":["cast%2:40:13::"],"candidates":[{"index":1,"sense_key":"cast%2:36:02::","synset_id":"project.v.10"},{"index":2,"sense_key":"cast%2:40:13::","synset_id":"cast.v.02"},{"index":3,"sense_key":"cast%2:36:01::","synset_id":"cast.v.03"},{"index":4,"sense_key":"cast%2:35:00::","synset_id":"hurl.v.01"},{"index":5,"sense_key":"cast%2:41:14::","synset_id":"cast.v.05"},{"index":6,"sense_key":"cast%2:38:01::","synset_id":"roll.v.12"},{"index":7,"sense_key":"cast%2:36:00::","synset_id":"cast.v.07"},{"index":8,"sense_key":"cast%2:35:01::","synset_id":"shed.v.01"},{"index":9,"sense_key":"cast%2:33:00::","synset_id":"draw.v.14"},{"index":10,"sense_key":"cast%2:32:00::","synset_id":"frame.v.04"},{"index":11,"sense_key":"cast%2:29:00::","synset_id":"vomit.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cast%2:40:13::","call_ids":["senseval3.d001.s010.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cast%2:40:13::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1318,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0027559999999999998,"input_uncached_usd":0.002636,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6952310830820352}
{"item_id":"senseval3.d001.s010.t013","gold_sense_keys":["vote%1:04:00::"],"candidates":[{"index":1,"sense_key":"vote%1:04:00::","synset_id":"vote.n.01"},{"index":2,"sense_key":"vote%1:04:01::","synset_id":"vote.n.02"},{"index":3,"sense_key":"vote%1:07:00::","synset_id":"right_to_vote.n.01"},{"index":4,"sense_key":"vote%1:14:00::","synset_id":"vote.n.04"},{"index":5,"sense_key":"vote%1:09:00::","synset_id":"vote.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"vote%1:04:00::","call_ids":["senseval3.d001.s010.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"vote%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":922,"cached_input_tokens":0,"output_tokens":82,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002664,"input_uncached_usd":0.001844,"input_cached_usd":0.0,"output_usd":0.0008200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5953121250495315}
{"item_id":"senseval3.d001.s010.t014","gold_sense_keys":["legislation%1:10:00::"],"candidates":[{"index":1,"sense_key":"legislation%1:10:00::","synset_id":"legislation.n.01"},{"index":2,"sense_key":"legislation%1:04:00::","synset_id":"legislation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"legislation%1:10:00::","call_ids":["senseval3.d001.s010.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"legislation%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":685,"cached_input_tokens":0,"output_tokens":78,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00215,"input_uncached_usd":0.00137,"input_cached_usd":0.0,"output_usd":0.0007800000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5902029169956222}
{"item_id":"senseval3.d001.s010.t015","gold_sense_keys":["impose%2:32:00::"],"candidates":[{"index":1,"sense_key":"impose%2:41:00::","synset_id":"enforce.v.02"},{"index":2,"sense_key":"impose%2:32:00::","synset_id":"inflict.v.01"},{"index":3,"sense_key":"impose%2:40:00::","synset_id":"levy.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"impose%2:40:00::","call_ids":["senseval3.d001.s010.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"impose%2:40:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":754,"cached_input_tokens":0,"output_tokens":135,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0028580000000000003,"input_uncached_usd":0.001508,"input_cached_usd":0.0,"output_usd":0.00135,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6368046670686454}
{"item_id":"senseval3.d001.s010.t016","gold_sense_keys":["cost%1:07:01::"],"candidates":[{"index":1,"sense_key":"cost%1:21:00::","synset_id":"cost.n.01"},{"index":2,"sense_key":"cost%1:07:00::","synset_id":"monetary_value.n.01"},{"index":3,"sense_key":"cost%1:07:01::","synset_id":"price.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"cost%1:07:01::","call_ids":["senseval3.d001.s010.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"cost%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":795,"cached_input_tokens":0,"output_tokens":145,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00304,"input_uncached_usd":0.0015899999999999998,"input_cached_usd":0.0,"output_usd":0.0014500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.778411125065759}
{"item_id":"senseval3.d001.s010.t017","gold_sense_keys":["confer%2:40:00::"],"candidates":[{"index":1,"sense_key":"confer%2:32:00::","synset_id":"confer.v.01"},{"index":2,"sense_key":"confer%2:40:00::","synset_id":"confer.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"confer%2:40:00::","call_ids":["senseval3.d001.s010.t017__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"confer%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":723,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0015660000000000001,"input_uncached_usd":0.001446,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.54915033292491}
{"item_id":"senseval3.d001.s010.t018","gold_sense_keys":["benefit%1:07:00::"],"candidates":[{"index":1,"sense_key":"benefit%1:21:00::","synset_id":"benefit.n.01"},{"index":2,"sense_key":"benefit%1:07:00::","synset_id":"benefit.n.02"},{"index":3,"sense_key":"benefit%1:10:00::","synset_id":"benefit.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"benefit%1:07:00::","call_ids":["senseval3.d001.s010.t018__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"benefit%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":718,"cached_input_tokens":0,"output_tokens":106,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002496,"input_uncached_usd":0.001436,"input_cached_usd":0.0,"output_usd":0.0010600000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7401515829842538}
{"item_id":"senseval3.d001.s010.t019","gold_sense_keys":["population%1:14:00::"],"candidates":[{"index":1,"sense_key":"population%1:14:00::","synset_id":"population.n.01"},{"index":2,"sense_key":"population%1:14:01::","synset_id":"population.n.02"},{"index":3,"sense_key":"population%1:09:00::","synset_id":"population.n.03"},{"index":4,"sense_key":"population%1:23:00::","synset_id":"population.n.04"},{"index":5,"sense_key":"population%1:04:00::","synset_id":"population.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"population%1:14:00::","call_ids":["senseval3.d001.s010.t019__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"population%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":928,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001976,"input_uncached_usd":0.001856,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.963268458028324}
{"item_id":"senseval3.d001.s010.t020","gold_sense_keys":["attend_to%2:31:00::"],"candidates":[{"index":1,"sense_key":"attend_to%2:31:00::","synset_id":"attend_to.v.01"},{"index":2,"sense_key":"attend_to%2:41:01::","synset_id":"serve.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"attend_to%2:31:00::","call_ids":["senseval3.d001.s010.t020__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"attend_to%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":731,"cached_input_tokens":0,"output_tokens":91,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002372,"input_uncached_usd":0.001462,"input_cached_usd":0.0,"output_usd":0.0009100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5197658330434933}
{"item_id":"senseval3.d001.s010.t021","gold_sense_keys":["local%3:00:01::"],"candidates":[{"index":1,"sense_key":"local%3:00:01::","synset_id":"local.a.01"},{"index":2,"sense_key":"local%3:01:01::","synset_id":"local.a.02"},{"index":3,"sense_key":"local%3:00:02::","synset_id":"local.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"local%3:01:01::","call_ids":["senseval3.d001.s010.t021__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"local%3:01:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":755,"cached_input_tokens":0,"output_tokens":134,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00285,"input_uncached_usd":0.0015099999999999998,"input_cached_usd":0.0,"output_usd":0.00134,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.5623487920966}
{"item_id":"senseval3.d001.s010.t022","gold_sense_keys":["issue%1:09:01::"],"candidates":[{"index":1,"sense_key":"issue%1:09:01::","synset_id":"issue.n.01"},{"index":2,"sense_key":"issue%1:10:00::","synset_id":"issue.n.02"},{"index":3,"sense_key":"issue%1:09:00::","synset_id":"topic.n.02"},{"index":4,"sense_key":"issue%1:04:01::","synset_id":"issue.n.04"},{"index":5,"sense_key":"issue%1:21:01::","synset_id":"issue.n.05"},{"index":6,"sense_key":"issue%1:21:00::","synset_id":"return.n.06"},{"index":7,"sense_key":"issue%1:19:00::","synset_id":"consequence.n.01"},{"index":8,"sense_key":"issue%1:18:00::","synset_id":"offspring.n.01"},{"index":9,"sense_key":"issue%1:11:00::","synset_id":"emergence.n.02"},{"index":10,"sense_key":"issue%1:06:00::","synset_id":"exit.n.01"},{"index":11,"sense_key":"issue%1:04:00::","synset_id":"issue.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"issue%1:09:01::","call_ids":["senseval3.d001.s010.t022__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"issue%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1370,"cached_input_tokens":0,"output_tokens":206,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0048000000000000004,"input_uncached_usd":0.00274,"input_cached_usd":0.0,"output_usd":0.00206,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2150142500177026}
{"item_id":"senseval3.d001.s010.t024","gold_sense_keys":["service%1:04:08::"],"candidates":[{"index":1,"sense_key":"service%1:04:08::","synset_id":"service.n.01"},{"index":2,"sense_key":"service%1:04:00::","synset_id":"service.n.02"},{"index":3,"sense_key":"service%1:04:01::","synset_id":"service.n.03"},{"index":4,"sense_key":"service%1:14:05::","synset_id":"service.n.04"},{"index":5,"sense_key":"service%1:04:07::","synset_id":"service.n.05"},{"index":6,"sense_key":"service%1:14:00::","synset_id":"military_service.n.01"},{"index":7,"sense_key":"service%1:18:00::","synset_id":"service.n.07"},{"index":8,"sense_key":"service%1:07:00::","synset_id":"avail.n.01"},{"index":9,"sense_key":"service%1:06:00::","synset_id":"service.n.09"},{"index":10,"sense_key":"service%1:04:03::","synset_id":"servicing.n.01"},{"index":11,"sense_key":"service%1:04:09::","synset_id":"service.n.11"},{"index":12,"sense_key":"service%1:04:02::","synset_id":"serve.n.01"},{"index":13,"sense_key":"service%1:04:04::","synset_id":"service.n.13"},{"index":14,"sense_key":"service%1:04:05::","synset_id":"overhaul.n.01"},{"index":15,"sense_key":"service%1:04:06::","synset_id":"service.n.15"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"service%1:04:08::","call_ids":["senseval3.d001.s010.t024__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"service%1:04:08::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1390,"cached_input_tokens":0,"output_tokens":186,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00464,"input_uncached_usd":0.00278,"input_cached_usd":0.0,"output_usd":0.00186,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.826319208019413}
{"item_id":"senseval3.d001.s011.t000","gold_sense_keys":["republican%3:01:01::"],"candidates":[{"index":1,"sense_key":"republican%3:01:01::","synset_id":"republican.a.01"},{"index":2,"sense_key":"republican%5:00:00:democratic:00","synset_id":"republican.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"republican%3:01:01::","call_ids":["senseval3.d001.s011.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"republican%3:01:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":815,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00175,"input_uncached_usd":0.00163,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8041360420174897}
{"item_id":"senseval3.d001.s011.t002","gold_sense_keys":["representative%1:18:01::"],"candidates":[{"index":1,"sense_key":"representative%1:18:00::","synset_id":"representative.n.01"},{"index":2,"sense_key":"representative%1:18:02::","synset_id":"spokesperson.n.01"},{"index":3,"sense_key":"representative%1:18:01::","synset_id":"congressman.n.01"},{"index":4,"sense_key":"representative%1:09:00::","synset_id":"example.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"representative%1:18:01::","call_ids":["senseval3.d001.s011.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"representative%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":866,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001852,"input_uncached_usd":0.0017319999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3174962080083787}
{"item_id":"senseval3.d001.s011.t003","gold_sense_keys":["belief%1:09:00::"],"candidates":[{"index":1,"sense_key":"belief%1:09:00::","synset_id":"belief.n.01"},{"index":2,"sense_key":"belief%1:09:01::","synset_id":"impression.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"belief%1:09:00::","call_ids":["senseval3.d001.s011.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"belief%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":740,"cached_input_tokens":0,"output_tokens":78,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00226,"input_uncached_usd":0.00148,"input_cached_usd":0.0,"output_usd":0.0007800000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5099858330795541}
{"item_id":"senseval3.d001.s011.t004","gold_sense_keys":["minimalist%5:00:00:moderate:00"],"candidates":[{"index":1,"sense_key":"minimalist%3:01:00::","synset_id":"minimalist.a.01"},{"index":2,"sense_key":"minimalist%5:00:00:moderate:00","synset_id":"minimalist.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"minimalist%5:00:00:moderate:00","call_ids":["senseval3.d001.s011.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"minimalist%5:00:00:moderate:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":714,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0015480000000000001,"input_uncached_usd":0.001428,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.341083542094566}
{"item_id":"senseval3.d001.s011.t005","gold_sense_keys":["state%1:14:00::","state%1:14:01::"],"candidates":[{"index":1,"sense_key":"state%1:15:01::","synset_id":"state.n.01"},{"index":2,"sense_key":"state%1:03:00::","synset_id":"state.n.02"},{"index":3,"sense_key":"state%1:14:01::","synset_id":"state.n.03"},{"index":4,"sense_key":"state%1:14:00::","synset_id":"state.n.04"},{"index":5,"sense_key":"state%1:26:02::","synset_id":"state_of_matter.n.01"},{"index":6,"sense_key":"state%1:26:01::","synset_id":"state.n.06"},{"index":7,"sense_key":"state%1:15:00::","synset_id":"country.n.02"},{"index":8,"sense_key":"state%1:14:02::","synset_id":"department_of_state.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"state%1:14:01::","call_ids":["senseval3.d001.s011.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"state%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1158,"cached_input_tokens":0,"output_tokens":89,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003206,"input_uncached_usd":0.002316,"input_cached_usd":0.0,"output_usd":0.0008900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.956962791038677}
{"item_id":"senseval3.d001.s011.t006","gold_sense_keys":["willing%3:00:00::"],"candidates":[{"index":1,"sense_key":"willing%3:00:00::","synset_id":"willing.a.01"},{"index":2,"sense_key":"willing%5:00:00:voluntary:01","synset_id":"uncoerced.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"willing%3:00:00::","call_ids":["senseval3.d001.s011.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"willing%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":740,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0016,"input_uncached_usd":0.00148,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2135027080075815}
{"item_id":"senseval3.d001.s011.t007","gold_sense_keys":["engage%2:41:06::"],"candidates":[{"index":1,"sense_key":"engage%2:41:06::","synset_id":"prosecute.v.03"},{"index":2,"sense_key":"engage%2:31:00::","synset_id":"absorb.v.09"},{"index":3,"sense_key":"engage%2:41:01::","synset_id":"hire.v.01"},{"index":4,"sense_key":"engage%2:41:00::","synset_id":"engage.v.04"},{"index":5,"sense_key":"engage%2:32:00::","synset_id":"betroth.v.01"},{"index":6,"sense_key":"engage%2:30:00::","synset_id":"engage.v.06"},{"index":7,"sense_key":"engage%2:41:12::","synset_id":"engage.v.07"},{"index":8,"sense_key":"engage%2:40:01::","synset_id":"engage.v.08"},{"index":9,"sense_key":"engage%2:40:00::","synset_id":"lease.v.04"},{"index":10,"sense_key":"engage%2:35:00::","synset_id":"engage.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"engage%2:41:06::","call_ids":["senseval3.d001.s011.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"engage%2:41:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1215,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0025499999999999997,"input_uncached_usd":0.00243,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.130890249973163}
{"item_id":"senseval3.d001.s011.t008","gold_sense_keys":["local%3:00:01::"],"candidates":[{"index":1,"sense_key":"local%3:00:01::","synset_id":"local.a.01"},{"index":2,"sense_key":"local%3:01:01::","synset_id":"local.a.02"},{"index":3,"sense_key":"local%3:00:02::","synset_id":"local.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"local%3:00:01::","call_ids":["senseval3.d001.s011.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"local%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":790,"cached_input_tokens":0,"output_tokens":168,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00326,"input_uncached_usd":0.00158,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.070066999993287}
{"item_id":"senseval3.d001.s011.t009","gold_sense_keys":["democratic%3:01:00::"],"candidates":[{"index":1,"sense_key":"democratic%3:00:00::","synset_id":"democratic.a.01"},{"index":2,"sense_key":"democratic%3:01:00::","synset_id":"democratic.a.02"},{"index":3,"sense_key":"democratic%5:00:00:common:01","synset_id":"democratic.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"democratic%3:01:00::","call_ids":["senseval3.d001.s011.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"democratic%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":825,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00177,"input_uncached_usd":0.00165,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0603871250059456}
{"item_id":"senseval3.d001.s011.t010","gold_sense_keys":["member%1:18:00::"],"candidates":[{"index":1,"sense_key":"member%1:18:00::","synset_id":"member.n.01"},{"index":2,"sense_key":"member%1:24:00::","synset_id":"member.n.02"},{"index":3,"sense_key":"member%1:08:00::","synset_id":"extremity.n.01"},{"index":4,"sense_key":"member%1:14:00::","synset_id":"member.n.04"},{"index":5,"sense_key":"member%1:08:01::","synset_id":"penis.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"member%1:18:00::","call_ids":["senseval3.d001.s011.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"member%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":956,"cached_input_tokens":0,"output_tokens":43,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002342,"input_uncached_usd":0.0019119999999999999,"input_cached_usd":0.0,"output_usd":0.00043000000000000004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2400938749779016}
{"item_id":"senseval3.d001.s012.t000","gold_sense_keys":["assumption%1:09:00::","assumption%1:10:00::"],"candidates":[{"index":1,"sense_key":"assumption%1:10:00::","synset_id":"premise.n.01"},{"index":2,"sense_key":"assumption%1:09:00::","synset_id":"assumption.n.02"},{"index":3,"sense_key":"assumption%1:04:00::","synset_id":"assumption.n.03"},{"index":4,"sense_key":"assumption%1:28:00::","synset_id":"assumption.n.04"},{"index":5,"sense_key":"assumption%1:11:00::","synset_id":"assumption.n.05"},{"index":6,"sense_key":"assumption%1:07:00::","synset_id":"presumption.n.03"},{"index":7,"sense_key":"assumption%1:04:01::","synset_id":"assumption.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"assumption%1:10:00::","call_ids":["senseval3.d001.s012.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"assumption%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1214,"cached_input_tokens":0,"output_tokens":119,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003618,"input_uncached_usd":0.002428,"input_cached_usd":0.0,"output_usd":0.00119,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8945061670383438}
{"item_id":"senseval3.d001.s012.t001","gold_sense_keys":["hold%2:42:02::"],"candidates":[{"index":1,"sense_key":"hold%2:42:00::","synset_id":"keep.v.01"},{"index":2,"sense_key":"hold%2:35:00::","synset_id":"hold.v.02"},{"index":3,"sense_key":"hold%2:36:00::","synset_id":"hold.v.03"},{"index":4,"sense_key":"hold%2:40:00::","synset_id":"have.v.01"},{"index":5,"sense_key":"hold%2:31:01::","synset_id":"deem.v.01"},{"index":6,"sense_key":"hold%2:37:00::","synset_id":"harbor.v.01"},{"index":7,"sense_key":"hold%2:35:03::","synset_id":"restrain.v.03"},{"index":8,"sense_key":"hold%2:40:02::","synset_id":"retain.v.03"},{"index":9,"sense_key":"hold%2:40:04::","synset_id":"bear.v.11"},{"index":10,"sense_key":"hold%2:35:01::","synset_id":"hold.v.10"},{"index":11,"sense_key":"hold%2:42:13::","synset_id":"hold.v.11"},{"index":12,"sense_key":"hold%2:42:05::","synset_id":"accommodate.v.04"},{"index":13,"sense_key":"hold%2:42:01::","synset_id":"hold.v.13"},{"index":14,"sense_key":"hold%2:35:02::","synset_id":"hold.v.14"},{"index":15,"sense_key":"hold%2:42:02::","synset_id":"prevail.v.02"},{"index":16,"sense_key":"hold%2:31:08::","synset_id":"hold.v.16"},{"index":17,"sense_key":"hold%2:42:07::","synset_id":"hold.v.17"},{"index":18,"sense_key":"hold%2:42:14::","synset_id":"contain.v.05"},{"index":19,"sense_key":"hold%2:41:00::","synset_id":"reserve.v.04"},{"index":20,"sense_key":"hold%2:33:05::","synset_id":"defend.v.03"},{"index":21,"sense_key":"hold%2:32:13::","synset_id":"oblige.v.02"},{"index":22,"sense_key":"hold%2:31:04::","synset_id":"hold.v.22"},{"index":23,"sense_key":"hold%2:31:10::","synset_id":"hold.v.23"},{"index":24,"sense_key":"hold%2:42:08::","synset_id":"defy.v.01"},{"index":25,"sense_key":"hold%2:42:04::","synset_id":"apply.v.02"},{"index":26,"sense_key":"hold%2:42:12::","synset_id":"hold.v.26"},{"index":27,"sense_key":"hold%2:41:15::","synset_id":"control.v.02"},{"index":28,"sense_key":"hold%2:41:13::","synset_id":"hold.v.28"},{"index":29,"sense_key":"hold%2:41:14::","synset_id":"hold.v.29"},{"index":30,"sense_key":"hold%2:38:05::","synset_id":"halt.v.01"},{"index":31,"sense_key":"hold%2:35:09::","synset_id":"hold.v.31"},{"index":32,"sense_key":"hold%2:34:00::","synset_id":"carry.v.33"},{"index":33,"sense_key":"hold%2:33:00::","synset_id":"hold.v.33"},{"index":34,"sense_key":"hold%2:32:11::","synset_id":"declare.v.04"},{"index":35,"sense_key":"hold%2:32:15::","synset_id":"agree.v.01"},{"index":36,"sense_key":"hold%2:29:04::","synset_id":"hold.v.36"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":15,"chosen_sense_key":"hold%2:42:02::","call_ids":["senseval3.d001.s012.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":15,"predicted_sense_key":"hold%2:42:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":3025,"cached_input_tokens":0,"output_tokens":144,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00749,"input_uncached_usd":0.00605,"input_cached_usd":0.0,"output_usd":0.00144,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8272580829216167}
{"item_id":"senseval3.d001.s012.t003","gold_sense_keys":["race%1:11:01::"],"candidates":[{"index":1,"sense_key":"race%1:11:01::","synset_id":"race.n.01"},{"index":2,"sense_key":"race%1:11:00::","synset_id":"race.n.02"},{"index":3,"sense_key":"race%1:14:00::","synset_id":"race.n.03"},{"index":4,"sense_key":"race%1:14:01::","synset_id":"subspecies.n.01"},{"index":5,"sense_key":"race%1:19:00::","synset_id":"slipstream.n.01"},{"index":6,"sense_key":"race%1:06:00::","synset_id":"raceway.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"race%1:11:01::","call_ids":["senseval3.d001.s012.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"race%1:11:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":988,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0020959999999999998,"input_uncached_usd":0.001976,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0208786249859259}
{"item_id":"senseval3.d001.s012.t004","gold_sense_keys":["face%2:32:00::"],"candidates":[{"index":1,"sense_key":"face%2:32:00::","synset_id":"confront.v.02"},{"index":2,"sense_key":"face%2:33:00::","synset_id":"confront.v.01"},{"index":3,"sense_key":"face%2:42:00::","synset_id":"front.v.01"},{"index":4,"sense_key":"face%2:42:02::","synset_id":"face.v.04"},{"index":5,"sense_key":"face%2:38:00::","synset_id":"face.v.05"},{"index":6,"sense_key":"face%2:32:03::","synset_id":"confront.v.03"},{"index":7,"sense_key":"face%2:39:04::","synset_id":"face.v.07"},{"index":8,"sense_key":"face%2:35:00::","synset_id":"face.v.08"},{"index":9,"sense_key":"face%2:35:01::","synset_id":"face.v.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"face%2:32:00::","call_ids":["senseval3.d001.s012.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"face%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1211,"cached_input_tokens":0,"output_tokens":187,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004292,"input_uncached_usd":0.0024219999999999997,"input_cached_usd":0.0,"output_usd":0.0018700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.8801915840012953}
{"item_id":"senseval3.d001.s012.t007","gold_sense_keys":["have%2:40:00::","have%2:42:12::"],"candidates":[{"index":1,"sense_key":"have%2:40:00::","synset_id":"have.v.01"},{"index":2,"sense_key":"have%2:42:00::","synset_id":"have.v.02"},{"index":3,"sense_key":"have%2:39:06::","synset_id":"experience.v.03"},{"index":4,"sense_key":"have%2:40:04::","synset_id":"own.v.01"},{"index":5,"sense_key":"have%2:30:00::","synset_id":"get.v.03"},{"index":6,"sense_key":"have%2:34:00::","synset_id":"consume.v.02"},{"index":7,"sense_key":"have%2:41:00::","synset_id":"have.v.07"},{"index":8,"sense_key":"have%2:36:00::","synset_id":"hold.v.03"},{"index":9,"sense_key":"have%2:40:01::","synset_id":"have.v.09"},{"index":10,"sense_key":"have%2:42:12::","synset_id":"have.v.10"},{"index":11,"sense_key":"have%2:30:01::","synset_id":"have.v.11"},{"index":12,"sense_key":"have%2:29:02::","synset_id":"have.v.12"},{"index":13,"sense_key":"have%2:32:00::","synset_id":"induce.v.02"},{"index":14,"sense_key":"have%2:40:05::","synset_id":"accept.v.02"},{"index":15,"sense_key":"have%2:40:03::","synset_id":"receive.v.01"},{"index":16,"sense_key":"have%2:29:05::","synset_id":"suffer.v.02"},{"index":17,"sense_key":"have%2:40:02::","synset_id":"have.v.17"},{"index":18,"sense_key":"have%2:29:00::","synset_id":"give_birth.v.01"},{"index":19,"sense_key":"have%2:35:00::","synset_id":"take.v.35"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"have%2:40:00::","call_ids":["senseval3.d001.s012.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"have%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1851,"cached_input_tokens":0,"output_tokens":86,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004562,"input_uncached_usd":0.003702,"input_cached_usd":0.0,"output_usd":0.0008600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9301057499833405}
{"item_id":"senseval3.d001.s012.t008","gold_sense_keys":["incentive%1:16:00::"],"candidates":[{"index":1,"sense_key":"incentive%1:16:00::","synset_id":"incentive.n.01"},{"index":2,"sense_key":"incentive%1:21:00::","synset_id":"bonus.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"incentive%1:16:00::","call_ids":["senseval3.d001.s012.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"incentive%1:16:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":786,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001692,"input_uncached_usd":0.0015719999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1287857090355828}
{"item_id":"senseval3.d001.s012.t010","gold_sense_keys":["lean%2:42:01::"],"candidates":[{"index":1,"sense_key":"lean%2:38:00::","synset_id":"lean.v.01"},{"index":2,"sense_key":"lean%2:35:00::","synset_id":"lean.v.02"},{"index":3,"sense_key":"lean%2:42:01::","synset_id":"tend.v.01"},{"index":4,"sense_key":"lean%2:31:10::","synset_id":"lean.v.04"},{"index":5,"sense_key":"lean%2:38:07::","synset_id":"list.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"lean%2:42:01::","call_ids":["senseval3.d001.s012.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"lean%2:42:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":964,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002698,"input_uncached_usd":0.0019279999999999998,"input_cached_usd":0.0,"output_usd":0.0007700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3210412089247257}
{"item_id":"senseval3.d001.s012.t011","gold_sense_keys":["democratic%3:01:00::"],"candidates":[{"index":1,"sense_key":"democratic%3:00:00::","synset_id":"democratic.a.01"},{"index":2,"sense_key":"democratic%3:01:00::","synset_id":"democratic.a.02"},{"index":3,"sense_key":"democratic%5:00:00:common:01","synset_id":"democratic.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"democratic%3:01:00::","call_ids":["senseval3.d001.s012.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"democratic%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":881,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001882,"input_uncached_usd":0.001762,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.277189000044018}
{"item_id":"senseval3.d001.s013.t000","gold_sense_keys":["put%2:35:00::"],"candidates":[{"index":1,"sense_key":"put%2:35:00::","synset_id":"put.v.01"},{"index":2,"sense_key":"put%2:35:05::","synset_id":"put.v.02"},{"index":3,"sense_key":"put%2:32:00::","synset_id":"frame.v.04"},{"index":4,"sense_key":"put%2:34:00::","synset_id":"put.v.04"},{"index":5,"sense_key":"put%2:40:00::","synset_id":"invest.v.01"},{"index":6,"sense_key":"put%2:31:13::","synset_id":"place.v.11"},{"index":7,"sense_key":"put%2:39:13::","synset_id":"put.v.07"},{"index":8,"sense_key":"put%2:36:13::","synset_id":"put.v.08"},{"index":9,"sense_key":"put%2:31:00::","synset_id":"arrange.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"put%2:35:00::","call_ids":["senseval3.d001.s013.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"put%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1183,"cached_input_tokens":0,"output_tokens":110,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0034660000000000003,"input_uncached_usd":0.002366,"input_cached_usd":0.0,"output_usd":0.0011,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6250720410607755}
{"item_id":"senseval3.d001.s013.t001","gold_sense_keys":["office%1:26:00::"],"candidates":[{"index":1,"sense_key":"office%1:06:00::","synset_id":"office.n.01"},{"index":2,"sense_key":"office%1:14:01::","synset_id":"agency.n.01"},{"index":3,"sense_key":"office%1:04:01::","synset_id":"function.n.03"},{"index":4,"sense_key":"office%1:26:00::","synset_id":"office.n.04"},{"index":5,"sense_key":"office%1:14:00::","synset_id":"office.n.05"},{"index":6,"sense_key":"office%1:04:00::","synset_id":"office.n.06"},{"index":7,"sense_key":"office%1:04:02::","synset_id":"position.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"office%1:26:00::","call_ids":["senseval3.d001.s013.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"office%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1121,"cached_input_tokens":0,"output_tokens":121,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003452,"input_uncached_usd":0.002242,"input_cached_usd":0.0,"output_usd":0.0012100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.888719749986194}
{"item_id":"senseval3.d001.s013.t002","gold_sense_keys":["acquire%2:40:00::"],"candidates":[{"index":1,"sense_key":"acquire%2:40:00::","synset_id":"get.v.01"},{"index":2,"sense_key":"acquire%2:30:00::","synset_id":"assume.v.03"},{"index":3,"sense_key":"acquire%2:29:00::","synset_id":"grow.v.08"},{"index":4,"sense_key":"acquire%2:42:00::","synset_id":"acquire.v.04"},{"index":5,"sense_key":"acquire%2:40:06::","synset_id":"acquire.v.05"},{"index":6,"sense_key":"acquire%2:31:00::","synset_id":"learn.v.01"},{"index":7,"sense_key":"acquire%2:30:05::","synset_id":"develop.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"acquire%2:40:00::","call_ids":["senseval3.d001.s013.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"acquire%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1159,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0029779999999999997,"input_uncached_usd":0.0023179999999999997,"input_cached_usd":0.0,"output_usd":0.0006600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.519020750070922}
{"item_id":"senseval3.d001.s013.t003","gold_sense_keys":["local%3:00:01::"],"candidates":[{"index":1,"sense_key":"local%3:00:01::","synset_id":"local.a.01"},{"index":2,"sense_key":"local%3:01:01::","synset_id":"local.a.02"},{"index":3,"sense_key":"local%3:00:02::","synset_id":"local.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"local%3:01:01::","call_ids":["senseval3.d001.s013.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"local%3:01:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":836,"cached_input_tokens":0,"output_tokens":108,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002752,"input_uncached_usd":0.0016719999999999999,"input_cached_usd":0.0,"output_usd":0.00108,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.801786124939099}
{"item_id":"senseval3.d001.s013.t004","gold_sense_keys":["benefit%1:07:00::"],"candidates":[{"index":1,"sense_key":"benefit%1:21:00::","synset_id":"benefit.n.01"},{"index":2,"sense_key":"benefit%1:07:00::","synset_id":"benefit.n.02"},{"index":3,"sense_key":"benefit%1:10:00::","synset_id":"benefit.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"benefit%1:07:00::","call_ids":["senseval3.d001.s013.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"benefit%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":799,"cached_input_tokens":0,"output_tokens":205,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003648,"input_uncached_usd":0.001598,"input_cached_usd":0.0,"output_usd":0.00205,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.334225125028752}
{"item_id":"senseval3.d001.s013.t007","gold_sense_keys":["prevent%2:41:00::"],"candidates":[{"index":1,"sense_key":"prevent%2:41:01::","synset_id":"prevent.v.01"},{"index":2,"sense_key":"prevent%2:41:00::","synset_id":"prevent.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"prevent%2:41:00::","call_ids":["senseval3.d001.s013.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"prevent%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":854,"cached_input_tokens":0,"output_tokens":94,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002648,"input_uncached_usd":0.0017079999999999999,"input_cached_usd":0.0,"output_usd":0.0009400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7293154579820111}
{"item_id":"senseval3.d001.s013.t008","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":2,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":3,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"other%3:00:00::","call_ids":["senseval3.d001.s013.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":907,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001934,"input_uncached_usd":0.0018139999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0735639580525458}
{"item_id":"senseval3.d001.s013.t010","gold_sense_keys":["bring_home_the_bacon%2:41:00::"],"candidates":[{"index":1,"sense_key":"bring_home_the_bacon%2:41:00::","synset_id":"succeed.v.01"},{"index":2,"sense_key":"bring_home_the_bacon%2:40:00::","synset_id":"provide.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bring_home_the_bacon%2:41:00::","call_ids":["senseval3.d001.s013.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bring_home_the_bacon%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":869,"cached_input_tokens":0,"output_tokens":85,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002588,"input_uncached_usd":0.001738,"input_cached_usd":0.0,"output_usd":0.0008500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4890617920318618}
{"item_id":"senseval3.d001.s014.t001","gold_sense_keys":["come_to%2:42:03::"],"candidates":[{"index":1,"sense_key":"come_to%2:39:00::","synset_id":"hit.v.09"},{"index":2,"sense_key":"come_to%2:42:03::","synset_id":"refer.v.02"},{"index":3,"sense_key":"come_to%2:41:00::","synset_id":"come_to.v.03"},{"index":4,"sense_key":"come_to%2:29:00::","synset_id":"come_to.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"come_to%2:42:03::","call_ids":["senseval3.d001.s014.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"come_to%2:42:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":954,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0024879999999999998,"input_uncached_usd":0.001908,"input_cached_usd":0.0,"output_usd":0.00058,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.574743999983184}
{"item_id":"senseval3.d001.s014.t002","gold_sense_keys":["national%3:00:00::","national%3:00:01::","national%3:01:00::","national%5:00:00:domestic:00"],"candidates":[{"index":1,"sense_key":"national%3:01:00::","synset_id":"national.a.01"},{"index":2,"sense_key":"national%3:00:00::","synset_id":"national.a.02"},{"index":3,"sense_key":"national%3:00:01::","synset_id":"national.a.03"},{"index":4,"sense_key":"national%5:00:00:public:00","synset_id":"national.s.01"},{"index":5,"sense_key":"national%5:00:00:domestic:00","synset_id":"home.s.01"},{"index":6,"sense_key":"national%3:01:02::","synset_id":"national.a.06"},{"index":7,"sense_key":"national%3:01:01::","synset_id":"national.a.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"national%3:00:01::","call_ids":["senseval3.d001.s014.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"national%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1050,"cached_input_tokens":0,"output_tokens":79,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0028899999999999998,"input_uncached_usd":0.0021,"input_cached_usd":0.0,"output_usd":0.00079,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7802578329574317}
{"item_id":"senseval3.d001.s014.t003","gold_sense_keys":["policy%1:09:00::"],"candidates":[{"index":1,"sense_key":"policy%1:09:00::","synset_id":"policy.n.01"},{"index":2,"sense_key":"policy%1:10:00::","synset_id":"policy.n.02"},{"index":3,"sense_key":"policy%1:10:02::","synset_id":"policy.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"policy%1:09:00::","call_ids":["senseval3.d001.s014.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"policy%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":876,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001872,"input_uncached_usd":0.0017519999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3144793750252575}
{"item_id":"senseval3.d001.s014.t004","gold_sense_keys":["making%1:04:00::"],"candidates":[{"index":1,"sense_key":"making%1:04:00::","synset_id":"devising.n.01"},{"index":2,"sense_key":"making%1:07:00::","synset_id":"qualification.n.01"},{"index":3,"sense_key":"making%1:06:00::","synset_id":"making.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"making%1:04:00::","call_ids":["senseval3.d001.s014.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"making%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":895,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00232,"input_uncached_usd":0.00179,"input_cached_usd":0.0,"output_usd":0.0005300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7035298750270158}
{"item_id":"senseval3.d001.s015.t000","gold_sense_keys":["race%1:11:01::"],"candidates":[{"index":1,"sense_key":"race%1:11:01::","synset_id":"race.n.01"},{"index":2,"sense_key":"race%1:11:00::","synset_id":"race.n.02"},{"index":3,"sense_key":"race%1:14:00::","synset_id":"race.n.03"},{"index":4,"sense_key":"race%1:14:01::","synset_id":"subspecies.n.01"},{"index":5,"sense_key":"race%1:19:00::","synset_id":"slipstream.n.01"},{"index":6,"sense_key":"race%1:06:00::","synset_id":"raceway.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"race%1:11:01::","call_ids":["senseval3.d001.s015.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"race%1:11:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":949,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002018,"input_uncached_usd":0.001898,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0412775420118123}
{"item_id":"senseval3.d001.s015.t002","gold_sense_keys":["incentive%1:16:00::"],"candidates":[{"index":1,"sense_key":"incentive%1:16:00::","synset_id":"incentive.n.01"},{"index":2,"sense_key":"incentive%1:21:00::","synset_id":"bonus.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"incentive%1:16:00::","call_ids":["senseval3.d001.s015.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"incentive%1:16:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":747,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001614,"input_uncached_usd":0.0014939999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1364652080228552}
{"item_id":"senseval3.d001.s015.t004","gold_sense_keys":["lean%2:42:01::"],"candidates":[{"index":1,"sense_key":"lean%2:38:00::","synset_id":"lean.v.01"},{"index":2,"sense_key":"lean%2:35:00::","synset_id":"lean.v.02"},{"index":3,"sense_key":"lean%2:42:01::","synset_id":"tend.v.01"},{"index":4,"sense_key":"lean%2:31:10::","synset_id":"lean.v.04"},{"index":5,"sense_key":"lean%2:38:07::","synset_id":"list.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"lean%2:42:01::","call_ids":["senseval3.d001.s015.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"lean%2:42:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":925,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0023899999999999998,"input_uncached_usd":0.0018499999999999999,"input_cached_usd":0.0,"output_usd":0.00054,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1327027910156175}
{"item_id":"senseval3.d001.s016.t000","gold_sense_keys":["president%1:18:04::"],"candidates":[{"index":1,"sense_key":"president%1:18:01::","synset_id":"president.n.01"},{"index":2,"sense_key":"president%1:18:04::","synset_id":"president_of_the_united_states.n.01"},{"index":3,"sense_key":"president%1:18:00::","synset_id":"president.n.03"},{"index":4,"sense_key":"president%1:18:03::","synset_id":"president.n.04"},{"index":5,"sense_key":"president%1:18:02::","synset_id":"president.n.05"},{"index":6,"sense_key":"president%1:04:04::","synset_id":"president_of_the_united_states.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"president%1:18:04::","call_ids":["senseval3.d001.s016.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"president%1:18:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":866,"cached_input_tokens":0,"output_tokens":83,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002562,"input_uncached_usd":0.0017319999999999998,"input_cached_usd":0.0,"output_usd":0.0008300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7498817919986323}
{"item_id":"senseval3.d001.s016.t001","gold_sense_keys":["limit%2:30:01::"],"candidates":[{"index":1,"sense_key":"limit%2:30:01::","synset_id":"restrict.v.03"},{"index":2,"sense_key":"limit%2:30:00::","synset_id":"limit.v.02"},{"index":3,"sense_key":"limit%2:32:00::","synset_id":"specify.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"limit%2:30:01::","call_ids":["senseval3.d001.s016.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"limit%2:30:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":769,"cached_input_tokens":0,"output_tokens":117,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002708,"input_uncached_usd":0.0015379999999999999,"input_cached_usd":0.0,"output_usd":0.00117,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.139269833918661}
{"item_id":"senseval3.d001.s016.t002","gold_sense_keys":["local%3:00:01::"],"candidates":[{"index":1,"sense_key":"local%3:00:01::","synset_id":"local.a.01"},{"index":2,"sense_key":"local%3:01:01::","synset_id":"local.a.02"},{"index":3,"sense_key":"local%3:00:02::","synset_id":"local.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"local%3:01:01::","call_ids":["senseval3.d001.s016.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"local%3:01:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":717,"cached_input_tokens":0,"output_tokens":124,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002674,"input_uncached_usd":0.001434,"input_cached_usd":0.0,"output_usd":0.00124,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8949652500450611}
{"item_id":"senseval3.d001.s016.t003","gold_sense_keys":["benefit%1:07:00::"],"candidates":[{"index":1,"sense_key":"benefit%1:21:00::","synset_id":"benefit.n.01"},{"index":2,"sense_key":"benefit%1:07:00::","synset_id":"benefit.n.02"},{"index":3,"sense_key":"benefit%1:10:00::","synset_id":"benefit.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"benefit%1:07:00::","call_ids":["senseval3.d001.s016.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"benefit%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":680,"cached_input_tokens":0,"output_tokens":156,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00292,"input_uncached_usd":0.0013599999999999999,"input_cached_usd":0.0,"output_usd":0.0015600000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9650141670135781}
{"item_id":"senseval3.d001.s016.t005","gold_sense_keys":["particular%5:00:02:specific:00"],"candidates":[{"index":1,"sense_key":"particular%5:00:00:specific:00","synset_id":"particular.s.01"},{"index":2,"sense_key":"particular%5:00:02:specific:00","synset_id":"particular.s.02"},{"index":3,"sense_key":"particular%5:00:00:uncommon:00","synset_id":"especial.s.01"},{"index":4,"sense_key":"particular%5:00:00:primary:00","synset_id":"particular.s.04"},{"index":5,"sense_key":"particular%5:00:00:fastidious:00","synset_id":"finical.s.01"},{"index":6,"sense_key":"particular%5:00:00:careful:00","synset_id":"particular.s.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"particular%5:00:02:specific:00","call_ids":["senseval3.d001.s016.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"particular%5:00:02:specific:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":968,"cached_input_tokens":0,"output_tokens":120,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003136,"input_uncached_usd":0.001936,"input_cached_usd":0.0,"output_usd":0.0012000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9229248749325052}
{"item_id":"senseval3.d001.s016.t008","gold_sense_keys":["president%1:18:04::"],"candidates":[{"index":1,"sense_key":"president%1:18:01::","synset_id":"president.n.01"},{"index":2,"sense_key":"president%1:18:04::","synset_id":"president_of_the_united_states.n.01"},{"index":3,"sense_key":"president%1:18:00::","synset_id":"president.n.03"},{"index":4,"sense_key":"president%1:18:03::","synset_id":"president.n.04"},{"index":5,"sense_key":"president%1:18:02::","synset_id":"president.n.05"},{"index":6,"sense_key":"president%1:04:04::","synset_id":"president_of_the_united_states.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"president%1:18:04::","call_ids":["senseval3.d001.s016.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"president%1:18:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":866,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002362,"input_uncached_usd":0.0017319999999999998,"input_cached_usd":0.0,"output_usd":0.00063,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8712617079727352}
{"item_id":"senseval3.d001.s016.t009","gold_sense_keys":["likely%3:00:00::"],"candidates":[{"index":1,"sense_key":"likely%3:00:00::","synset_id":"likely.a.01"},{"index":2,"sense_key":"likely%3:00:04::","synset_id":"probable.a.01"},{"index":3,"sense_key":"likely%5:00:00:prospective:00","synset_id":"likely.s.01"},{"index":4,"sense_key":"likely%5:00:00:credible:00","synset_id":"likely.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"likely%3:00:00::","call_ids":["senseval3.d001.s016.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"likely%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":778,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002286,"input_uncached_usd":0.0015559999999999999,"input_cached_usd":0.0,"output_usd":0.0007300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4679664579452947}
{"item_id":"senseval3.d001.s016.t010","gold_sense_keys":["effective%3:00:00::","effective%5:00:00:competent:00"],"candidates":[{"index":1,"sense_key":"effective%3:00:00::","synset_id":"effective.a.01"},{"index":2,"sense_key":"effective%5:00:00:competent:00","synset_id":"effective.s.01"},{"index":3,"sense_key":"effective%5:00:00:efficacious:00","synset_id":"effective.s.02"},{"index":4,"sense_key":"effective%5:00:00:operative:00","synset_id":"effective.s.03"},{"index":5,"sense_key":"effective%5:00:00:actual:00","synset_id":"effective.s.04"},{"index":6,"sense_key":"effective%5:00:00:operational:00","synset_id":"effective.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"effective%5:00:00:competent:00","call_ids":["senseval3.d001.s016.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"effective%5:00:00:competent:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1021,"cached_input_tokens":0,"output_tokens":174,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003782,"input_uncached_usd":0.002042,"input_cached_usd":0.0,"output_usd":0.0017400000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.146461208933033}
{"item_id":"senseval3.d001.s016.t011","gold_sense_keys":["prevent%2:41:00::"],"candidates":[{"index":1,"sense_key":"prevent%2:41:01::","synset_id":"prevent.v.01"},{"index":2,"sense_key":"prevent%2:41:00::","synset_id":"prevent.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"prevent%2:41:00::","call_ids":["senseval3.d001.s016.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"prevent%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":735,"cached_input_tokens":0,"output_tokens":80,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00227,"input_uncached_usd":0.00147,"input_cached_usd":0.0,"output_usd":0.0008,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4501239160308614}
{"item_id":"senseval3.d001.s016.t012","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":2,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":3,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"other%3:00:00::","call_ids":["senseval3.d001.s016.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":788,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001696,"input_uncached_usd":0.001576,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0600747501011938}
{"item_id":"senseval3.d001.s016.t016","gold_sense_keys":["local%3:00:01::"],"candidates":[{"index":1,"sense_key":"local%3:00:01::","synset_id":"local.a.01"},{"index":2,"sense_key":"local%3:01:01::","synset_id":"local.a.02"},{"index":3,"sense_key":"local%3:00:02::","synset_id":"local.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"local%3:01:01::","call_ids":["senseval3.d001.s016.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"local%3:01:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":717,"cached_input_tokens":0,"output_tokens":91,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002344,"input_uncached_usd":0.001434,"input_cached_usd":0.0,"output_usd":0.0009100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.548556833062321}
{"item_id":"senseval3.d001.s016.t017","gold_sense_keys":["benefit%1:07:00::"],"candidates":[{"index":1,"sense_key":"benefit%1:21:00::","synset_id":"benefit.n.01"},{"index":2,"sense_key":"benefit%1:07:00::","synset_id":"benefit.n.02"},{"index":3,"sense_key":"benefit%1:10:00::","synset_id":"benefit.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"benefit%1:07:00::","call_ids":["senseval3.d001.s016.t017__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"benefit%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":680,"cached_input_tokens":0,"output_tokens":98,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00234,"input_uncached_usd":0.0013599999999999999,"input_cached_usd":0.0,"output_usd":0.0009800000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5478976659942418}
{"item_id":"senseval3.d001.s017.t000","gold_sense_keys":["individual%3:00:00::","individual%5:00:00:independent:00"],"candidates":[{"index":1,"sense_key":"individual%3:00:00::","synset_id":"individual.a.01"},{"index":2,"sense_key":"individual%5:00:00:independent:00","synset_id":"individual.s.01"},{"index":3,"sense_key":"individual%5:00:00:unshared:00","synset_id":"individual.s.02"},{"index":4,"sense_key":"individual%5:00:00:personal:00","synset_id":"individual.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"individual%3:00:00::","call_ids":["senseval3.d001.s017.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"individual%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":783,"cached_input_tokens":0,"output_tokens":108,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002646,"input_uncached_usd":0.001566,"input_cached_usd":0.0,"output_usd":0.00108,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9061506250873208}
{"item_id":"senseval3.d001.s017.t002","gold_sense_keys":["standing%1:26:00::"],"candidates":[{"index":1,"sense_key":"standing%1:26:00::","synset_id":"standing.n.01"},{"index":2,"sense_key":"standing%1:10:00::","synset_id":"standing.n.02"},{"index":3,"sense_key":"standing%1:04:00::","synset_id":"standing.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"standing%1:26:00::","call_ids":["senseval3.d001.s017.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"standing%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":666,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0019519999999999997,"input_uncached_usd":0.0013319999999999999,"input_cached_usd":0.0,"output_usd":0.00062,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4570799160283059}
{"item_id":"senseval3.d001.s017.t003","gold_sense_keys":["enhance%2:30:01::"],"candidates":[{"index":1,"sense_key":"enhance%2:30:00::","synset_id":"enhance.v.01"},{"index":2,"sense_key":"enhance%2:30:01::","synset_id":"enhance.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"enhance%2:30:01::","call_ids":["senseval3.d001.s017.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"enhance%2:30:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":631,"cached_input_tokens":0,"output_tokens":99,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0022519999999999997,"input_uncached_usd":0.0012619999999999999,"input_cached_usd":0.0,"output_usd":0.00099,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0447226249380037}
{"item_id":"senseval3.d001.s018.t001","gold_sense_keys":["simple%3:00:02::","simple%5:00:00:easy:01"],"candidates":[{"index":1,"sense_key":"simple%3:00:02::","synset_id":"simple.a.01"},{"index":2,"sense_key":"simple%5:00:00:easy:01","synset_id":"elementary.s.01"},{"index":3,"sense_key":"simple%5:00:02:plain:01","synset_id":"bare.s.04"},{"index":4,"sense_key":"simple%5:00:00:naive:00","synset_id":"childlike.s.02"},{"index":5,"sense_key":"simple%5:00:00:retarded:00","synset_id":"dim-witted.s.01"},{"index":6,"sense_key":"simple%3:00:01::","synset_id":"simple.a.06"},{"index":7,"sense_key":"simple%5:00:00:plain:01","synset_id":"simple.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"simple%3:00:02::","call_ids":["senseval3.d001.s018.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"simple%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1043,"cached_input_tokens":0,"output_tokens":79,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0028759999999999997,"input_uncached_usd":0.0020859999999999997,"input_cached_usd":0.0,"output_usd":0.00079,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9838358340784907}
{"item_id":"senseval3.d001.s018.t002","gold_sense_keys":["appear%2:39:00::"],"candidates":[{"index":1,"sense_key":"appear%2:39:00::","synset_id":"look.v.02"},{"index":2,"sense_key":"appear%2:30:00::","synset_id":"appear.v.02"},{"index":3,"sense_key":"appear%2:30:01::","synset_id":"appear.v.03"},{"index":4,"sense_key":"appear%2:39:01::","synset_id":"appear.v.04"},{"index":5,"sense_key":"appear%2:30:02::","synset_id":"appear.v.05"},{"index":6,"sense_key":"appear%2:36:00::","synset_id":"appear.v.06"},{"index":7,"sense_key":"appear%2:41:03::","synset_id":"appear.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"appear%2:39:00::","call_ids":["senseval3.d001.s018.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"appear%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1057,"cached_input_tokens":0,"output_tokens":118,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003294,"input_uncached_usd":0.002114,"input_cached_usd":0.0,"output_usd":0.00118,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.812134417006746}
{"item_id":"senseval3.d001.s018.t003","gold_sense_keys":["explain%2:32:02::"],"candidates":[{"index":1,"sense_key":"explain%2:32:00::","synset_id":"explain.v.01"},{"index":2,"sense_key":"explain%2:32:01::","synset_id":"explain.v.02"},{"index":3,"sense_key":"explain%2:32:02::","synset_id":"excuse.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"explain%2:32:02::","call_ids":["senseval3.d001.s018.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"explain%2:32:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":758,"cached_input_tokens":0,"output_tokens":90,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002416,"input_uncached_usd":0.001516,"input_cached_usd":0.0,"output_usd":0.0009000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6103577909525484}
{"item_id":"senseval3.d001.s018.t004","gold_sense_keys":["several%5:00:00:some:00"],"candidates":[{"index":1,"sense_key":"several%5:00:00:some:00","synset_id":"several.s.01"},{"index":2,"sense_key":"several%5:00:00:individual:00","synset_id":"respective.s.01"},{"index":3,"sense_key":"several%5:00:00:different:00","synset_id":"several.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"several%5:00:00:some:00","call_ids":["senseval3.d001.s018.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"several%5:00:00:some:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":734,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001588,"input_uncached_usd":0.001468,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.41390283301007}
{"item_id":"senseval3.d001.s018.t005","gold_sense_keys":["thing%1:09:02::"],"candidates":[{"index":1,"sense_key":"thing%1:26:00::","synset_id":"thing.n.01"},{"index":2,"sense_key":"thing%1:04:00::","synset_id":"thing.n.02"},{"index":3,"sense_key":"thing%1:09:00::","synset_id":"thing.n.03"},{"index":4,"sense_key":"thing%1:06:00::","synset_id":"thing.n.04"},{"index":5,"sense_key":"thing%1:11:00::","synset_id":"thing.n.05"},{"index":6,"sense_key":"thing%1:09:02::","synset_id":"matter.n.01"},{"index":7,"sense_key":"thing%1:10:00::","synset_id":"thing.n.07"},{"index":8,"sense_key":"thing%1:06:01::","synset_id":"thing.n.08"},{"index":9,"sense_key":"thing%1:07:00::","synset_id":"thing.n.09"},{"index":10,"sense_key":"thing%1:09:01::","synset_id":"thing.n.10"},{"index":11,"sense_key":"thing%1:12:00::","synset_id":"thing.n.11"},{"index":12,"sense_key":"thing%1:03:00::","synset_id":"thing.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"thing%1:09:02::","call_ids":["senseval3.d001.s018.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"thing%1:09:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1124,"cached_input_tokens":0,"output_tokens":217,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004418,"input_uncached_usd":0.002248,"input_cached_usd":0.0,"output_usd":0.00217,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.8304350830148906}
{"item_id":"senseval3.d001.s019.t000","gold_sense_keys":["increase%2:30:00::"],"candidates":[{"index":1,"sense_key":"increase%2:30:00::","synset_id":"increase.v.01"},{"index":2,"sense_key":"increase%2:30:02::","synset_id":"increase.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"increase%2:30:00::","call_ids":["senseval3.d001.s019.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"increase%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":686,"cached_input_tokens":0,"output_tokens":92,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0022919999999999998,"input_uncached_usd":0.001372,"input_cached_usd":0.0,"output_usd":0.00092,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.113574916962534}
{"item_id":"senseval3.d001.s019.t001","gold_sense_keys":["take%2:30:01::"],"candidates":[{"index":1,"sense_key":"take%2:41:04::","synset_id":"take.v.01"},{"index":2,"sense_key":"take%2:40:06::","synset_id":"take.v.02"},{"index":3,"sense_key":"take%2:38:09::","synset_id":"lead.v.01"},{"index":4,"sense_key":"take%2:35:00::","synset_id":"take.v.04"},{"index":5,"sense_key":"take%2:30:01::","synset_id":"assume.v.03"},{"index":6,"sense_key":"take%2:31:07::","synset_id":"take.v.06"},{"index":7,"sense_key":"take%2:38:10::","synset_id":"bring.v.01"},{"index":8,"sense_key":"take%2:40:00::","synset_id":"take.v.08"},{"index":9,"sense_key":"take%2:38:02::","synset_id":"take.v.09"},{"index":10,"sense_key":"take%2:31:01::","synset_id":"choose.v.01"},{"index":11,"sense_key":"take%2:40:05::","synset_id":"accept.v.02"},{"index":12,"sense_key":"take%2:41:00::","synset_id":"fill.v.04"},{"index":13,"sense_key":"take%2:31:03::","synset_id":"consider.v.03"},{"index":14,"sense_key":"take%2:42:00::","synset_id":"necessitate.v.01"},{"index":15,"sense_key":"take%2:39:00::","synset_id":"take.v.15"},{"index":16,"sense_key":"take%2:32:02::","synset_id":"film.v.01"},{"index":17,"sense_key":"take%2:30:00::","synset_id":"remove.v.01"},{"index":18,"sense_key":"take%2:34:00::","synset_id":"consume.v.02"},{"index":19,"sense_key":"take%2:31:09::","synset_id":"take.v.19"},{"index":20,"sense_key":"take%2:40:15::","synset_id":"take.v.20"},{"index":21,"sense_key":"take%2:40:01::","synset_id":"take.v.21"},{"index":22,"sense_key":"take%2:38:00::","synset_id":"assume.v.05"},{"index":23,"sense_key":"take%2:40:07::","synset_id":"accept.v.05"},{"index":24,"sense_key":"take%2:30:08::","synset_id":"take.v.24"},{"index":25,"sense_key":"take%2:31:02::","synset_id":"learn.v.04"},{"index":26,"sense_key":"take%2:32:00::","synset_id":"claim.v.05"},{"index":27,"sense_key":"take%2:38:05::","synset_id":"take.v.27"},{"index":28,"sense_key":"take%2:33:09::","synset_id":"aim.v.01"},{"index":29,"sense_key":"take%2:30:09::","synset_id":"take.v.29"},{"index":30,"sense_key":"take%2:42:10::","synset_id":"carry.v.02"},{"index":31,"sense_key":"take%2:40:03::","synset_id":"lease.v.04"},{"index":32,"sense_key":"take%2:40:04::","synset_id":"subscribe.v.05"},{"index":33,"sense_key":"take%2:40:02::","synset_id":"take.v.33"},{"index":34,"sense_key":"take%2:40:08::","synset_id":"take.v.34"},{"index":35,"sense_key":"take%2:35:14::","synset_id":"take.v.35"},{"index":36,"sense_key":"take%2:32:09::","synset_id":"claim.v.04"},{"index":37,"sense_key":"take%2:42:03::","synset_id":"accept.v.08"},{"index":38,"sense_key":"take%2:42:15::","synset_id":"contain.v.05"},{"index":39,"sense_key":"take%2:41:13::","synset_id":"take.v.39"},{"index":40,"sense_key":"take%2:38:11::","synset_id":"drive.v.16"},{"index":41,"sense_key":"take%2:33:08::","synset_id":"take.v.41"},{"index":42,"sense_key":"take%2:29:08::","synset_id":"contract.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"take%2:30:01::","call_ids":["senseval3.d001.s019.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"take%2:30:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":3189,"cached_input_tokens":0,"output_tokens":102,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.007398,"input_uncached_usd":0.006378,"input_cached_usd":0.0,"output_usd":0.00102,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5020600840216503}
{"item_id":"senseval3.d001.s019.t004","gold_sense_keys":["past%3:00:00::"],"candidates":[{"index":1,"sense_key":"past%3:00:00::","synset_id":"past.a.01"},{"index":2,"sense_key":"past%5:00:00:outgoing:00","synset_id":"past.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"past%3:00:00::","call_ids":["senseval3.d001.s019.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"past%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":698,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001516,"input_uncached_usd":0.0013959999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3478937080362812}
{"item_id":"senseval3.d001.s019.t006","gold_sense_keys":["election%1:26:00::"],"candidates":[{"index":1,"sense_key":"election%1:04:01::","synset_id":"election.n.01"},{"index":2,"sense_key":"election%1:04:00::","synset_id":"election.n.02"},{"index":3,"sense_key":"election%1:26:00::","synset_id":"election.n.03"},{"index":4,"sense_key":"election%1:09:00::","synset_id":"election.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"election%1:26:00::","call_ids":["senseval3.d001.s019.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"election%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":788,"cached_input_tokens":0,"output_tokens":228,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003856,"input_uncached_usd":0.001576,"input_cached_usd":0.0,"output_usd":0.0022800000000000003,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.918868999928236}
{"item_id":"senseval3.d001.s019.t007","gold_sense_keys":["president%1:18:04::"],"candidates":[{"index":1,"sense_key":"president%1:18:01::","synset_id":"president.n.01"},{"index":2,"sense_key":"president%1:18:04::","synset_id":"president_of_the_united_states.n.01"},{"index":3,"sense_key":"president%1:18:00::","synset_id":"president.n.03"},{"index":4,"sense_key":"president%1:18:03::","synset_id":"president.n.04"},{"index":5,"sense_key":"president%1:18:02::","synset_id":"president.n.05"},{"index":6,"sense_key":"president%1:04:04::","synset_id":"president_of_the_united_states.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"president%1:04:04::","call_ids":["senseval3.d001.s019.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"president%1:04:04::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":899,"cached_input_tokens":0,"output_tokens":187,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0036680000000000003,"input_uncached_usd":0.001798,"input_cached_usd":0.0,"output_usd":0.0018700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.4131406659726053}
{"item_id":"senseval3.d001.s019.t008","gold_sense_keys":["advent%1:04:00::"],"candidates":[{"index":1,"sense_key":"advent%1:04:00::","synset_id":"advent.n.01"},{"index":2,"sense_key":"advent%1:28:01::","synset_id":"advent.n.02"},{"index":3,"sense_key":"advent%1:11:00::","synset_id":"second_coming.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"advent%1:04:00::","call_ids":["senseval3.d001.s019.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"advent%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":767,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001654,"input_uncached_usd":0.001534,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.502036124933511}
{"item_id":"senseval3.d001.s019.t009","gold_sense_keys":["government%1:14:00::"],"candidates":[{"index":1,"sense_key":"government%1:14:00::","synset_id":"government.n.01"},{"index":2,"sense_key":"government%1:04:00::","synset_id":"government.n.02"},{"index":3,"sense_key":"government%1:09:00::","synset_id":"government.n.03"},{"index":4,"sense_key":"government%1:09:01::","synset_id":"politics.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"government%1:14:00::","call_ids":["senseval3.d001.s019.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"government%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":827,"cached_input_tokens":0,"output_tokens":137,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0030239999999999998,"input_uncached_usd":0.0016539999999999999,"input_cached_usd":0.0,"output_usd":0.0013700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9313805409474298}
{"item_id":"senseval3.d001.s019.t010","gold_sense_keys":["occupy%2:41:04::"],"candidates":[{"index":1,"sense_key":"occupy%2:41:00::","synset_id":"busy.v.01"},{"index":2,"sense_key":"occupy%2:42:00::","synset_id":"occupy.v.02"},{"index":3,"sense_key":"occupy%2:42:01::","synset_id":"occupy.v.03"},{"index":4,"sense_key":"occupy%2:42:04::","synset_id":"concern.v.02"},{"index":5,"sense_key":"occupy%2:33:00::","synset_id":"invade.v.01"},{"index":6,"sense_key":"occupy%2:40:08::","synset_id":"take.v.02"},{"index":7,"sense_key":"occupy%2:31:00::","synset_id":"absorb.v.09"},{"index":8,"sense_key":"occupy%2:41:04::","synset_id":"fill.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"occupy%2:41:04::","call_ids":["senseval3.d001.s019.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"occupy%2:41:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1102,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002774,"input_uncached_usd":0.002204,"input_cached_usd":0.0,"output_usd":0.0005700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1107554999180138}
{"item_id":"senseval3.d001.s019.t011","gold_sense_keys":["role%1:04:00::"],"candidates":[{"index":1,"sense_key":"role%1:04:00::","synset_id":"function.n.03"},{"index":2,"sense_key":"role%1:09:00::","synset_id":"character.n.04"},{"index":3,"sense_key":"role%1:07:00::","synset_id":"function.n.02"},{"index":4,"sense_key":"role%1:04:01::","synset_id":"role.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"role%1:04:00::","call_ids":["senseval3.d001.s019.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"role%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":843,"cached_input_tokens":0,"output_tokens":156,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003246,"input_uncached_usd":0.001686,"input_cached_usd":0.0,"output_usd":0.0015600000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.6121724999975413}
{"item_id":"senseval3.d001.s019.t012","gold_sense_keys":["society%1:14:00::"],"candidates":[{"index":1,"sense_key":"society%1:14:00::","synset_id":"society.n.01"},{"index":2,"sense_key":"society%1:14:01::","synset_id":"club.n.02"},{"index":3,"sense_key":"society%1:26:00::","synset_id":"company.n.03"},{"index":4,"sense_key":"society%1:14:02::","synset_id":"society.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"society%1:14:00::","call_ids":["senseval3.d001.s019.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"society%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":817,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001754,"input_uncached_usd":0.0016339999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.3317979590501636}
{"item_id":"senseval3.d001.s019.t013","gold_sense_keys":["problem%1:09:00::","problem%1:26:00::"],"candidates":[{"index":1,"sense_key":"problem%1:26:00::","synset_id":"problem.n.01"},{"index":2,"sense_key":"problem%1:10:00::","synset_id":"problem.n.02"},{"index":3,"sense_key":"problem%1:09:00::","synset_id":"trouble.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"problem%1:26:00::","call_ids":["senseval3.d001.s019.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"problem%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":766,"cached_input_tokens":0,"output_tokens":277,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004302,"input_uncached_usd":0.001532,"input_cached_usd":0.0,"output_usd":0.0027700000000000003,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.8005744160618633}
{"item_id":"senseval3.d001.s019.t016","gold_sense_keys":["race%1:11:01::"],"candidates":[{"index":1,"sense_key":"race%1:11:01::","synset_id":"race.n.01"},{"index":2,"sense_key":"race%1:11:00::","synset_id":"race.n.02"},{"index":3,"sense_key":"race%1:14:00::","synset_id":"race.n.03"},{"index":4,"sense_key":"race%1:14:01::","synset_id":"subspecies.n.01"},{"index":5,"sense_key":"race%1:19:00::","synset_id":"slipstream.n.01"},{"index":6,"sense_key":"race%1:06:00::","synset_id":"raceway.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"race%1:11:01::","call_ids":["senseval3.d001.s019.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"race%1:11:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":892,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001904,"input_uncached_usd":0.001784,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1417870000004768}
{"item_id":"senseval3.d001.s020.t000","gold_sense_keys":["explain%2:32:02::"],"candidates":[{"index":1,"sense_key":"explain%2:32:00::","synset_id":"explain.v.01"},{"index":2,"sense_key":"explain%2:32:01::","synset_id":"explain.v.02"},{"index":3,"sense_key":"explain%2:32:02::","synset_id":"excuse.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"explain%2:32:02::","call_ids":["senseval3.d001.s020.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"explain%2:32:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":793,"cached_input_tokens":0,"output_tokens":105,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0026360000000000003,"input_uncached_usd":0.001586,"input_cached_usd":0.0,"output_usd":0.0010500000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.421686750021763}
{"item_id":"senseval3.d001.s020.t002","gold_sense_keys":["hold%2:31:01::"],"candidates":[{"index":1,"sense_key":"hold%2:42:00::","synset_id":"keep.v.01"},{"index":2,"sense_key":"hold%2:35:00::","synset_id":"hold.v.02"},{"index":3,"sense_key":"hold%2:36:00::","synset_id":"hold.v.03"},{"index":4,"sense_key":"hold%2:40:00::","synset_id":"have.v.01"},{"index":5,"sense_key":"hold%2:31:01::","synset_id":"deem.v.01"},{"index":6,"sense_key":"hold%2:37:00::","synset_id":"harbor.v.01"},{"index":7,"sense_key":"hold%2:35:03::","synset_id":"restrain.v.03"},{"index":8,"sense_key":"hold%2:40:02::","synset_id":"retain.v.03"},{"index":9,"sense_key":"hold%2:40:04::","synset_id":"bear.v.11"},{"index":10,"sense_key":"hold%2:35:01::","synset_id":"hold.v.10"},{"index":11,"sense_key":"hold%2:42:13::","synset_id":"hold.v.11"},{"index":12,"sense_key":"hold%2:42:05::","synset_id":"accommodate.v.04"},{"index":13,"sense_key":"hold%2:42:01::","synset_id":"hold.v.13"},{"index":14,"sense_key":"hold%2:35:02::","synset_id":"hold.v.14"},{"index":15,"sense_key":"hold%2:42:02::","synset_id":"prevail.v.02"},{"index":16,"sense_key":"hold%2:31:08::","synset_id":"hold.v.16"},{"index":17,"sense_key":"hold%2:42:07::","synset_id":"hold.v.17"},{"index":18,"sense_key":"hold%2:42:14::","synset_id":"contain.v.05"},{"index":19,"sense_key":"hold%2:41:00::","synset_id":"reserve.v.04"},{"index":20,"sense_key":"hold%2:33:05::","synset_id":"defend.v.03"},{"index":21,"sense_key":"hold%2:32:13::","synset_id":"oblige.v.02"},{"index":22,"sense_key":"hold%2:31:04::","synset_id":"hold.v.22"},{"index":23,"sense_key":"hold%2:31:10::","synset_id":"hold.v.23"},{"index":24,"sense_key":"hold%2:42:08::","synset_id":"defy.v.01"},{"index":25,"sense_key":"hold%2:42:04::","synset_id":"apply.v.02"},{"index":26,"sense_key":"hold%2:42:12::","synset_id":"hold.v.26"},{"index":27,"sense_key":"hold%2:41:15::","synset_id":"control.v.02"},{"index":28,"sense_key":"hold%2:41:13::","synset_id":"hold.v.28"},{"index":29,"sense_key":"hold%2:41:14::","synset_id":"hold.v.29"},{"index":30,"sense_key":"hold%2:38:05::","synset_id":"halt.v.01"},{"index":31,"sense_key":"hold%2:35:09::","synset_id":"hold.v.31"},{"index":32,"sense_key":"hold%2:34:00::","synset_id":"carry.v.33"},{"index":33,"sense_key":"hold%2:33:00::","synset_id":"hold.v.33"},{"index":34,"sense_key":"hold%2:32:11::","synset_id":"declare.v.04"},{"index":35,"sense_key":"hold%2:32:15::","synset_id":"agree.v.01"},{"index":36,"sense_key":"hold%2:29:04::","synset_id":"hold.v.36"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"hold%2:31:01::","call_ids":["senseval3.d001.s020.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"hold%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2945,"cached_input_tokens":0,"output_tokens":294,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00883,"input_uncached_usd":0.005889999999999999,"input_cached_usd":0.0,"output_usd":0.0029400000000000003,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.4962592080701143}
{"item_id":"senseval3.d001.s020.t003","gold_sense_keys":["disdain%1:12:00::"],"candidates":[{"index":1,"sense_key":"disdain%1:12:00::","synset_id":"contempt.n.01"},{"index":2,"sense_key":"disdain%1:10:00::","synset_id":"condescension.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"disdain%1:12:00::","call_ids":["senseval3.d001.s020.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"disdain%1:12:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":749,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001618,"input_uncached_usd":0.001498,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.6836135829798877}
{"item_id":"senseval3.d001.s020.t004","gold_sense_keys":["love%2:37:00::"],"candidates":[{"index":1,"sense_key":"love%2:37:00::","synset_id":"love.v.01"},{"index":2,"sense_key":"love%2:37:02::","synset_id":"love.v.02"},{"index":3,"sense_key":"love%2:37:01::","synset_id":"love.v.03"},{"index":4,"sense_key":"love%2:35:00::","synset_id":"sleep_together.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"love%2:37:00::","call_ids":["senseval3.d001.s020.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"love%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":831,"cached_input_tokens":0,"output_tokens":84,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002502,"input_uncached_usd":0.0016619999999999998,"input_cached_usd":0.0,"output_usd":0.00084,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3892553329933435}
{"item_id":"senseval3.d001.s020.t007","gold_sense_keys":["representative%1:18:01::"],"candidates":[{"index":1,"sense_key":"representative%1:18:00::","synset_id":"representative.n.01"},{"index":2,"sense_key":"representative%1:18:02::","synset_id":"spokesperson.n.01"},{"index":3,"sense_key":"representative%1:18:01::","synset_id":"congressman.n.01"},{"index":4,"sense_key":"representative%1:09:00::","synset_id":"example.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"representative%1:18:00::","call_ids":["senseval3.d001.s020.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"representative%1:18:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":842,"cached_input_tokens":0,"output_tokens":140,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003084,"input_uncached_usd":0.001684,"input_cached_usd":0.0,"output_usd":0.0014000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9797897499520332}
{"item_id":"senseval3.d001.s020.t008","gold_sense_keys":["individual%3:00:00::","individual%5:00:00:independent:00"],"candidates":[{"index":1,"sense_key":"individual%3:00:00::","synset_id":"individual.a.01"},{"index":2,"sense_key":"individual%5:00:00:independent:00","synset_id":"individual.s.01"},{"index":3,"sense_key":"individual%5:00:00:unshared:00","synset_id":"individual.s.02"},{"index":4,"sense_key":"individual%5:00:00:personal:00","synset_id":"individual.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"individual%3:00:00::","call_ids":["senseval3.d001.s020.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"individual%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":868,"cached_input_tokens":0,"output_tokens":93,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002666,"input_uncached_usd":0.001736,"input_cached_usd":0.0,"output_usd":0.00093,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7657705000601709}
{"item_id":"senseval3.d001.s020.t010","gold_sense_keys":["constituent%1:18:00::"],"candidates":[{"index":1,"sense_key":"constituent%1:06:00::","synset_id":"component.n.03"},{"index":2,"sense_key":"constituent%1:18:00::","synset_id":"constituent.n.02"},{"index":3,"sense_key":"constituent%1:24:00::","synset_id":"part.n.01"},{"index":4,"sense_key":"constituent%1:10:00::","synset_id":"constituent.n.04"},{"index":5,"sense_key":"constituent%1:09:00::","synset_id":"component.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"constituent%1:18:00::","call_ids":["senseval3.d001.s020.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"constituent%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":996,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0021119999999999997,"input_uncached_usd":0.001992,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1639944170601666}
{"item_id":"senseval3.d001.s020.t011","gold_sense_keys":["appreciate%2:37:00::"],"candidates":[{"index":1,"sense_key":"appreciate%2:37:00::","synset_id":"appreciate.v.01"},{"index":2,"sense_key":"appreciate%2:31:00::","synset_id":"appreciate.v.02"},{"index":3,"sense_key":"appreciate%2:40:00::","synset_id":"prize.v.01"},{"index":4,"sense_key":"appreciate%2:30:00::","synset_id":"appreciate.v.04"},{"index":5,"sense_key":"appreciate%2:30:01::","synset_id":"appreciate.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"appreciate%2:37:00::","call_ids":["senseval3.d001.s020.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"appreciate%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":887,"cached_input_tokens":0,"output_tokens":177,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0035440000000000003,"input_uncached_usd":0.001774,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.550153417047113}
{"item_id":"senseval3.d001.s020.t012","gold_sense_keys":["specific%3:00:00::"],"candidates":[{"index":1,"sense_key":"specific%3:00:00::","synset_id":"specific.a.01"},{"index":2,"sense_key":"specific%5:00:00:specified:00","synset_id":"specific.s.01"},{"index":3,"sense_key":"specific%3:01:00::","synset_id":"specific.a.03"},{"index":4,"sense_key":"specific%3:00:02::","synset_id":"specific.a.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"specific%3:00:00::","call_ids":["senseval3.d001.s020.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"specific%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":858,"cached_input_tokens":0,"output_tokens":153,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0032459999999999998,"input_uncached_usd":0.0017159999999999999,"input_cached_usd":0.0,"output_usd":0.0015300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.4179980830522254}
{"item_id":"senseval3.d001.s020.t013","gold_sense_keys":["benefit%1:07:00::"],"candidates":[{"index":1,"sense_key":"benefit%1:21:00::","synset_id":"benefit.n.01"},{"index":2,"sense_key":"benefit%1:07:00::","synset_id":"benefit.n.02"},{"index":3,"sense_key":"benefit%1:10:00::","synset_id":"benefit.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"benefit%1:07:00::","call_ids":["senseval3.d001.s020.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"benefit%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":729,"cached_input_tokens":0,"output_tokens":124,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002698,"input_uncached_usd":0.0014579999999999999,"input_cached_usd":0.0,"output_usd":0.00124,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6429671660298482}
{"item_id":"senseval3.d001.s020.t015","gold_sense_keys":["win%2:40:00::"],"candidates":[{"index":1,"sense_key":"win%2:33:00::","synset_id":"win.v.01"},{"index":2,"sense_key":"win%2:40:00::","synset_id":"acquire.v.05"},{"index":3,"sense_key":"win%2:33:01::","synset_id":"gain.v.05"},{"index":4,"sense_key":"win%2:41:04::","synset_id":"succeed.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"win%2:40:00::","call_ids":["senseval3.d001.s020.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"win%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":950,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00253,"input_uncached_usd":0.0019,"input_cached_usd":0.0,"output_usd":0.00063,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3458069160114974}
{"item_id":"senseval3.d001.s020.t016","gold_sense_keys":["overall%5:00:00:gross:00"],"candidates":[{"index":1,"sense_key":"overall%5:00:00:general:00","synset_id":"overall.s.01"},{"index":2,"sense_key":"overall%5:00:00:gross:00","synset_id":"overall.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"overall%5:00:00:gross:00","call_ids":["senseval3.d001.s020.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"overall%5:00:00:gross:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":685,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00204,"input_uncached_usd":0.00137,"input_cached_usd":0.0,"output_usd":0.00067,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8361625419929624}
{"item_id":"senseval3.d001.s020.t017","gold_sense_keys":["cost%1:07:01::"],"candidates":[{"index":1,"sense_key":"cost%1:21:00::","synset_id":"cost.n.01"},{"index":2,"sense_key":"cost%1:07:00::","synset_id":"monetary_value.n.01"},{"index":3,"sense_key":"cost%1:07:01::","synset_id":"price.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"cost%1:07:01::","call_ids":["senseval3.d001.s020.t017__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"cost%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":806,"cached_input_tokens":0,"output_tokens":140,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003012,"input_uncached_usd":0.001612,"input_cached_usd":0.0,"output_usd":0.0014000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.4712541250046343}
{"item_id":"senseval3.d001.s020.t019","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":2,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":3,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"other%3:00:00::","call_ids":["senseval3.d001.s020.t019__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":837,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001794,"input_uncached_usd":0.001674,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9946807080414146}
{"item_id":"senseval3.d001.s020.t021","gold_sense_keys":["do%2:29:09::"],"candidates":[{"index":1,"sense_key":"do%2:41:01::","synset_id":"make.v.01"},{"index":2,"sense_key":"do%2:36:01::","synset_id":"perform.v.01"},{"index":3,"sense_key":"do%2:41:04::","synset_id":"do.v.03"},{"index":4,"sense_key":"do%2:42:00::","synset_id":"do.v.04"},{"index":5,"sense_key":"do%2:36:02::","synset_id":"cause.v.01"},{"index":6,"sense_key":"do%2:41:02::","synset_id":"practice.v.01"},{"index":7,"sense_key":"do%2:42:01::","synset_id":"suffice.v.01"},{"index":8,"sense_key":"do%2:36:00::","synset_id":"do.v.08"},{"index":9,"sense_key":"do%2:29:09::","synset_id":"act.v.02"},{"index":10,"sense_key":"do%2:42:02::","synset_id":"serve.v.09"},{"index":11,"sense_key":"do%2:41:03::","synset_id":"do.v.11"},{"index":12,"sense_key":"do%2:29:04::","synset_id":"dress.v.16"},{"index":13,"sense_key":"do%2:38:00::","synset_id":"do.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"do%2:36:01::","call_ids":["senseval3.d001.s020.t021__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"do%2:36:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1414,"cached_input_tokens":0,"output_tokens":128,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004108,"input_uncached_usd":0.002828,"input_cached_usd":0.0,"output_usd":0.00128,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9163913329830393}
{"item_id":"senseval3.d001.s021.t001","gold_sense_keys":["suggest%2:32:02::"],"candidates":[{"index":1,"sense_key":"suggest%2:32:00::","synset_id":"propose.v.01"},{"index":2,"sense_key":"suggest%2:32:04::","synset_id":"hint.v.01"},{"index":3,"sense_key":"suggest%2:32:02::","synset_id":"suggest.v.03"},{"index":4,"sense_key":"suggest%2:32:03::","synset_id":"indicate.v.05"},{"index":5,"sense_key":"suggest%2:32:01::","synset_id":"suggest.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"suggest%2:32:02::","call_ids":["senseval3.d001.s021.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"suggest%2:32:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":906,"cached_input_tokens":0,"output_tokens":82,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0026320000000000002,"input_uncached_usd":0.001812,"input_cached_usd":0.0,"output_usd":0.0008200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6188017500098795}
{"item_id":"senseval3.d001.s021.t004","gold_sense_keys":["national%3:00:00::","national%3:00:01::","national%3:01:00::","national%5:00:00:domestic:00"],"candidates":[{"index":1,"sense_key":"national%3:01:00::","synset_id":"national.a.01"},{"index":2,"sense_key":"national%3:00:00::","synset_id":"national.a.02"},{"index":3,"sense_key":"national%3:00:01::","synset_id":"national.a.03"},{"index":4,"sense_key":"national%5:00:00:public:00","synset_id":"national.s.01"},{"index":5,"sense_key":"national%5:00:00:domestic:00","synset_id":"home.s.01"},{"index":6,"sense_key":"national%3:01:02::","synset_id":"national.a.06"},{"index":7,"sense_key":"national%3:01:01::","synset_id":"national.a.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"national%3:00:01::","call_ids":["senseval3.d001.s021.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"national%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":976,"cached_input_tokens":0,"output_tokens":108,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003032,"input_uncached_usd":0.001952,"input_cached_usd":0.0,"output_usd":0.00108,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.3525897919898853}
{"item_id":"senseval3.d001.s021.t005","gold_sense_keys":["policy%1:09:00::"],"candidates":[{"index":1,"sense_key":"policy%1:09:00::","synset_id":"policy.n.01"},{"index":2,"sense_key":"policy%1:10:00::","synset_id":"policy.n.02"},{"index":3,"sense_key":"policy%1:10:02::","synset_id":"policy.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"policy%1:09:00::","call_ids":["senseval3.d001.s021.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"policy%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":802,"cached_input_tokens":0,"output_tokens":106,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002664,"input_uncached_usd":0.001604,"input_cached_usd":0.0,"output_usd":0.0010600000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5399785420158878}
{"item_id":"senseval3.d001.s021.t006","gold_sense_keys":["making%1:04:00::"],"candidates":[{"index":1,"sense_key":"making%1:04:00::","synset_id":"devising.n.01"},{"index":2,"sense_key":"making%1:07:00::","synset_id":"qualification.n.01"},{"index":3,"sense_key":"making%1:06:00::","synset_id":"making.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"making%1:04:00::","call_ids":["senseval3.d001.s021.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"making%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":821,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0017620000000000001,"input_uncached_usd":0.001642,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0297748330049217}
{"item_id":"senseval3.d001.s021.t007","gold_sense_keys":["relative%5:00:00:proportionate:00"],"candidates":[{"index":1,"sense_key":"relative%3:00:00::","synset_id":"relative.a.01"},{"index":2,"sense_key":"relative%5:00:00:proportionate:00","synset_id":"proportional.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"relative%5:00:00:proportionate:00","call_ids":["senseval3.d001.s021.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"relative%5:00:00:proportionate:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":761,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002142,"input_uncached_usd":0.001522,"input_cached_usd":0.0,"output_usd":0.00062,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5651659589493647}
{"item_id":"senseval3.d001.s021.t008","gold_sense_keys":["local%3:00:01::"],"candidates":[{"index":1,"sense_key":"local%3:00:01::","synset_id":"local.a.01"},{"index":2,"sense_key":"local%3:01:01::","synset_id":"local.a.02"},{"index":3,"sense_key":"local%3:00:02::","synset_id":"local.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"local%3:01:01::","call_ids":["senseval3.d001.s021.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"local%3:01:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":777,"cached_input_tokens":0,"output_tokens":108,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002634,"input_uncached_usd":0.0015539999999999998,"input_cached_usd":0.0,"output_usd":0.00108,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.639794290997088}
{"item_id":"senseval3.d001.s021.t009","gold_sense_keys":["have%2:30:01::","have%2:40:00::","have%2:42:00::","have%2:42:12::"],"candidates":[{"index":1,"sense_key":"have%2:40:00::","synset_id":"have.v.01"},{"index":2,"sense_key":"have%2:42:00::","synset_id":"have.v.02"},{"index":3,"sense_key":"have%2:39:06::","synset_id":"experience.v.03"},{"index":4,"sense_key":"have%2:40:04::","synset_id":"own.v.01"},{"index":5,"sense_key":"have%2:30:00::","synset_id":"get.v.03"},{"index":6,"sense_key":"have%2:34:00::","synset_id":"consume.v.02"},{"index":7,"sense_key":"have%2:41:00::","synset_id":"have.v.07"},{"index":8,"sense_key":"have%2:36:00::","synset_id":"hold.v.03"},{"index":9,"sense_key":"have%2:40:01::","synset_id":"have.v.09"},{"index":10,"sense_key":"have%2:42:12::","synset_id":"have.v.10"},{"index":11,"sense_key":"have%2:30:01::","synset_id":"have.v.11"},{"index":12,"sense_key":"have%2:29:02::","synset_id":"have.v.12"},{"index":13,"sense_key":"have%2:32:00::","synset_id":"induce.v.02"},{"index":14,"sense_key":"have%2:40:05::","synset_id":"accept.v.02"},{"index":15,"sense_key":"have%2:40:03::","synset_id":"receive.v.01"},{"index":16,"sense_key":"have%2:29:05::","synset_id":"suffer.v.02"},{"index":17,"sense_key":"have%2:40:02::","synset_id":"have.v.17"},{"index":18,"sense_key":"have%2:29:00::","synset_id":"give_birth.v.01"},{"index":19,"sense_key":"have%2:35:00::","synset_id":"take.v.35"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"have%2:40:00::","call_ids":["senseval3.d001.s021.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"have%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1782,"cached_input_tokens":0,"output_tokens":70,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004264,"input_uncached_usd":0.003564,"input_cached_usd":0.0,"output_usd":0.0007000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9952719589928165}
{"item_id":"senseval3.d001.s021.t011","gold_sense_keys":["office%1:26:00::"],"candidates":[{"index":1,"sense_key":"office%1:06:00::","synset_id":"office.n.01"},{"index":2,"sense_key":"office%1:14:01::","synset_id":"agency.n.01"},{"index":3,"sense_key":"office%1:04:01::","synset_id":"function.n.03"},{"index":4,"sense_key":"office%1:26:00::","synset_id":"office.n.04"},{"index":5,"sense_key":"office%1:14:00::","synset_id":"office.n.05"},{"index":6,"sense_key":"office%1:04:00::","synset_id":"office.n.06"},{"index":7,"sense_key":"office%1:04:02::","synset_id":"position.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"office%1:26:00::","call_ids":["senseval3.d001.s021.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"office%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1062,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002754,"input_uncached_usd":0.002124,"input_cached_usd":0.0,"output_usd":0.00063,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.680138083989732}
{"item_id":"senseval3.d001.s022.t002","gold_sense_keys":["member%1:18:00::"],"candidates":[{"index":1,"sense_key":"member%1:18:00::","synset_id":"member.n.01"},{"index":2,"sense_key":"member%1:24:00::","synset_id":"member.n.02"},{"index":3,"sense_key":"member%1:08:00::","synset_id":"extremity.n.01"},{"index":4,"sense_key":"member%1:14:00::","synset_id":"member.n.04"},{"index":5,"sense_key":"member%1:08:01::","synset_id":"penis.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"member%1:18:00::","call_ids":["senseval3.d001.s022.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"member%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":941,"cached_input_tokens":0,"output_tokens":45,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002332,"input_uncached_usd":0.001882,"input_cached_usd":0.0,"output_usd":0.00045000000000000004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2920959589537233}
{"item_id":"senseval3.d001.s022.t003","gold_sense_keys":["vulnerable%3:00:00::"],"candidates":[{"index":1,"sense_key":"vulnerable%3:00:00::","synset_id":"vulnerable.a.01"},{"index":2,"sense_key":"vulnerable%5:00:00:susceptible:00","synset_id":"vulnerable.s.01"},{"index":3,"sense_key":"vulnerable%5:00:00:weak:00","synset_id":"vulnerable.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"vulnerable%3:00:00::","call_ids":["senseval3.d001.s022.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"vulnerable%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":792,"cached_input_tokens":0,"output_tokens":139,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002974,"input_uncached_usd":0.001584,"input_cached_usd":0.0,"output_usd":0.0013900000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9228186249965802}
{"item_id":"senseval3.d001.s022.t005","gold_sense_keys":["become%2:30:00::"],"candidates":[{"index":1,"sense_key":"become%2:30:00::","synset_id":"become.v.01"},{"index":2,"sense_key":"become%2:42:01::","synset_id":"become.v.02"},{"index":3,"sense_key":"become%2:42:00::","synset_id":"become.v.03"},{"index":4,"sense_key":"become%2:42:03::","synset_id":"become.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"become%2:30:00::","call_ids":["senseval3.d001.s022.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"become%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":840,"cached_input_tokens":0,"output_tokens":134,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00302,"input_uncached_usd":0.0016799999999999999,"input_cached_usd":0.0,"output_usd":0.00134,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8858470839913934}
{"item_id":"senseval3.d001.s022.t006","gold_sense_keys":["immune%5:00:00:unaffected:00"],"candidates":[{"index":1,"sense_key":"immune%3:01:00::","synset_id":"immune.a.01"},{"index":2,"sense_key":"immune%5:00:00:exempt:00","synset_id":"immune.s.01"},{"index":3,"sense_key":"immune%5:00:00:unsusceptible:00","synset_id":"immune.s.02"},{"index":4,"sense_key":"immune%5:00:00:unaffected:00","synset_id":"immune.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"immune%5:00:00:exempt:00","call_ids":["senseval3.d001.s022.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"immune%5:00:00:exempt:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":829,"cached_input_tokens":0,"output_tokens":115,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002808,"input_uncached_usd":0.001658,"input_cached_usd":0.0,"output_usd":0.0011500000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2345239999704063}
{"item_id":"senseval3.d001.s022.t007","gold_sense_keys":["defeat%1:11:00::"],"candidates":[{"index":1,"sense_key":"defeat%1:11:00::","synset_id":"defeat.n.01"},{"index":2,"sense_key":"defeat%1:12:00::","synset_id":"frustration.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"defeat%1:11:00::","call_ids":["senseval3.d001.s022.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"defeat%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":741,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0016020000000000001,"input_uncached_usd":0.001482,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5242518750019372}
{"item_id":"senseval3.d001.s023.t000","gold_sense_keys":["exception%1:09:01::"],"candidates":[{"index":1,"sense_key":"exception%1:09:00::","synset_id":"exception.n.01"},{"index":2,"sense_key":"exception%1:09:01::","synset_id":"exception.n.02"},{"index":3,"sense_key":"exception%1:10:00::","synset_id":"exception.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"exception%1:09:01::","call_ids":["senseval3.d001.s023.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"exception%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":820,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00176,"input_uncached_usd":0.00164,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4290186669677496}
{"item_id":"senseval3.d001.s023.t002","gold_sense_keys":["trend%1:15:02::"],"candidates":[{"index":1,"sense_key":"trend%1:15:02::","synset_id":"tendency.n.04"},{"index":2,"sense_key":"trend%1:15:01::","synset_id":"course.n.03"},{"index":3,"sense_key":"trend%1:09:00::","synset_id":"drift.n.05"},{"index":4,"sense_key":"trend%1:09:02::","synset_id":"vogue.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"trend%1:15:02::","call_ids":["senseval3.d001.s023.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"trend%1:15:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":911,"cached_input_tokens":0,"output_tokens":221,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004032,"input_uncached_usd":0.0018219999999999998,"input_cached_usd":0.0,"output_usd":0.00221,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.4192395419813693}
{"item_id":"senseval3.d001.s023.t003","gold_sense_keys":["defeat%1:11:00::"],"candidates":[{"index":1,"sense_key":"defeat%1:11:00::","synset_id":"defeat.n.01"},{"index":2,"sense_key":"defeat%1:12:00::","synset_id":"frustration.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"defeat%1:11:00::","call_ids":["senseval3.d001.s023.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"defeat%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":772,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0016640000000000001,"input_uncached_usd":0.001544,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0995104999747127}
{"item_id":"senseval3.d001.s023.t004","gold_sense_keys":["freshman%1:18:01::"],"candidates":[{"index":1,"sense_key":"freshman%1:18:00::","synset_id":"freshman.n.01"},{"index":2,"sense_key":"freshman%1:18:01::","synset_id":"newcomer.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"freshman%1:18:01::","call_ids":["senseval3.d001.s023.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"freshman%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":751,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002052,"input_uncached_usd":0.0015019999999999999,"input_cached_usd":0.0,"output_usd":0.00055,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5670918750111014}
{"item_id":"senseval3.d001.s023.t005","gold_sense_keys":["republican%1:18:01::"],"candidates":[{"index":1,"sense_key":"republican%1:18:01::","synset_id":"republican.n.01"},{"index":2,"sense_key":"republican%1:18:00::","synset_id":"republican.n.02"},{"index":3,"sense_key":"republican%1:17:00::","synset_id":"republican.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"republican%1:18:01::","call_ids":["senseval3.d001.s023.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"republican%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":789,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001698,"input_uncached_usd":0.001578,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0169549999991432}
{"item_id":"senseval3.d001.s023.t006","gold_sense_keys":["bring%2:35:04::"],"candidates":[{"index":1,"sense_key":"bring%2:38:00::","synset_id":"bring.v.01"},{"index":2,"sense_key":"bring%2:35:04::","synset_id":"bring.v.02"},{"index":3,"sense_key":"bring%2:36:00::","synset_id":"bring.v.03"},{"index":4,"sense_key":"bring%2:35:00::","synset_id":"bring.v.04"},{"index":5,"sense_key":"bring%2:30:00::","synset_id":"bring.v.05"},{"index":6,"sense_key":"bring%2:35:02::","synset_id":"bring.v.06"},{"index":7,"sense_key":"bring%2:36:01::","synset_id":"institute.v.02"},{"index":8,"sense_key":"bring%2:40:02::","synset_id":"lend.v.01"},{"index":9,"sense_key":"bring%2:40:00::","synset_id":"fetch.v.02"},{"index":10,"sense_key":"bring%2:37:05::","synset_id":"bring.v.10"},{"index":11,"sense_key":"bring%2:32:03::","synset_id":"bring.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bring%2:35:04::","call_ids":["senseval3.d001.s023.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bring%2:35:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1317,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0032240000000000003,"input_uncached_usd":0.002634,"input_cached_usd":0.0,"output_usd":0.00059,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9938827499281615}
{"item_id":"senseval3.d001.s023.t007","gold_sense_keys":["office%1:26:00::"],"candidates":[{"index":1,"sense_key":"office%1:06:00::","synset_id":"office.n.01"},{"index":2,"sense_key":"office%1:14:01::","synset_id":"agency.n.01"},{"index":3,"sense_key":"office%1:04:01::","synset_id":"function.n.03"},{"index":4,"sense_key":"office%1:26:00::","synset_id":"office.n.04"},{"index":5,"sense_key":"office%1:14:00::","synset_id":"office.n.05"},{"index":6,"sense_key":"office%1:04:00::","synset_id":"office.n.06"},{"index":7,"sense_key":"office%1:04:02::","synset_id":"position.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"office%1:26:00::","call_ids":["senseval3.d001.s023.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"office%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1091,"cached_input_tokens":0,"output_tokens":123,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003412,"input_uncached_usd":0.002182,"input_cached_usd":0.0,"output_usd":0.0012300000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9571512079564855}
{"item_id":"senseval3.d001.s023.t008","gold_sense_keys":["revolution%1:11:01::"],"candidates":[{"index":1,"sense_key":"revolution%1:11:01::","synset_id":"revolution.n.01"},{"index":2,"sense_key":"revolution%1:04:00::","synset_id":"revolution.n.02"},{"index":3,"sense_key":"revolution%1:11:00::","synset_id":"rotation.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"revolution%1:11:01::","call_ids":["senseval3.d001.s023.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"revolution%1:11:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":832,"cached_input_tokens":0,"output_tokens":103,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002694,"input_uncached_usd":0.001664,"input_cached_usd":0.0,"output_usd":0.00103,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6843959579709917}
{"item_id":"senseval3.d001.s023.t009","gold_sense_keys":["run%2:33:06::"],"candidates":[{"index":1,"sense_key":"run%2:38:00::","synset_id":"run.v.01"},{"index":2,"sense_key":"run%2:38:04::","synset_id":"scat.v.01"},{"index":3,"sense_key":"run%2:42:00::","synset_id":"run.v.03"},{"index":4,"sense_key":"run%2:41:02::","synset_id":"operate.v.01"},{"index":5,"sense_key":"run%2:42:04::","synset_id":"run.v.05"},{"index":6,"sense_key":"run%2:38:01::","synset_id":"run.v.06"},{"index":7,"sense_key":"run%2:35:00::","synset_id":"function.v.01"},{"index":8,"sense_key":"run%2:42:07::","synset_id":"range.v.01"},{"index":9,"sense_key":"run%2:33:06::","synset_id":"campaign.v.01"},{"index":10,"sense_key":"run%2:36:01::","synset_id":"play.v.18"},{"index":11,"sense_key":"run%2:38:11::","synset_id":"run.v.11"},{"index":12,"sense_key":"run%2:42:01::","synset_id":"tend.v.01"},{"index":13,"sense_key":"run%2:35:04::","synset_id":"run.v.13"},{"index":14,"sense_key":"run%2:30:03::","synset_id":"run.v.14"},{"index":15,"sense_key":"run%2:30:07::","synset_id":"run.v.15"},{"index":16,"sense_key":"run%2:42:02::","synset_id":"run.v.16"},{"index":17,"sense_key":"run%2:42:03::","synset_id":"prevail.v.03"},{"index":18,"sense_key":"run%2:42:06::","synset_id":"run.v.18"},{"index":19,"sense_key":"run%2:41:00::","synset_id":"run.v.19"},{"index":20,"sense_key":"run%2:36:02::","synset_id":"carry.v.15"},{"index":21,"sense_key":"run%2:36:00::","synset_id":"run.v.21"},{"index":22,"sense_key":"run%2:35:01::","synset_id":"guide.v.05"},{"index":23,"sense_key":"run%2:42:08::","synset_id":"run.v.23"},{"index":24,"sense_key":"run%2:41:01::","synset_id":"run.v.24"},{"index":25,"sense_key":"run%2:40:06::","synset_id":"run.v.25"},{"index":26,"sense_key":"run%2:38:10::","synset_id":"run.v.26"},{"index":27,"sense_key":"run%2:38:08::","synset_id":"run.v.27"},{"index":28,"sense_key":"run%2:38:03::","synset_id":"run.v.28"},{"index":29,"sense_key":"run%2:38:05::","synset_id":"run.v.29"},{"index":30,"sense_key":"run%2:42:05::","synset_id":"run.v.30"},{"index":31,"sense_key":"run%2:41:04::","synset_id":"run.v.31"},{"index":32,"sense_key":"run%2:38:02::","synset_id":"run.v.32"},{"index":33,"sense_key":"run%2:38:06::","synset_id":"run.v.33"},{"index":34,"sense_key":"run%2:38:13::","synset_id":"run.v.34"},{"index":35,"sense_key":"run%2:38:09::","synset_id":"ply.v.03"},{"index":36,"sense_key":"run%2:33:02::","synset_id":"hunt.v.01"},{"index":37,"sense_key":"run%2:33:01::","synset_id":"race.v.02"},{"index":38,"sense_key":"run%2:30:02::","synset_id":"move.v.13"},{"index":39,"sense_key":"run%2:30:00::","synset_id":"melt.v.01"},{"index":40,"sense_key":"run%2:30:01::","synset_id":"ladder.v.01"},{"index":41,"sense_key":"run%2:30:04::","synset_id":"run.v.41"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"run%2:33:06::","call_ids":["senseval3.d001.s023.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"run%2:33:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2986,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.006092,"input_uncached_usd":0.005972,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0874876249581575}
{"item_id":"senseval3.d001.s024.t000","gold_sense_keys":["freshman%1:18:01::"],"candidates":[{"index":1,"sense_key":"freshman%1:18:00::","synset_id":"freshman.n.01"},{"index":2,"sense_key":"freshman%1:18:01::","synset_id":"newcomer.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"freshman%1:18:01::","call_ids":["senseval3.d001.s024.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"freshman%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":769,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001658,"input_uncached_usd":0.0015379999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.173416499979794}
{"item_id":"senseval3.d001.s024.t001","gold_sense_keys":["place%2:35:00::"],"candidates":[{"index":1,"sense_key":"place%2:35:00::","synset_id":"put.v.01"},{"index":2,"sense_key":"place%2:41:01::","synset_id":"place.v.02"},{"index":3,"sense_key":"place%2:31:01::","synset_id":"rate.v.01"},{"index":4,"sense_key":"place%2:40:00::","synset_id":"locate.v.03"},{"index":5,"sense_key":"place%2:32:01::","synset_id":"place.v.05"},{"index":6,"sense_key":"place%2:31:03::","synset_id":"place.v.06"},{"index":7,"sense_key":"place%2:33:03::","synset_id":"target.v.01"},{"index":8,"sense_key":"place%2:31:00::","synset_id":"identify.v.01"},{"index":9,"sense_key":"place%2:41:00::","synset_id":"place.v.09"},{"index":10,"sense_key":"place%2:36:00::","synset_id":"set.v.09"},{"index":11,"sense_key":"place%2:31:13::","synset_id":"place.v.11"},{"index":12,"sense_key":"place%2:42:00::","synset_id":"place.v.12"},{"index":13,"sense_key":"place%2:40:03::","synset_id":"invest.v.01"},{"index":14,"sense_key":"place%2:33:00::","synset_id":"station.v.01"},{"index":15,"sense_key":"place%2:33:02::","synset_id":"place.v.15"},{"index":16,"sense_key":"place%2:32:00::","synset_id":"place.v.16"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"place%2:35:00::","call_ids":["senseval3.d001.s024.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"place%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1500,"cached_input_tokens":0,"output_tokens":108,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00408,"input_uncached_usd":0.003,"input_cached_usd":0.0,"output_usd":0.00108,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7843590419506654}
{"item_id":"senseval3.d001.s024.t002","gold_sense_keys":["emphasis%1:07:00::","emphasis%1:26:00::"],"candidates":[{"index":1,"sense_key":"emphasis%1:26:00::","synset_id":"emphasis.n.01"},{"index":2,"sense_key":"emphasis%1:07:00::","synset_id":"vehemence.n.01"},{"index":3,"sense_key":"emphasis%1:10:01::","synset_id":"emphasis.n.03"},{"index":4,"sense_key":"emphasis%1:10:00::","synset_id":"stress.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"emphasis%1:26:00::","call_ids":["senseval3.d001.s024.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"emphasis%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":936,"cached_input_tokens":0,"output_tokens":135,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003222,"input_uncached_usd":0.001872,"input_cached_usd":0.0,"output_usd":0.00135,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.326497499947436}
{"item_id":"senseval3.d001.s024.t004","gold_sense_keys":["role%1:04:00::","role%1:04:01::"],"candidates":[{"index":1,"sense_key":"role%1:04:00::","synset_id":"function.n.03"},{"index":2,"sense_key":"role%1:09:00::","synset_id":"character.n.04"},{"index":3,"sense_key":"role%1:07:00::","synset_id":"function.n.02"},{"index":4,"sense_key":"role%1:04:01::","synset_id":"role.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"role%1:04:00::","call_ids":["senseval3.d001.s024.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"role%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":916,"cached_input_tokens":0,"output_tokens":209,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003922,"input_uncached_usd":0.0018319999999999999,"input_cached_usd":0.0,"output_usd":0.0020900000000000003,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.3564529169816524}
{"item_id":"senseval3.d001.s024.t005","gold_sense_keys":["spread%2:32:00::","spread%2:35:00::"],"candidates":[{"index":1,"sense_key":"spread%2:35:00::","synset_id":"spread.v.01"},{"index":2,"sense_key":"spread%2:35:07::","synset_id":"spread.v.02"},{"index":3,"sense_key":"spread%2:42:00::","synset_id":"spread.v.03"},{"index":4,"sense_key":"spread%2:35:03::","synset_id":"unfold.v.04"},{"index":5,"sense_key":"spread%2:32:00::","synset_id":"circulate.v.02"},{"index":6,"sense_key":"spread%2:32:02::","synset_id":"go_around.v.02"},{"index":7,"sense_key":"spread%2:38:00::","synset_id":"spread.v.07"},{"index":8,"sense_key":"spread%2:38:02::","synset_id":"diffuse.v.01"},{"index":9,"sense_key":"spread%2:35:12::","synset_id":"spread.v.09"},{"index":10,"sense_key":"spread%2:35:13::","synset_id":"spread.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"spread%2:32:00::","call_ids":["senseval3.d001.s024.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"spread%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1242,"cached_input_tokens":0,"output_tokens":138,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0038640000000000002,"input_uncached_usd":0.002484,"input_cached_usd":0.0,"output_usd":0.0013800000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.337696374976076}
{"item_id":"senseval3.d001.s024.t006","gold_sense_keys":["revolution%1:11:01::"],"candidates":[{"index":1,"sense_key":"revolution%1:11:01::","synset_id":"revolution.n.01"},{"index":2,"sense_key":"revolution%1:04:00::","synset_id":"revolution.n.02"},{"index":3,"sense_key":"revolution%1:11:00::","synset_id":"rotation.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"revolution%1:11:01::","call_ids":["senseval3.d001.s024.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"revolution%1:11:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":849,"cached_input_tokens":0,"output_tokens":90,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002598,"input_uncached_usd":0.0016979999999999999,"input_cached_usd":0.0,"output_usd":0.0009000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3378301659831777}
{"item_id":"senseval3.d001.s024.t007","gold_sense_keys":["national%3:00:00::","national%3:00:01::","national%3:01:00::","national%5:00:00:domestic:00"],"candidates":[{"index":1,"sense_key":"national%3:01:00::","synset_id":"national.a.01"},{"index":2,"sense_key":"national%3:00:00::","synset_id":"national.a.02"},{"index":3,"sense_key":"national%3:00:01::","synset_id":"national.a.03"},{"index":4,"sense_key":"national%5:00:00:public:00","synset_id":"national.s.01"},{"index":5,"sense_key":"national%5:00:00:domestic:00","synset_id":"home.s.01"},{"index":6,"sense_key":"national%3:01:02::","synset_id":"national.a.06"},{"index":7,"sense_key":"national%3:01:01::","synset_id":"national.a.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"national%3:00:01::","call_ids":["senseval3.d001.s024.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"national%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1022,"cached_input_tokens":0,"output_tokens":121,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003254,"input_uncached_usd":0.002044,"input_cached_usd":0.0,"output_usd":0.0012100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.053254917031154}
{"item_id":"senseval3.d001.s024.t008","gold_sense_keys":["policy%1:09:00::"],"candidates":[{"index":1,"sense_key":"policy%1:09:00::","synset_id":"policy.n.01"},{"index":2,"sense_key":"policy%1:10:00::","synset_id":"policy.n.02"},{"index":3,"sense_key":"policy%1:10:02::","synset_id":"policy.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"policy%1:09:00::","call_ids":["senseval3.d001.s024.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"policy%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":848,"cached_input_tokens":0,"output_tokens":108,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002776,"input_uncached_usd":0.001696,"input_cached_usd":0.0,"output_usd":0.00108,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6619005829561502}
{"item_id":"senseval3.d001.s024.t009","gold_sense_keys":["making%1:04:00::"],"candidates":[{"index":1,"sense_key":"making%1:04:00::","synset_id":"devising.n.01"},{"index":2,"sense_key":"making%1:07:00::","synset_id":"qualification.n.01"},{"index":3,"sense_key":"making%1:06:00::","synset_id":"making.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"making%1:04:00::","call_ids":["senseval3.d001.s024.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"making%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":867,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001854,"input_uncached_usd":0.0017339999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0696258749812841}
{"item_id":"senseval3.d001.s025.t002","gold_sense_keys":["indicate%2:32:02::"],"candidates":[{"index":1,"sense_key":"indicate%2:32:01::","synset_id":"bespeak.v.01"},{"index":2,"sense_key":"indicate%2:32:03::","synset_id":"indicate.v.02"},{"index":3,"sense_key":"indicate%2:32:00::","synset_id":"indicate.v.03"},{"index":4,"sense_key":"indicate%2:32:02::","synset_id":"argue.v.03"},{"index":5,"sense_key":"indicate%2:32:04::","synset_id":"indicate.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"indicate%2:32:02::","call_ids":["senseval3.d001.s025.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"indicate%2:32:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":946,"cached_input_tokens":0,"output_tokens":122,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003112,"input_uncached_usd":0.001892,"input_cached_usd":0.0,"output_usd":0.0012200000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.87734929099679}
{"item_id":"senseval3.d001.s025.t006","gold_sense_keys":["attract%2:35:00::"],"candidates":[{"index":1,"sense_key":"attract%2:35:00::","synset_id":"attract.v.01"},{"index":2,"sense_key":"attract%2:37:00::","synset_id":"attract.v.02"},{"index":3,"sense_key":"attract%2:35:01::","synset_id":"attract.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"attract%2:35:00::","call_ids":["senseval3.d001.s025.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"attract%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":831,"cached_input_tokens":0,"output_tokens":135,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003012,"input_uncached_usd":0.0016619999999999998,"input_cached_usd":0.0,"output_usd":0.00135,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0554795840289444}
{"item_id":"senseval3.d001.s025.t008","gold_sense_keys":["candidate%1:18:01::"],"candidates":[{"index":1,"sense_key":"candidate%1:18:01::","synset_id":"campaigner.n.01"},{"index":2,"sense_key":"candidate%1:18:00::","synset_id":"candidate.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"candidate%1:18:01::","call_ids":["senseval3.d001.s025.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"candidate%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":703,"cached_input_tokens":0,"output_tokens":142,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002826,"input_uncached_usd":0.001406,"input_cached_usd":0.0,"output_usd":0.00142,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9347925829933956}
{"item_id":"senseval3.d001.s025.t010","gold_sense_keys":["office%1:26:00::"],"candidates":[{"index":1,"sense_key":"office%1:06:00::","synset_id":"office.n.01"},{"index":2,"sense_key":"office%1:14:01::","synset_id":"agency.n.01"},{"index":3,"sense_key":"office%1:04:01::","synset_id":"function.n.03"},{"index":4,"sense_key":"office%1:26:00::","synset_id":"office.n.04"},{"index":5,"sense_key":"office%1:14:00::","synset_id":"office.n.05"},{"index":6,"sense_key":"office%1:04:00::","synset_id":"office.n.06"},{"index":7,"sense_key":"office%1:04:02::","synset_id":"position.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"office%1:04:02::","call_ids":["senseval3.d001.s025.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"office%1:04:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1048,"cached_input_tokens":0,"output_tokens":92,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0030159999999999996,"input_uncached_usd":0.0020959999999999998,"input_cached_usd":0.0,"output_usd":0.00092,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5510624169837683}
{"item_id":"senseval3.d001.s026.t000","gold_sense_keys":["potential%5:00:00:prospective:00"],"candidates":[{"index":1,"sense_key":"potential%3:00:00::","synset_id":"potential.a.01"},{"index":2,"sense_key":"potential%5:00:00:prospective:00","synset_id":"likely.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"potential%5:00:00:prospective:00","call_ids":["senseval3.d001.s026.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"potential%5:00:00:prospective:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":701,"cached_input_tokens":0,"output_tokens":112,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002522,"input_uncached_usd":0.001402,"input_cached_usd":0.0,"output_usd":0.0011200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7885797910857946}
{"item_id":"senseval3.d001.s026.t001","gold_sense_keys":["candidate%1:18:01::"],"candidates":[{"index":1,"sense_key":"candidate%1:18:01::","synset_id":"campaigner.n.01"},{"index":2,"sense_key":"candidate%1:18:00::","synset_id":"candidate.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"candidate%1:18:00::","call_ids":["senseval3.d001.s026.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"candidate%1:18:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":692,"cached_input_tokens":0,"output_tokens":141,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002794,"input_uncached_usd":0.001384,"input_cached_usd":0.0,"output_usd":0.00141,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.039986834046431}
{"item_id":"senseval3.d001.s026.t002","gold_sense_keys":["discourage%2:37:00::"],"candidates":[{"index":1,"sense_key":"discourage%2:32:01::","synset_id":"deter.v.01"},{"index":2,"sense_key":"discourage%2:37:00::","synset_id":"discourage.v.02"},{"index":3,"sense_key":"discourage%2:32:02::","synset_id":"warn.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"discourage%2:32:01::","call_ids":["senseval3.d001.s026.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"discourage%2:32:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":778,"cached_input_tokens":0,"output_tokens":170,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003256,"input_uncached_usd":0.0015559999999999999,"input_cached_usd":0.0,"output_usd":0.0017000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1811025000642985}
{"item_id":"senseval3.d001.s026.t003","gold_sense_keys":["run%2:33:06::"],"candidates":[{"index":1,"sense_key":"run%2:38:00::","synset_id":"run.v.01"},{"index":2,"sense_key":"run%2:38:04::","synset_id":"scat.v.01"},{"index":3,"sense_key":"run%2:42:00::","synset_id":"run.v.03"},{"index":4,"sense_key":"run%2:41:02::","synset_id":"operate.v.01"},{"index":5,"sense_key":"run%2:42:04::","synset_id":"run.v.05"},{"index":6,"sense_key":"run%2:38:01::","synset_id":"run.v.06"},{"index":7,"sense_key":"run%2:35:00::","synset_id":"function.v.01"},{"index":8,"sense_key":"run%2:42:07::","synset_id":"range.v.01"},{"index":9,"sense_key":"run%2:33:06::","synset_id":"campaign.v.01"},{"index":10,"sense_key":"run%2:36:01::","synset_id":"play.v.18"},{"index":11,"sense_key":"run%2:38:11::","synset_id":"run.v.11"},{"index":12,"sense_key":"run%2:42:01::","synset_id":"tend.v.01"},{"index":13,"sense_key":"run%2:35:04::","synset_id":"run.v.13"},{"index":14,"sense_key":"run%2:30:03::","synset_id":"run.v.14"},{"index":15,"sense_key":"run%2:30:07::","synset_id":"run.v.15"},{"index":16,"sense_key":"run%2:42:02::","synset_id":"run.v.16"},{"index":17,"sense_key":"run%2:42:03::","synset_id":"prevail.v.03"},{"index":18,"sense_key":"run%2:42:06::","synset_id":"run.v.18"},{"index":19,"sense_key":"run%2:41:00::","synset_id":"run.v.19"},{"index":20,"sense_key":"run%2:36:02::","synset_id":"carry.v.15"},{"index":21,"sense_key":"run%2:36:00::","synset_id":"run.v.21"},{"index":22,"sense_key":"run%2:35:01::","synset_id":"guide.v.05"},{"index":23,"sense_key":"run%2:42:08::","synset_id":"run.v.23"},{"index":24,"sense_key":"run%2:41:01::","synset_id":"run.v.24"},{"index":25,"sense_key":"run%2:40:06::","synset_id":"run.v.25"},{"index":26,"sense_key":"run%2:38:10::","synset_id":"run.v.26"},{"index":27,"sense_key":"run%2:38:08::","synset_id":"run.v.27"},{"index":28,"sense_key":"run%2:38:03::","synset_id":"run.v.28"},{"index":29,"sense_key":"run%2:38:05::","synset_id":"run.v.29"},{"index":30,"sense_key":"run%2:42:05::","synset_id":"run.v.30"},{"index":31,"sense_key":"run%2:41:04::","synset_id":"run.v.31"},{"index":32,"sense_key":"run%2:38:02::","synset_id":"run.v.32"},{"index":33,"sense_key":"run%2:38:06::","synset_id":"run.v.33"},{"index":34,"sense_key":"run%2:38:13::","synset_id":"run.v.34"},{"index":35,"sense_key":"run%2:38:09::","synset_id":"ply.v.03"},{"index":36,"sense_key":"run%2:33:02::","synset_id":"hunt.v.01"},{"index":37,"sense_key":"run%2:33:01::","synset_id":"race.v.02"},{"index":38,"sense_key":"run%2:30:02::","synset_id":"move.v.13"},{"index":39,"sense_key":"run%2:30:00::","synset_id":"melt.v.01"},{"index":40,"sense_key":"run%2:30:01::","synset_id":"ladder.v.01"},{"index":41,"sense_key":"run%2:30:04::","synset_id":"run.v.41"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"run%2:33:06::","call_ids":["senseval3.d001.s026.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"run%2:33:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2932,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.005984,"input_uncached_usd":0.005863999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2016251250170171}
{"item_id":"senseval3.d001.s026.t006","gold_sense_keys":["prospect%1:09:01::"],"candidates":[{"index":1,"sense_key":"prospect%1:26:00::","synset_id":"prospect.n.01"},{"index":2,"sense_key":"prospect%1:09:01::","synset_id":"expectation.n.01"},{"index":3,"sense_key":"prospect%1:18:00::","synset_id":"candidate.n.02"},{"index":4,"sense_key":"prospect%1:09:00::","synset_id":"view.n.02"},{"index":5,"sense_key":"prospect%1:04:00::","synset_id":"prognosis.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"prospect%1:09:01::","call_ids":["senseval3.d001.s026.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"prospect%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":873,"cached_input_tokens":0,"output_tokens":87,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0026160000000000003,"input_uncached_usd":0.001746,"input_cached_usd":0.0,"output_usd":0.0008700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5458092499757186}
{"item_id":"senseval3.d001.s026.t007","gold_sense_keys":["defeat%1:11:00::"],"candidates":[{"index":1,"sense_key":"defeat%1:11:00::","synset_id":"defeat.n.01"},{"index":2,"sense_key":"defeat%1:12:00::","synset_id":"frustration.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"defeat%1:11:00::","call_ids":["senseval3.d001.s026.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"defeat%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":718,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001556,"input_uncached_usd":0.001436,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.616537750000134}
{"item_id":"senseval3.d001.s026.t009","gold_sense_keys":["democratic%3:01:00::"],"candidates":[{"index":1,"sense_key":"democratic%3:00:00::","synset_id":"democratic.a.01"},{"index":2,"sense_key":"democratic%3:01:00::","synset_id":"democratic.a.02"},{"index":3,"sense_key":"democratic%5:00:00:common:01","synset_id":"democratic.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"democratic%3:01:00::","call_ids":["senseval3.d001.s026.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"democratic%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":787,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001694,"input_uncached_usd":0.0015739999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1528255410958081}
{"item_id":"senseval3.d001.s026.t010","gold_sense_keys":["opponent%1:18:02::"],"candidates":[{"index":1,"sense_key":"opponent%1:18:02::","synset_id":"opposition.n.04"},{"index":2,"sense_key":"opponent%1:18:00::","synset_id":"adversary.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"opponent%1:18:02::","call_ids":["senseval3.d001.s026.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"opponent%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":690,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00199,"input_uncached_usd":0.00138,"input_cached_usd":0.0,"output_usd":0.0006100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5579061660682783}
{"item_id":"senseval3.d001.s027.t000","gold_sense_keys":["extent%1:26:00::"],"candidates":[{"index":1,"sense_key":"extent%1:26:00::","synset_id":"extent.n.01"},{"index":2,"sense_key":"extent%1:07:00::","synset_id":"extent.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"extent%1:26:00::","call_ids":["senseval3.d001.s027.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"extent%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":696,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002042,"input_uncached_usd":0.001392,"input_cached_usd":0.0,"output_usd":0.0006500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9448605839861557}
{"item_id":"senseval3.d001.s027.t001","gold_sense_keys":["potential%5:00:00:prospective:00"],"candidates":[{"index":1,"sense_key":"potential%3:00:00::","synset_id":"potential.a.01"},{"index":2,"sense_key":"potential%5:00:00:prospective:00","synset_id":"likely.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"potential%5:00:00:prospective:00","call_ids":["senseval3.d001.s027.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"potential%5:00:00:prospective:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":696,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002062,"input_uncached_usd":0.001392,"input_cached_usd":0.0,"output_usd":0.00067,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5322725840378553}
{"item_id":"senseval3.d001.s027.t002","gold_sense_keys":["candidate%1:18:01::"],"candidates":[{"index":1,"sense_key":"candidate%1:18:01::","synset_id":"campaigner.n.01"},{"index":2,"sense_key":"candidate%1:18:00::","synset_id":"candidate.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"candidate%1:18:01::","call_ids":["senseval3.d001.s027.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"candidate%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":688,"cached_input_tokens":0,"output_tokens":177,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003146,"input_uncached_usd":0.0013759999999999998,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.7021255420986563}
{"item_id":"senseval3.d001.s027.t004","gold_sense_keys":["realize%2:31:00::","realize%2:31:01::"],"candidates":[{"index":1,"sense_key":"realize%2:31:01::","synset_id":"recognize.v.02"},{"index":2,"sense_key":"realize%2:31:00::","synset_id":"understand.v.02"},{"index":3,"sense_key":"realize%2:36:00::","synset_id":"realize.v.03"},{"index":4,"sense_key":"realize%2:40:01::","synset_id":"gain.v.08"},{"index":5,"sense_key":"realize%2:40:00::","synset_id":"realize.v.05"},{"index":6,"sense_key":"realize%2:36:07::","synset_id":"realize.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"realize%2:31:01::","call_ids":["senseval3.d001.s027.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"realize%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":982,"cached_input_tokens":0,"output_tokens":227,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004234,"input_uncached_usd":0.001964,"input_cached_usd":0.0,"output_usd":0.0022700000000000003,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.23863787506707}
{"item_id":"senseval3.d001.s027.t007","gold_sense_keys":["work%2:41:03::"],"candidates":[{"index":1,"sense_key":"work%2:41:02::","synset_id":"work.v.01"},{"index":2,"sense_key":"work%2:41:00::","synset_id":"work.v.02"},{"index":3,"sense_key":"work%2:41:03::","synset_id":"work.v.03"},{"index":4,"sense_key":"work%2:35:02::","synset_id":"function.v.01"},{"index":5,"sense_key":"work%2:36:00::","synset_id":"work.v.05"},{"index":6,"sense_key":"work%2:29:00::","synset_id":"exercise.v.03"},{"index":7,"sense_key":"work%2:38:00::","synset_id":"make.v.36"},{"index":8,"sense_key":"work%2:41:04::","synset_id":"work.v.08"},{"index":9,"sense_key":"work%2:41:05::","synset_id":"work.v.09"},{"index":10,"sense_key":"work%2:38:02::","synset_id":"work.v.10"},{"index":11,"sense_key":"work%2:36:01::","synset_id":"bring.v.03"},{"index":12,"sense_key":"work%2:41:01::","synset_id":"work.v.12"},{"index":13,"sense_key":"work%2:36:09::","synset_id":"cultivate.v.02"},{"index":14,"sense_key":"work%2:42:10::","synset_id":"work.v.14"},{"index":15,"sense_key":"work%2:41:10::","synset_id":"influence.v.01"},{"index":16,"sense_key":"work%2:41:11::","synset_id":"work.v.16"},{"index":17,"sense_key":"work%2:41:12::","synset_id":"work.v.17"},{"index":18,"sense_key":"work%2:39:10::","synset_id":"work.v.18"},{"index":19,"sense_key":"work%2:37:10::","synset_id":"work.v.19"},{"index":20,"sense_key":"work%2:36:12::","synset_id":"shape.v.02"},{"index":21,"sense_key":"work%2:35:14::","synset_id":"work.v.21"},{"index":22,"sense_key":"work%2:35:00::","synset_id":"knead.v.01"},{"index":23,"sense_key":"work%2:34:10::","synset_id":"exploit.v.01"},{"index":24,"sense_key":"work%2:31:13::","synset_id":"solve.v.01"},{"index":25,"sense_key":"work%2:30:14::","synset_id":"ferment.v.03"},{"index":26,"sense_key":"work%2:30:13::","synset_id":"sour.v.01"},{"index":27,"sense_key":"work%2:30:10::","synset_id":"work.v.27"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"work%2:41:03::","call_ids":["senseval3.d001.s027.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"work%2:41:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2224,"cached_input_tokens":0,"output_tokens":89,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.005338,"input_uncached_usd":0.004448,"input_cached_usd":0.0,"output_usd":0.0008900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6032237079925835}
{"item_id":"senseval3.d001.s027.t009","gold_sense_keys":["hinder%2:41:00::"],"candidates":[{"index":1,"sense_key":"hinder%2:41:01::","synset_id":"impede.v.01"},{"index":2,"sense_key":"hinder%2:41:00::","synset_id":"obstruct.v.01"},{"index":3,"sense_key":"hinder%2:33:00::","synset_id":"handicap.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"hinder%2:33:00::","call_ids":["senseval3.d001.s027.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"hinder%2:33:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":793,"cached_input_tokens":0,"output_tokens":140,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002986,"input_uncached_usd":0.001586,"input_cached_usd":0.0,"output_usd":0.0014000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.233480375027284}
{"item_id":"senseval3.d001.s027.t010","gold_sense_keys":["attempt%1:04:00::"],"candidates":[{"index":1,"sense_key":"attempt%1:04:00::","synset_id":"attempt.n.01"},{"index":2,"sense_key":"attempt%1:04:02::","synset_id":"attack.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"attempt%1:04:00::","call_ids":["senseval3.d001.s027.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"attempt%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":739,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001598,"input_uncached_usd":0.001478,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1979875420220196}
{"item_id":"senseval3.d001.s027.t012","gold_sense_keys":["slate%1:10:02::"],"candidates":[{"index":1,"sense_key":"slate%1:06:00::","synset_id":"slate.n.01"},{"index":2,"sense_key":"slate%1:27:01::","synset_id":"slate.n.02"},{"index":3,"sense_key":"slate%1:27:00::","synset_id":"slate.n.03"},{"index":4,"sense_key":"slate%1:10:02::","synset_id":"slate.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"slate%1:10:02::","call_ids":["senseval3.d001.s027.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"slate%1:10:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":761,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001642,"input_uncached_usd":0.001522,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.425234208931215}
{"item_id":"senseval3.d001.s027.t014","gold_sense_keys":["candidate%1:18:01::"],"candidates":[{"index":1,"sense_key":"candidate%1:18:01::","synset_id":"campaigner.n.01"},{"index":2,"sense_key":"candidate%1:18:00::","synset_id":"candidate.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"candidate%1:18:01::","call_ids":["senseval3.d001.s027.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"candidate%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":688,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0019359999999999998,"input_uncached_usd":0.0013759999999999998,"input_cached_usd":0.0,"output_usd":0.0005600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3702032500877976}
{"item_id":"senseval3.d001.s028.t003","gold_sense_keys":["partial%5:00:00:incomplete:00"],"candidates":[{"index":1,"sense_key":"partial%5:00:00:incomplete:00","synset_id":"partial.s.01"},{"index":2,"sense_key":"partial%3:00:01::","synset_id":"partial.a.02"},{"index":3,"sense_key":"partial%5:00:00:inclined:02","synset_id":"fond.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"partial%5:00:00:incomplete:00","call_ids":["senseval3.d001.s028.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"partial%5:00:00:incomplete:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":797,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001714,"input_uncached_usd":0.001594,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.177419832907617}
{"item_id":"senseval3.d001.s028.t004","gold_sense_keys":["reason%1:09:01::","reason%1:10:00::","reason%1:10:01::"],"candidates":[{"index":1,"sense_key":"reason%1:16:00::","synset_id":"reason.n.01"},{"index":2,"sense_key":"reason%1:10:01::","synset_id":"reason.n.02"},{"index":3,"sense_key":"reason%1:09:00::","synset_id":"reason.n.03"},{"index":4,"sense_key":"reason%1:26:00::","synset_id":"rationality.n.01"},{"index":5,"sense_key":"reason%1:10:00::","synset_id":"cause.n.02"},{"index":6,"sense_key":"reason%1:09:01::","synset_id":"reason.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"reason%1:10:01::","call_ids":["senseval3.d001.s028.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"reason%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1049,"cached_input_tokens":0,"output_tokens":151,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003608,"input_uncached_usd":0.002098,"input_cached_usd":0.0,"output_usd":0.00151,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9220417499309406}
{"item_id":"senseval3.d001.s029.t000","gold_sense_keys":["extent%1:26:00::"],"candidates":[{"index":1,"sense_key":"extent%1:26:00::","synset_id":"extent.n.01"},{"index":2,"sense_key":"extent%1:07:00::","synset_id":"extent.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"extent%1:26:00::","call_ids":["senseval3.d001.s029.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"extent%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":722,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0020239999999999998,"input_uncached_usd":0.001444,"input_cached_usd":0.0,"output_usd":0.00058,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4233246250078082}
{"item_id":"senseval3.d001.s029.t001","gold_sense_keys":["democratic%3:01:00::"],"candidates":[{"index":1,"sense_key":"democratic%3:00:00::","synset_id":"democratic.a.01"},{"index":2,"sense_key":"democratic%3:01:00::","synset_id":"democratic.a.02"},{"index":3,"sense_key":"democratic%5:00:00:common:01","synset_id":"democratic.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"democratic%3:01:00::","call_ids":["senseval3.d001.s029.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"democratic%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":809,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001738,"input_uncached_usd":0.0016179999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.03612345806323}
{"item_id":"senseval3.d001.s029.t003","gold_sense_keys":["hold%2:40:00::"],"candidates":[{"index":1,"sense_key":"hold%2:42:00::","synset_id":"keep.v.01"},{"index":2,"sense_key":"hold%2:35:00::","synset_id":"hold.v.02"},{"index":3,"sense_key":"hold%2:36:00::","synset_id":"hold.v.03"},{"index":4,"sense_key":"hold%2:40:00::","synset_id":"have.v.01"},{"index":5,"sense_key":"hold%2:31:01::","synset_id":"deem.v.01"},{"index":6,"sense_key":"hold%2:37:00::","synset_id":"harbor.v.01"},{"index":7,"sense_key":"hold%2:35:03::","synset_id":"restrain.v.03"},{"index":8,"sense_key":"hold%2:40:02::","synset_id":"retain.v.03"},{"index":9,"sense_key":"hold%2:40:04::","synset_id":"bear.v.11"},{"index":10,"sense_key":"hold%2:35:01::","synset_id":"hold.v.10"},{"index":11,"sense_key":"hold%2:42:13::","synset_id":"hold.v.11"},{"index":12,"sense_key":"hold%2:42:05::","synset_id":"accommodate.v.04"},{"index":13,"sense_key":"hold%2:42:01::","synset_id":"hold.v.13"},{"index":14,"sense_key":"hold%2:35:02::","synset_id":"hold.v.14"},{"index":15,"sense_key":"hold%2:42:02::","synset_id":"prevail.v.02"},{"index":16,"sense_key":"hold%2:31:08::","synset_id":"hold.v.16"},{"index":17,"sense_key":"hold%2:42:07::","synset_id":"hold.v.17"},{"index":18,"sense_key":"hold%2:42:14::","synset_id":"contain.v.05"},{"index":19,"sense_key":"hold%2:41:00::","synset_id":"reserve.v.04"},{"index":20,"sense_key":"hold%2:33:05::","synset_id":"defend.v.03"},{"index":21,"sense_key":"hold%2:32:13::","synset_id":"oblige.v.02"},{"index":22,"sense_key":"hold%2:31:04::","synset_id":"hold.v.22"},{"index":23,"sense_key":"hold%2:31:10::","synset_id":"hold.v.23"},{"index":24,"sense_key":"hold%2:42:08::","synset_id":"defy.v.01"},{"index":25,"sense_key":"hold%2:42:04::","synset_id":"apply.v.02"},{"index":26,"sense_key":"hold%2:42:12::","synset_id":"hold.v.26"},{"index":27,"sense_key":"hold%2:41:15::","synset_id":"control.v.02"},{"index":28,"sense_key":"hold%2:41:13::","synset_id":"hold.v.28"},{"index":29,"sense_key":"hold%2:41:14::","synset_id":"hold.v.29"},{"index":30,"sense_key":"hold%2:38:05::","synset_id":"halt.v.01"},{"index":31,"sense_key":"hold%2:35:09::","synset_id":"hold.v.31"},{"index":32,"sense_key":"hold%2:34:00::","synset_id":"carry.v.33"},{"index":33,"sense_key":"hold%2:33:00::","synset_id":"hold.v.33"},{"index":34,"sense_key":"hold%2:32:11::","synset_id":"declare.v.04"},{"index":35,"sense_key":"hold%2:32:15::","synset_id":"agree.v.01"},{"index":36,"sense_key":"hold%2:29:04::","synset_id":"hold.v.36"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"hold%2:40:00::","call_ids":["senseval3.d001.s029.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"hold%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2952,"cached_input_tokens":0,"output_tokens":128,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.007183999999999999,"input_uncached_usd":0.0059039999999999995,"input_cached_usd":0.0,"output_usd":0.00128,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.815397790982388}
{"item_id":"senseval3.d001.s029.t005","gold_sense_keys":["share%1:04:00::"],"candidates":[{"index":1,"sense_key":"share%1:21:00::","synset_id":"share.n.01"},{"index":2,"sense_key":"share%1:21:01::","synset_id":"share.n.02"},{"index":3,"sense_key":"share%1:04:00::","synset_id":"parcel.n.02"},{"index":4,"sense_key":"share%1:04:01::","synset_id":"contribution.n.01"},{"index":5,"sense_key":"share%1:06:00::","synset_id":"plowshare.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"share%1:04:00::","call_ids":["senseval3.d001.s029.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"share%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":955,"cached_input_tokens":0,"output_tokens":131,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0032199999999999998,"input_uncached_usd":0.0019099999999999998,"input_cached_usd":0.0,"output_usd":0.0013100000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8379074590047821}
{"item_id":"senseval3.d001.s029.t006","gold_sense_keys":["power%1:07:00::"],"candidates":[{"index":1,"sense_key":"power%1:07:00::","synset_id":"power.n.01"},{"index":2,"sense_key":"power%1:19:00::","synset_id":"power.n.02"},{"index":3,"sense_key":"power%1:09:01::","synset_id":"ability.n.02"},{"index":4,"sense_key":"power%1:26:00::","synset_id":"office.n.04"},{"index":5,"sense_key":"power%1:18:00::","synset_id":"power.n.05"},{"index":6,"sense_key":"power%1:10:00::","synset_id":"exponent.n.03"},{"index":7,"sense_key":"power%1:07:02::","synset_id":"might.n.01"},{"index":8,"sense_key":"power%1:14:00::","synset_id":"world_power.n.01"},{"index":9,"sense_key":"power%1:18:01::","synset_id":"baron.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"power%1:07:00::","call_ids":["senseval3.d001.s029.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"power%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1148,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002826,"input_uncached_usd":0.002296,"input_cached_usd":0.0,"output_usd":0.0005300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0226476249517873}
{"item_id":"senseval3.d001.s029.t007","gold_sense_keys":["able%3:00:00::"],"candidates":[{"index":1,"sense_key":"able%3:00:00::","synset_id":"able.a.01"},{"index":2,"sense_key":"able%5:00:00:competent:00","synset_id":"able.s.01"},{"index":3,"sense_key":"able%5:00:00:capable:00","synset_id":"able.s.02"},{"index":4,"sense_key":"able%5:00:00:fit:01","synset_id":"able.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"able%3:00:00::","call_ids":["senseval3.d001.s029.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"able%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":864,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002338,"input_uncached_usd":0.001728,"input_cached_usd":0.0,"output_usd":0.0006100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6769857499748468}
{"item_id":"senseval3.d001.s029.t008","gold_sense_keys":["translate%2:30:01::"],"candidates":[{"index":1,"sense_key":"translate%2:32:00::","synset_id":"translate.v.01"},{"index":2,"sense_key":"translate%2:30:01::","synset_id":"translate.v.02"},{"index":3,"sense_key":"translate%2:31:00::","synset_id":"understand.v.03"},{"index":4,"sense_key":"translate%2:35:00::","synset_id":"translate.v.04"},{"index":5,"sense_key":"translate%2:30:00::","synset_id":"translate.v.05"},{"index":6,"sense_key":"translate%2:42:03::","synset_id":"translate.v.06"},{"index":7,"sense_key":"translate%2:42:00::","synset_id":"translate.v.07"},{"index":8,"sense_key":"translate%2:38:04::","synset_id":"translate.v.08"},{"index":9,"sense_key":"translate%2:32:06::","synset_id":"translate.v.09"},{"index":10,"sense_key":"translate%2:32:05::","synset_id":"translate.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"translate%2:30:01::","call_ids":["senseval3.d001.s029.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"translate%2:30:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1183,"cached_input_tokens":0,"output_tokens":130,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003666,"input_uncached_usd":0.002366,"input_cached_usd":0.0,"output_usd":0.0013000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.849051250028424}
{"item_id":"senseval3.d001.s029.t010","gold_sense_keys":["local%3:01:01::"],"candidates":[{"index":1,"sense_key":"local%3:00:01::","synset_id":"local.a.01"},{"index":2,"sense_key":"local%3:01:01::","synset_id":"local.a.02"},{"index":3,"sense_key":"local%3:00:02::","synset_id":"local.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"local%3:01:01::","call_ids":["senseval3.d001.s029.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"local%3:01:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":774,"cached_input_tokens":0,"output_tokens":129,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0028380000000000002,"input_uncached_usd":0.001548,"input_cached_usd":0.0,"output_usd":0.0012900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0360286249779165}
{"item_id":"senseval3.d001.s029.t011","gold_sense_keys":["benefit%1:07:00::"],"candidates":[{"index":1,"sense_key":"benefit%1:21:00::","synset_id":"benefit.n.01"},{"index":2,"sense_key":"benefit%1:07:00::","synset_id":"benefit.n.02"},{"index":3,"sense_key":"benefit%1:10:00::","synset_id":"benefit.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"benefit%1:07:00::","call_ids":["senseval3.d001.s029.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"benefit%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":737,"cached_input_tokens":0,"output_tokens":125,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002724,"input_uncached_usd":0.0014739999999999998,"input_cached_usd":0.0,"output_usd":0.00125,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0414332080399618}
{"item_id":"senseval3.d001.s029.t015","gold_sense_keys":["have%2:40:00::","have%2:42:12::"],"candidates":[{"index":1,"sense_key":"have%2:40:00::","synset_id":"have.v.01"},{"index":2,"sense_key":"have%2:42:00::","synset_id":"have.v.02"},{"index":3,"sense_key":"have%2:39:06::","synset_id":"experience.v.03"},{"index":4,"sense_key":"have%2:40:04::","synset_id":"own.v.01"},{"index":5,"sense_key":"have%2:30:00::","synset_id":"get.v.03"},{"index":6,"sense_key":"have%2:34:00::","synset_id":"consume.v.02"},{"index":7,"sense_key":"have%2:41:00::","synset_id":"have.v.07"},{"index":8,"sense_key":"have%2:36:00::","synset_id":"hold.v.03"},{"index":9,"sense_key":"have%2:40:01::","synset_id":"have.v.09"},{"index":10,"sense_key":"have%2:42:12::","synset_id":"have.v.10"},{"index":11,"sense_key":"have%2:30:01::","synset_id":"have.v.11"},{"index":12,"sense_key":"have%2:29:02::","synset_id":"have.v.12"},{"index":13,"sense_key":"have%2:32:00::","synset_id":"induce.v.02"},{"index":14,"sense_key":"have%2:40:05::","synset_id":"accept.v.02"},{"index":15,"sense_key":"have%2:40:03::","synset_id":"receive.v.01"},{"index":16,"sense_key":"have%2:29:05::","synset_id":"suffer.v.02"},{"index":17,"sense_key":"have%2:40:02::","synset_id":"have.v.17"},{"index":18,"sense_key":"have%2:29:00::","synset_id":"give_birth.v.01"},{"index":19,"sense_key":"have%2:35:00::","synset_id":"take.v.35"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"have%2:40:00::","call_ids":["senseval3.d001.s029.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"have%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1779,"cached_input_tokens":0,"output_tokens":113,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004688,"input_uncached_usd":0.003558,"input_cached_usd":0.0,"output_usd":0.0011300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8054130830569193}
{"item_id":"senseval3.d001.s029.t016","gold_sense_keys":["strong%3:00:00::"],"candidates":[{"index":1,"sense_key":"strong%3:00:00::","synset_id":"strong.a.01"},{"index":2,"sense_key":"strong%5:00:00:intense:00","synset_id":"strong.s.01"},{"index":3,"sense_key":"strong%5:00:00:powerful:00","synset_id":"potent.s.02"},{"index":4,"sense_key":"strong%3:00:04::","synset_id":"potent.a.03"},{"index":5,"sense_key":"strong%5:00:00:invulnerable:00","synset_id":"impregnable.s.01"},{"index":6,"sense_key":"strong%5:00:00:sound:01","synset_id":"solid.s.05"},{"index":7,"sense_key":"strong%5:00:00:irregular:00","synset_id":"strong.s.05"},{"index":8,"sense_key":"strong%5:00:00:alcoholic:00","synset_id":"hard.s.04"},{"index":9,"sense_key":"strong%5:00:00:fresh:01","synset_id":"strong.s.07"},{"index":10,"sense_key":"strong%5:00:00:forceful:00","synset_id":"firm.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"strong%3:00:00::","call_ids":["senseval3.d001.s029.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"strong%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1257,"cached_input_tokens":0,"output_tokens":103,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003544,"input_uncached_usd":0.0025139999999999997,"input_cached_usd":0.0,"output_usd":0.00103,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.651702875038609}
{"item_id":"senseval3.d001.s029.t017","gold_sense_keys":["incentive%1:16:00::"],"candidates":[{"index":1,"sense_key":"incentive%1:16:00::","synset_id":"incentive.n.01"},{"index":2,"sense_key":"incentive%1:21:00::","synset_id":"bonus.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"incentive%1:16:00::","call_ids":["senseval3.d001.s029.t017__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"incentive%1:16:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":714,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0015480000000000001,"input_uncached_usd":0.001428,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0951617088867351}
{"item_id":"senseval3.d001.s029.t018","gold_sense_keys":["keep%2:41:02::"],"candidates":[{"index":1,"sense_key":"keep%2:42:00::","synset_id":"keep.v.01"},{"index":2,"sense_key":"keep%2:42:07::","synset_id":"continue.v.01"},{"index":3,"sense_key":"keep%2:40:00::","synset_id":"keep.v.03"},{"index":4,"sense_key":"keep%2:41:01::","synset_id":"prevent.v.02"},{"index":5,"sense_key":"keep%2:41:03::","synset_id":"observe.v.09"},{"index":6,"sense_key":"keep%2:31:00::","synset_id":"observe.v.08"},{"index":7,"sense_key":"keep%2:40:01::","synset_id":"keep.v.07"},{"index":8,"sense_key":"keep%2:32:00::","synset_id":"keep.v.08"},{"index":9,"sense_key":"keep%2:42:01::","synset_id":"keep.v.09"},{"index":10,"sense_key":"keep%2:41:02::","synset_id":"retain.v.02"},{"index":11,"sense_key":"keep%2:34:00::","synset_id":"sustain.v.04"},{"index":12,"sense_key":"keep%2:42:03::","synset_id":"keep.v.12"},{"index":13,"sense_key":"keep%2:41:05::","synset_id":"observe.v.06"},{"index":14,"sense_key":"keep%2:41:00::","synset_id":"restrain.v.01"},{"index":15,"sense_key":"keep%2:42:02::","synset_id":"keep.v.15"},{"index":16,"sense_key":"keep%2:42:12::","synset_id":"keep.v.16"},{"index":17,"sense_key":"keep%2:40:09::","synset_id":"keep_open.v.01"},{"index":18,"sense_key":"keep%2:40:13::","synset_id":"keep.v.18"},{"index":19,"sense_key":"keep%2:40:12::","synset_id":"keep.v.19"},{"index":20,"sense_key":"keep%2:40:10::","synset_id":"keep.v.20"},{"index":21,"sense_key":"keep%2:35:10::","synset_id":"keep.v.21"},{"index":22,"sense_key":"keep%2:30:10::","synset_id":"preserve.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"keep%2:42:00::","call_ids":["senseval3.d001.s029.t018__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"keep%2:42:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1957,"cached_input_tokens":0,"output_tokens":112,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.005034,"input_uncached_usd":0.003914,"input_cached_usd":0.0,"output_usd":0.0011200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4741790000116453}
{"item_id":"senseval3.d001.s029.t020","gold_sense_keys":["democrat%1:18:01::"],"candidates":[{"index":1,"sense_key":"democrat%1:18:01::","synset_id":"democrat.n.01"},{"index":2,"sense_key":"democrat%1:18:00::","synset_id":"democrat.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"democrat%1:18:01::","call_ids":["senseval3.d001.s029.t020__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"democrat%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":689,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001498,"input_uncached_usd":0.0013779999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2691607910674065}
{"item_id":"senseval3.d001.s029.t021","gold_sense_keys":["office%1:26:00::"],"candidates":[{"index":1,"sense_key":"office%1:06:00::","synset_id":"office.n.01"},{"index":2,"sense_key":"office%1:14:01::","synset_id":"agency.n.01"},{"index":3,"sense_key":"office%1:04:01::","synset_id":"function.n.03"},{"index":4,"sense_key":"office%1:26:00::","synset_id":"office.n.04"},{"index":5,"sense_key":"office%1:14:00::","synset_id":"office.n.05"},{"index":6,"sense_key":"office%1:04:00::","synset_id":"office.n.06"},{"index":7,"sense_key":"office%1:04:02::","synset_id":"position.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"office%1:26:00::","call_ids":["senseval3.d001.s029.t021__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"office%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1059,"cached_input_tokens":0,"output_tokens":71,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0028280000000000002,"input_uncached_usd":0.002118,"input_cached_usd":0.0,"output_usd":0.00071,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9476536250440404}
{"item_id":"senseval3.d001.s030.t001","gold_sense_keys":["suggest%2:32:02::"],"candidates":[{"index":1,"sense_key":"suggest%2:32:00::","synset_id":"propose.v.01"},{"index":2,"sense_key":"suggest%2:32:04::","synset_id":"hint.v.01"},{"index":3,"sense_key":"suggest%2:32:02::","synset_id":"suggest.v.03"},{"index":4,"sense_key":"suggest%2:32:03::","synset_id":"indicate.v.05"},{"index":5,"sense_key":"suggest%2:32:01::","synset_id":"suggest.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"suggest%2:32:02::","call_ids":["senseval3.d001.s030.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"suggest%2:32:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":926,"cached_input_tokens":0,"output_tokens":76,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0026119999999999997,"input_uncached_usd":0.001852,"input_cached_usd":0.0,"output_usd":0.00076,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8602867090376094}
{"item_id":"senseval3.d001.s030.t002","gold_sense_keys":["republican%1:18:01::"],"candidates":[{"index":1,"sense_key":"republican%1:18:01::","synset_id":"republican.n.01"},{"index":2,"sense_key":"republican%1:18:00::","synset_id":"republican.n.02"},{"index":3,"sense_key":"republican%1:17:00::","synset_id":"republican.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"republican%1:18:01::","call_ids":["senseval3.d001.s030.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"republican%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":780,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00168,"input_uncached_usd":0.00156,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.158181666978635}
{"item_id":"senseval3.d001.s030.t003","gold_sense_keys":["fare%2:42:00::"],"candidates":[{"index":1,"sense_key":"fare%2:42:00::","synset_id":"do.v.04"},{"index":2,"sense_key":"fare%2:34:00::","synset_id":"fare.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"fare%2:42:00::","call_ids":["senseval3.d001.s030.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"fare%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":729,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001578,"input_uncached_usd":0.0014579999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.40138808300253}
{"item_id":"senseval3.d001.s030.t004","gold_sense_keys":["race%1:11:01::"],"candidates":[{"index":1,"sense_key":"race%1:11:01::","synset_id":"race.n.01"},{"index":2,"sense_key":"race%1:11:00::","synset_id":"race.n.02"},{"index":3,"sense_key":"race%1:14:00::","synset_id":"race.n.03"},{"index":4,"sense_key":"race%1:14:01::","synset_id":"subspecies.n.01"},{"index":5,"sense_key":"race%1:19:00::","synset_id":"slipstream.n.01"},{"index":6,"sense_key":"race%1:06:00::","synset_id":"raceway.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"race%1:11:01::","call_ids":["senseval3.d001.s030.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"race%1:11:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":939,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0019979999999999998,"input_uncached_usd":0.001878,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.7785100840264931}
{"item_id":"senseval3.d001.s030.t005","gold_sense_keys":["campaign%1:11:00::"],"candidates":[{"index":1,"sense_key":"campaign%1:11:00::","synset_id":"political_campaign.n.01"},{"index":2,"sense_key":"campaign%1:04:02::","synset_id":"campaign.n.02"},{"index":3,"sense_key":"campaign%1:04:00::","synset_id":"campaign.n.03"},{"index":4,"sense_key":"campaign%1:04:01::","synset_id":"campaign.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"campaign%1:11:00::","call_ids":["senseval3.d001.s030.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"campaign%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":917,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001954,"input_uncached_usd":0.001834,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.101512874942273}
{"item_id":"senseval3.d001.s031.t000","gold_sense_keys":["local%3:01:01::"],"candidates":[{"index":1,"sense_key":"local%3:00:01::","synset_id":"local.a.01"},{"index":2,"sense_key":"local%3:01:01::","synset_id":"local.a.02"},{"index":3,"sense_key":"local%3:00:02::","synset_id":"local.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"local%3:01:01::","call_ids":["senseval3.d001.s031.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"local%3:01:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":771,"cached_input_tokens":0,"output_tokens":122,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002762,"input_uncached_usd":0.001542,"input_cached_usd":0.0,"output_usd":0.0012200000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9513549170223996}
{"item_id":"senseval3.d001.s031.t003","gold_sense_keys":["national%3:00:00::","national%3:00:01::","national%3:01:00::","national%5:00:00:domestic:00"],"candidates":[{"index":1,"sense_key":"national%3:01:00::","synset_id":"national.a.01"},{"index":2,"sense_key":"national%3:00:00::","synset_id":"national.a.02"},{"index":3,"sense_key":"national%3:00:01::","synset_id":"national.a.03"},{"index":4,"sense_key":"national%5:00:00:public:00","synset_id":"national.s.01"},{"index":5,"sense_key":"national%5:00:00:domestic:00","synset_id":"home.s.01"},{"index":6,"sense_key":"national%3:01:02::","synset_id":"national.a.06"},{"index":7,"sense_key":"national%3:01:01::","synset_id":"national.a.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"national%3:00:01::","call_ids":["senseval3.d001.s031.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"national%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":970,"cached_input_tokens":0,"output_tokens":93,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00287,"input_uncached_usd":0.0019399999999999999,"input_cached_usd":0.0,"output_usd":0.00093,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.773007999989204}
{"item_id":"senseval3.d001.s031.t004","gold_sense_keys":["policy%1:09:00::"],"candidates":[{"index":1,"sense_key":"policy%1:09:00::","synset_id":"policy.n.01"},{"index":2,"sense_key":"policy%1:10:00::","synset_id":"policy.n.02"},{"index":3,"sense_key":"policy%1:10:02::","synset_id":"policy.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"policy%1:09:00::","call_ids":["senseval3.d001.s031.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"policy%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":796,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001712,"input_uncached_usd":0.0015919999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.804158707964234}
{"item_id":"senseval3.d001.s031.t005","gold_sense_keys":["making%1:04:00::"],"candidates":[{"index":1,"sense_key":"making%1:04:00::","synset_id":"devising.n.01"},{"index":2,"sense_key":"making%1:07:00::","synset_id":"qualification.n.01"},{"index":3,"sense_key":"making%1:06:00::","synset_id":"making.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"making%1:04:00::","call_ids":["senseval3.d001.s031.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"making%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":815,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00175,"input_uncached_usd":0.00163,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":10.098525417037308}
{"item_id":"senseval3.d001.s031.t007","gold_sense_keys":["chamber%1:14:00::"],"candidates":[{"index":1,"sense_key":"chamber%1:06:00::","synset_id":"chamber.n.01"},{"index":2,"sense_key":"chamber%1:08:00::","synset_id":"chamber.n.02"},{"index":3,"sense_key":"chamber%1:06:02::","synset_id":"chamber.n.03"},{"index":4,"sense_key":"chamber%1:14:00::","synset_id":"chamber.n.04"},{"index":5,"sense_key":"chamber%1:06:01::","synset_id":"bedroom.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"chamber%1:14:00::","call_ids":["senseval3.d001.s031.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"chamber%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":850,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00182,"input_uncached_usd":0.0017,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1442705410299823}
{"item_id":"senseval3.d001.s031.t009","gold_sense_keys":["expect%2:31:00::"],"candidates":[{"index":1,"sense_key":"expect%2:31:00::","synset_id":"expect.v.01"},{"index":2,"sense_key":"expect%2:32:00::","synset_id":"ask.v.04"},{"index":3,"sense_key":"expect%2:31:01::","synset_id":"expect.v.03"},{"index":4,"sense_key":"expect%2:31:05::","synset_id":"expect.v.04"},{"index":5,"sense_key":"expect%2:29:02::","synset_id":"expect.v.05"},{"index":6,"sense_key":"expect%2:29:00::","synset_id":"have_a_bun_in_the_oven.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"expect%2:31:00::","call_ids":["senseval3.d001.s031.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"expect%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":987,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002594,"input_uncached_usd":0.001974,"input_cached_usd":0.0,"output_usd":0.00062,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.931441375054419}
{"item_id":"senseval3.d001.s031.t010","gold_sense_keys":["republican%1:18:01::"],"candidates":[{"index":1,"sense_key":"republican%1:18:01::","synset_id":"republican.n.01"},{"index":2,"sense_key":"republican%1:18:00::","synset_id":"republican.n.02"},{"index":3,"sense_key":"republican%1:17:00::","synset_id":"republican.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"republican%1:18:01::","call_ids":["senseval3.d001.s031.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"republican%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":754,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001628,"input_uncached_usd":0.001508,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0934584999922663}
{"item_id":"senseval3.d001.s031.t011","gold_sense_keys":["willing%3:00:00::"],"candidates":[{"index":1,"sense_key":"willing%3:00:00::","synset_id":"willing.a.01"},{"index":2,"sense_key":"willing%5:00:00:voluntary:01","synset_id":"uncoerced.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"willing%3:00:00::","call_ids":["senseval3.d001.s031.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"willing%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":721,"cached_input_tokens":0,"output_tokens":96,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002402,"input_uncached_usd":0.001442,"input_cached_usd":0.0,"output_usd":0.0009600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5302376670297235}
{"item_id":"senseval3.d001.s031.t012","gold_sense_keys":["engage%2:41:06::"],"candidates":[{"index":1,"sense_key":"engage%2:41:06::","synset_id":"prosecute.v.03"},{"index":2,"sense_key":"engage%2:31:00::","synset_id":"absorb.v.09"},{"index":3,"sense_key":"engage%2:41:01::","synset_id":"hire.v.01"},{"index":4,"sense_key":"engage%2:41:00::","synset_id":"engage.v.04"},{"index":5,"sense_key":"engage%2:32:00::","synset_id":"betroth.v.01"},{"index":6,"sense_key":"engage%2:30:00::","synset_id":"engage.v.06"},{"index":7,"sense_key":"engage%2:41:12::","synset_id":"engage.v.07"},{"index":8,"sense_key":"engage%2:40:01::","synset_id":"engage.v.08"},{"index":9,"sense_key":"engage%2:40:00::","synset_id":"lease.v.04"},{"index":10,"sense_key":"engage%2:35:00::","synset_id":"engage.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"engage%2:41:06::","call_ids":["senseval3.d001.s031.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"engage%2:41:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1196,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002512,"input_uncached_usd":0.002392,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2374035420361906}
{"item_id":"senseval3.d001.s031.t013","gold_sense_keys":["local%3:00:01::"],"candidates":[{"index":1,"sense_key":"local%3:00:01::","synset_id":"local.a.01"},{"index":2,"sense_key":"local%3:01:01::","synset_id":"local.a.02"},{"index":3,"sense_key":"local%3:00:02::","synset_id":"local.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"local%3:01:01::","call_ids":["senseval3.d001.s031.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"local%3:01:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":771,"cached_input_tokens":0,"output_tokens":97,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002512,"input_uncached_usd":0.001542,"input_cached_usd":0.0,"output_usd":0.00097,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6682962919585407}
{"item_id":"senseval3.d001.s031.t015","gold_sense_keys":["democratic%3:01:00::"],"candidates":[{"index":1,"sense_key":"democratic%3:00:00::","synset_id":"democratic.a.01"},{"index":2,"sense_key":"democratic%3:01:00::","synset_id":"democratic.a.02"},{"index":3,"sense_key":"democratic%5:00:00:common:01","synset_id":"democratic.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"democratic%3:01:00::","call_ids":["senseval3.d001.s031.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"democratic%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":806,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001732,"input_uncached_usd":0.001612,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.09257316600997}
{"item_id":"senseval3.d001.s031.t016","gold_sense_keys":["counterpart%1:09:00::"],"candidates":[{"index":1,"sense_key":"counterpart%1:09:00::","synset_id":"counterpart.n.01"},{"index":2,"sense_key":"counterpart%1:06:00::","synset_id":"counterpart.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"counterpart%1:09:00::","call_ids":["senseval3.d001.s031.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"counterpart%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":714,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0015480000000000001,"input_uncached_usd":0.001428,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1455087920185179}
{"item_id":"senseval3.d001.s032.t000","gold_sense_keys":["empirical%3:00:00::"],"candidates":[{"index":1,"sense_key":"empirical%3:00:00::","synset_id":"empirical.a.01"},{"index":2,"sense_key":"empirical%3:01:00::","synset_id":"empiric.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"empirical%3:00:00::","call_ids":["senseval3.d001.s032.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"empirical%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":736,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001592,"input_uncached_usd":0.001472,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.02696945797652}
{"item_id":"senseval3.d001.s032.t001","gold_sense_keys":["support%1:04:02::"],"candidates":[{"index":1,"sense_key":"support%1:04:04::","synset_id":"support.n.01"},{"index":2,"sense_key":"support%1:04:00::","synset_id":"support.n.02"},{"index":3,"sense_key":"support%1:09:00::","synset_id":"support.n.03"},{"index":4,"sense_key":"support%1:04:03::","synset_id":"support.n.04"},{"index":5,"sense_key":"support%1:04:02::","synset_id":"documentation.n.03"},{"index":6,"sense_key":"support%1:21:00::","synset_id":"support.n.06"},{"index":7,"sense_key":"support%1:06:01::","synset_id":"support.n.07"},{"index":8,"sense_key":"support%1:04:01::","synset_id":"support.n.08"},{"index":9,"sense_key":"support%1:10:01::","synset_id":"accompaniment.n.02"},{"index":10,"sense_key":"support%1:06:00::","synset_id":"support.n.10"},{"index":11,"sense_key":"support%1:21:01::","synset_id":"support.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"support%1:04:02::","call_ids":["senseval3.d001.s032.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"support%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1341,"cached_input_tokens":0,"output_tokens":76,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0034419999999999997,"input_uncached_usd":0.002682,"input_cached_usd":0.0,"output_usd":0.00076,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.418747500050813}
{"item_id":"senseval3.d001.s033.t000","gold_sense_keys":["piece%1:06:00::"],"candidates":[{"index":1,"sense_key":"piece%1:06:00::","synset_id":"piece.n.01"},{"index":2,"sense_key":"piece%1:06:05::","synset_id":"piece.n.02"},{"index":3,"sense_key":"piece%1:17:00::","synset_id":"part.n.03"},{"index":4,"sense_key":"piece%1:10:01::","synset_id":"musical_composition.n.01"},{"index":5,"sense_key":"piece%1:11:00::","synset_id":"piece.n.05"},{"index":6,"sense_key":"piece%1:10:02::","synset_id":"piece.n.06"},{"index":7,"sense_key":"piece%1:06:03::","synset_id":"firearm.n.01"},{"index":8,"sense_key":"piece%1:13:00::","synset_id":"piece.n.08"},{"index":9,"sense_key":"piece%1:07:00::","synset_id":"piece.n.09"},{"index":10,"sense_key":"piece%1:06:04::","synset_id":"objet_d'art.n.01"},{"index":11,"sense_key":"piece%1:28:00::","synset_id":"while.n.01"},{"index":12,"sense_key":"piece%1:21:00::","synset_id":"slice.n.01"},{"index":13,"sense_key":"piece%1:06:02::","synset_id":"man.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"piece%1:06:05::","call_ids":["senseval3.d001.s033.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"piece%1:06:05::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1389,"cached_input_tokens":0,"output_tokens":318,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.005958,"input_uncached_usd":0.002778,"input_cached_usd":0.0,"output_usd":0.00318,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.9191705000121146}
{"item_id":"senseval3.d001.s033.t001","gold_sense_keys":["evidence%1:09:00::","evidence%1:10:01::"],"candidates":[{"index":1,"sense_key":"evidence%1:09:00::","synset_id":"evidence.n.01"},{"index":2,"sense_key":"evidence%1:10:01::","synset_id":"evidence.n.02"},{"index":3,"sense_key":"evidence%1:10:00::","synset_id":"evidence.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"evidence%1:09:00::","call_ids":["senseval3.d001.s033.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"evidence%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":783,"cached_input_tokens":0,"output_tokens":148,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003046,"input_uncached_usd":0.001566,"input_cached_usd":0.0,"output_usd":0.0014800000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.280396083020605}
{"item_id":"senseval3.d001.s033.t004","gold_sense_keys":["assumption%1:09:00::","assumption%1:10:00::"],"candidates":[{"index":1,"sense_key":"assumption%1:10:00::","synset_id":"premise.n.01"},{"index":2,"sense_key":"assumption%1:09:00::","synset_id":"assumption.n.02"},{"index":3,"sense_key":"assumption%1:04:00::","synset_id":"assumption.n.03"},{"index":4,"sense_key":"assumption%1:28:00::","synset_id":"assumption.n.04"},{"index":5,"sense_key":"assumption%1:11:00::","synset_id":"assumption.n.05"},{"index":6,"sense_key":"assumption%1:07:00::","synset_id":"presumption.n.03"},{"index":7,"sense_key":"assumption%1:04:01::","synset_id":"assumption.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"assumption%1:09:00::","call_ids":["senseval3.d001.s033.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"assumption%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1124,"cached_input_tokens":0,"output_tokens":312,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.005368,"input_uncached_usd":0.002248,"input_cached_usd":0.0,"output_usd":0.0031200000000000004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":6.236623749951832}
{"item_id":"senseval3.d001.s033.t005","gold_sense_keys":["democratic%3:01:00::"],"candidates":[{"index":1,"sense_key":"democratic%3:00:00::","synset_id":"democratic.a.01"},{"index":2,"sense_key":"democratic%3:01:00::","synset_id":"democratic.a.02"},{"index":3,"sense_key":"democratic%5:00:00:common:01","synset_id":"democratic.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"democratic%3:01:00::","call_ids":["senseval3.d001.s033.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"democratic%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":792,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001704,"input_uncached_usd":0.001584,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0624609999358654}
{"item_id":"senseval3.d001.s033.t007","gold_sense_keys":["willing%3:00:00::"],"candidates":[{"index":1,"sense_key":"willing%3:00:00::","synset_id":"willing.a.01"},{"index":2,"sense_key":"willing%5:00:00:voluntary:01","synset_id":"uncoerced.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"willing%3:00:00::","call_ids":["senseval3.d001.s033.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"willing%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":707,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001534,"input_uncached_usd":0.001414,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1823624579701573}
{"item_id":"senseval3.d001.s033.t008","gold_sense_keys":["engage%2:41:06::"],"candidates":[{"index":1,"sense_key":"engage%2:41:06::","synset_id":"prosecute.v.03"},{"index":2,"sense_key":"engage%2:31:00::","synset_id":"absorb.v.09"},{"index":3,"sense_key":"engage%2:41:01::","synset_id":"hire.v.01"},{"index":4,"sense_key":"engage%2:41:00::","synset_id":"engage.v.04"},{"index":5,"sense_key":"engage%2:32:00::","synset_id":"betroth.v.01"},{"index":6,"sense_key":"engage%2:30:00::","synset_id":"engage.v.06"},{"index":7,"sense_key":"engage%2:41:12::","synset_id":"engage.v.07"},{"index":8,"sense_key":"engage%2:40:01::","synset_id":"engage.v.08"},{"index":9,"sense_key":"engage%2:40:00::","synset_id":"lease.v.04"},{"index":10,"sense_key":"engage%2:35:00::","synset_id":"engage.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"engage%2:41:06::","call_ids":["senseval3.d001.s033.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"engage%2:41:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1182,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0024839999999999997,"input_uncached_usd":0.0023639999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.477326333988458}
{"item_id":"senseval3.d001.s033.t009","gold_sense_keys":["local%3:00:01::"],"candidates":[{"index":1,"sense_key":"local%3:00:01::","synset_id":"local.a.01"},{"index":2,"sense_key":"local%3:01:01::","synset_id":"local.a.02"},{"index":3,"sense_key":"local%3:00:02::","synset_id":"local.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"local%3:01:01::","call_ids":["senseval3.d001.s033.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"local%3:01:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":757,"cached_input_tokens":0,"output_tokens":107,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002584,"input_uncached_usd":0.001514,"input_cached_usd":0.0,"output_usd":0.00107,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5042080839630216}
{"item_id":"senseval3.d001.s033.t011","gold_sense_keys":["republican%3:01:01::"],"candidates":[{"index":1,"sense_key":"republican%3:01:01::","synset_id":"republican.a.01"},{"index":2,"sense_key":"republican%5:00:00:democratic:00","synset_id":"republican.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"republican%3:01:01::","call_ids":["senseval3.d001.s033.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"republican%3:01:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":782,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001684,"input_uncached_usd":0.0015639999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1665878749918193}
{"item_id":"senseval3.d001.s033.t012","gold_sense_keys":["colleague%1:18:01::"],"candidates":[{"index":1,"sense_key":"colleague%1:18:00::","synset_id":"colleague.n.01"},{"index":2,"sense_key":"colleague%1:18:01::","synset_id":"colleague.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"colleague%1:18:01::","call_ids":["senseval3.d001.s033.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"colleague%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":730,"cached_input_tokens":0,"output_tokens":137,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00283,"input_uncached_usd":0.00146,"input_cached_usd":0.0,"output_usd":0.0013700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8079347499879077}
{"item_id":"senseval3.d001.s034.t000","gold_sense_keys":["first%4:02:00::"],"candidates":[{"index":1,"sense_key":"first%4:02:00::","synset_id":"first.r.01"},{"index":2,"sense_key":"first%4:02:02::","synset_id":"first.r.02"},{"index":3,"sense_key":"first%4:02:03::","synset_id":"first.r.03"},{"index":4,"sense_key":"first%4:02:01::","synset_id":"foremost.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"first%4:02:00::","call_ids":["senseval3.d001.s034.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"first%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":820,"cached_input_tokens":0,"output_tokens":90,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00254,"input_uncached_usd":0.00164,"input_cached_usd":0.0,"output_usd":0.0009000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9799442089861259}
{"item_id":"senseval3.d001.s034.t002","gold_sense_keys":["find%2:31:09::","find%2:32:00::","find%2:39:02::"],"candidates":[{"index":1,"sense_key":"find%2:40:02::","synset_id":"find.v.01"},{"index":2,"sense_key":"find%2:39:02::","synset_id":"detect.v.01"},{"index":3,"sense_key":"find%2:40:00::","synset_id":"find.v.03"},{"index":4,"sense_key":"find%2:32:00::","synset_id":"determine.v.01"},{"index":5,"sense_key":"find%2:31:10::","synset_id":"find.v.05"},{"index":6,"sense_key":"find%2:39:01::","synset_id":"witness.v.02"},{"index":7,"sense_key":"find%2:40:01::","synset_id":"line_up.v.02"},{"index":8,"sense_key":"find%2:36:00::","synset_id":"discover.v.03"},{"index":9,"sense_key":"find%2:31:09::","synset_id":"discover.v.04"},{"index":10,"sense_key":"find%2:40:03::","synset_id":"find.v.10"},{"index":11,"sense_key":"find%2:32:01::","synset_id":"rule.v.04"},{"index":12,"sense_key":"find%2:30:00::","synset_id":"receive.v.02"},{"index":13,"sense_key":"find%2:39:05::","synset_id":"find.v.13"},{"index":14,"sense_key":"find%2:40:15::","synset_id":"recover.v.01"},{"index":15,"sense_key":"find%2:38:10::","synset_id":"find.v.15"},{"index":16,"sense_key":"find%2:30:13::","synset_id":"find_oneself.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"find%2:32:00::","call_ids":["senseval3.d001.s034.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"find%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1767,"cached_input_tokens":0,"output_tokens":127,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004804,"input_uncached_usd":0.003534,"input_cached_usd":0.0,"output_usd":0.00127,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.3766564169200137}
{"item_id":"senseval3.d001.s034.t006","gold_sense_keys":["personal%3:00:00::","personal%5:00:00:subjective:00"],"candidates":[{"index":1,"sense_key":"personal%3:00:00::","synset_id":"personal.a.01"},{"index":2,"sense_key":"personal%5:00:00:subjective:00","synset_id":"personal.s.01"},{"index":3,"sense_key":"personal%3:01:01::","synset_id":"personal.a.03"},{"index":4,"sense_key":"personal%5:00:00:physical:00","synset_id":"personal.s.02"},{"index":5,"sense_key":"personal%3:01:00::","synset_id":"personal.a.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"personal%5:00:00:subjective:00","call_ids":["senseval3.d001.s034.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"personal%5:00:00:subjective:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":838,"cached_input_tokens":0,"output_tokens":124,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002916,"input_uncached_usd":0.001676,"input_cached_usd":0.0,"output_usd":0.00124,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.057975332951173}
{"item_id":"senseval3.d001.s034.t007","gold_sense_keys":["staff%1:14:01::"],"candidates":[{"index":1,"sense_key":"staff%1:14:01::","synset_id":"staff.n.01"},{"index":2,"sense_key":"staff%1:06:00::","synset_id":"staff.n.02"},{"index":3,"sense_key":"staff%1:14:00::","synset_id":"staff.n.03"},{"index":4,"sense_key":"staff%1:27:00::","synset_id":"staff.n.04"},{"index":5,"sense_key":"staff%1:10:01::","synset_id":"staff.n.05"},{"index":6,"sense_key":"staff%1:10:00::","synset_id":"staff.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"staff%1:14:01::","call_ids":["senseval3.d001.s034.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"staff%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":966,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002052,"input_uncached_usd":0.001932,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.851574374944903}
{"item_id":"senseval3.d001.s034.t008","gold_sense_keys":["budget%1:21:02::"],"candidates":[{"index":1,"sense_key":"budget%1:21:02::","synset_id":"budget.n.01"},{"index":2,"sense_key":"budget%1:21:03::","synset_id":"budget.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"budget%1:21:02::","call_ids":["senseval3.d001.s034.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"budget%1:21:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":737,"cached_input_tokens":0,"output_tokens":52,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001994,"input_uncached_usd":0.0014739999999999998,"input_cached_usd":0.0,"output_usd":0.0005200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5185102500254288}
{"item_id":"senseval3.d001.s034.t009","gold_sense_keys":["democrat%1:18:01::"],"candidates":[{"index":1,"sense_key":"democrat%1:18:01::","synset_id":"democrat.n.01"},{"index":2,"sense_key":"democrat%1:18:00::","synset_id":"democrat.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"democrat%1:18:01::","call_ids":["senseval3.d001.s034.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"democrat%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":695,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00151,"input_uncached_usd":0.00139,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1683668749174103}
{"item_id":"senseval3.d001.s035.t000","gold_sense_keys":["extent%1:26:00::"],"candidates":[{"index":1,"sense_key":"extent%1:26:00::","synset_id":"extent.n.01"},{"index":2,"sense_key":"extent%1:07:00::","synset_id":"extent.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"extent%1:26:00::","call_ids":["senseval3.d001.s035.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"extent%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":728,"cached_input_tokens":0,"output_tokens":37,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001826,"input_uncached_usd":0.0014559999999999998,"input_cached_usd":0.0,"output_usd":0.00037000000000000005,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3044195000547916}
{"item_id":"senseval3.d001.s035.t001","gold_sense_keys":["primary%3:00:00::","primary%5:00:00:important:00"],"candidates":[{"index":1,"sense_key":"primary%3:00:00::","synset_id":"primary.a.01"},{"index":2,"sense_key":"primary%5:00:00:underived:00","synset_id":"primary.s.01"},{"index":3,"sense_key":"primary%5:00:00:important:00","synset_id":"chief.s.01"},{"index":4,"sense_key":"primary%5:00:00:basic:00","synset_id":"elementary.s.02"},{"index":5,"sense_key":"primary%5:00:00:essential:00","synset_id":"basal.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"primary%5:00:00:important:00","call_ids":["senseval3.d001.s035.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"primary%5:00:00:important:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":895,"cached_input_tokens":0,"output_tokens":100,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00279,"input_uncached_usd":0.00179,"input_cached_usd":0.0,"output_usd":0.001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7469788329908624}
{"item_id":"senseval3.d001.s035.t002","gold_sense_keys":["duty%1:04:02::"],"candidates":[{"index":1,"sense_key":"duty%1:04:00::","synset_id":"duty.n.01"},{"index":2,"sense_key":"duty%1:04:02::","synset_id":"duty.n.02"},{"index":3,"sense_key":"duty%1:21:00::","synset_id":"duty.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"duty%1:04:02::","call_ids":["senseval3.d001.s035.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"duty%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":850,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00233,"input_uncached_usd":0.0017,"input_cached_usd":0.0,"output_usd":0.00063,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.319537500035949}
{"item_id":"senseval3.d001.s035.t003","gold_sense_keys":["personal%3:00:00::","personal%5:00:00:subjective:00"],"candidates":[{"index":1,"sense_key":"personal%3:00:00::","synset_id":"personal.a.01"},{"index":2,"sense_key":"personal%5:00:00:subjective:00","synset_id":"personal.s.01"},{"index":3,"sense_key":"personal%3:01:01::","synset_id":"personal.a.03"},{"index":4,"sense_key":"personal%5:00:00:physical:00","synset_id":"personal.s.02"},{"index":5,"sense_key":"personal%3:01:00::","synset_id":"personal.a.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"personal%3:00:00::","call_ids":["senseval3.d001.s035.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"personal%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":838,"cached_input_tokens":0,"output_tokens":164,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0033160000000000004,"input_uncached_usd":0.001676,"input_cached_usd":0.0,"output_usd":0.0016400000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.4612370419781655}
{"item_id":"senseval3.d001.s035.t004","gold_sense_keys":["staff%1:14:01::"],"candidates":[{"index":1,"sense_key":"staff%1:14:01::","synset_id":"staff.n.01"},{"index":2,"sense_key":"staff%1:06:00::","synset_id":"staff.n.02"},{"index":3,"sense_key":"staff%1:14:00::","synset_id":"staff.n.03"},{"index":4,"sense_key":"staff%1:27:00::","synset_id":"staff.n.04"},{"index":5,"sense_key":"staff%1:10:01::","synset_id":"staff.n.05"},{"index":6,"sense_key":"staff%1:10:00::","synset_id":"staff.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"staff%1:14:01::","call_ids":["senseval3.d001.s035.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"staff%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":966,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002052,"input_uncached_usd":0.001932,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2608546670526266}
{"item_id":"senseval3.d001.s035.t005","gold_sense_keys":["involve%2:42:00::"],"candidates":[{"index":1,"sense_key":"involve%2:42:01::","synset_id":"involve.v.01"},{"index":2,"sense_key":"involve%2:42:04::","synset_id":"involve.v.02"},{"index":3,"sense_key":"involve%2:42:00::","synset_id":"imply.v.05"},{"index":4,"sense_key":"involve%2:42:07::","synset_id":"necessitate.v.01"},{"index":5,"sense_key":"involve%2:42:02::","synset_id":"involve.v.05"},{"index":6,"sense_key":"involve%2:31:00::","synset_id":"involve.v.06"},{"index":7,"sense_key":"involve%2:30:00::","synset_id":"involve.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"involve%2:42:00::","call_ids":["senseval3.d001.s035.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"involve%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":994,"cached_input_tokens":0,"output_tokens":175,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003738,"input_uncached_usd":0.0019879999999999997,"input_cached_usd":0.0,"output_usd":0.00175,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.211834750021808}
{"item_id":"senseval3.d001.s035.t006","gold_sense_keys":["local%3:00:01::"],"candidates":[{"index":1,"sense_key":"local%3:00:01::","synset_id":"local.a.01"},{"index":2,"sense_key":"local%3:01:01::","synset_id":"local.a.02"},{"index":3,"sense_key":"local%3:00:02::","synset_id":"local.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"local%3:01:01::","call_ids":["senseval3.d001.s035.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"local%3:01:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":780,"cached_input_tokens":0,"output_tokens":117,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00273,"input_uncached_usd":0.00156,"input_cached_usd":0.0,"output_usd":0.00117,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9054802079917863}
{"item_id":"senseval3.d001.s035.t008","gold_sense_keys":["indicate%2:32:01::","indicate%2:32:02::"],"candidates":[{"index":1,"sense_key":"indicate%2:32:01::","synset_id":"bespeak.v.01"},{"index":2,"sense_key":"indicate%2:32:03::","synset_id":"indicate.v.02"},{"index":3,"sense_key":"indicate%2:32:00::","synset_id":"indicate.v.03"},{"index":4,"sense_key":"indicate%2:32:02::","synset_id":"argue.v.03"},{"index":5,"sense_key":"indicate%2:32:04::","synset_id":"indicate.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"indicate%2:32:02::","call_ids":["senseval3.d001.s035.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"indicate%2:32:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":963,"cached_input_tokens":0,"output_tokens":209,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0040160000000000005,"input_uncached_usd":0.001926,"input_cached_usd":0.0,"output_usd":0.0020900000000000003,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.602848832961172}
{"item_id":"senseval3.d001.s035.t009","gold_sense_keys":["political%3:01:01::"],"candidates":[{"index":1,"sense_key":"political%3:00:00::","synset_id":"political.a.01"},{"index":2,"sense_key":"political%3:01:01::","synset_id":"political.a.02"},{"index":3,"sense_key":"political%3:01:02::","synset_id":"political.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"political%3:01:01::","call_ids":["senseval3.d001.s035.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"political%3:01:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":771,"cached_input_tokens":0,"output_tokens":52,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002062,"input_uncached_usd":0.001542,"input_cached_usd":0.0,"output_usd":0.0005200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8219068750040606}
{"item_id":"senseval3.d001.s035.t010","gold_sense_keys":["philosophy%1:09:01::"],"candidates":[{"index":1,"sense_key":"philosophy%1:09:01::","synset_id":"doctrine.n.01"},{"index":2,"sense_key":"philosophy%1:09:00::","synset_id":"philosophy.n.02"},{"index":3,"sense_key":"philosophy%1:09:02::","synset_id":"philosophy.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"philosophy%1:09:01::","call_ids":["senseval3.d001.s035.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"philosophy%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":820,"cached_input_tokens":0,"output_tokens":108,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00272,"input_uncached_usd":0.00164,"input_cached_usd":0.0,"output_usd":0.00108,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0673787919804454}
{"item_id":"senseval3.d001.s035.t011","gold_sense_keys":["lead%2:32:00::"],"candidates":[{"index":1,"sense_key":"lead%2:38:01::","synset_id":"lead.v.01"},{"index":2,"sense_key":"lead%2:42:04::","synset_id":"leave.v.07"},{"index":3,"sense_key":"lead%2:42:12::","synset_id":"lead.v.03"},{"index":4,"sense_key":"lead%2:38:00::","synset_id":"lead.v.04"},{"index":5,"sense_key":"lead%2:32:00::","synset_id":"lead.v.05"},{"index":6,"sense_key":"lead%2:42:00::","synset_id":"run.v.03"},{"index":7,"sense_key":"lead%2:41:00::","synset_id":"head.v.02"},{"index":8,"sense_key":"lead%2:42:01::","synset_id":"lead.v.08"},{"index":9,"sense_key":"lead%2:41:12::","synset_id":"contribute.v.03"},{"index":10,"sense_key":"lead%2:36:00::","synset_id":"conduct.v.02"},{"index":11,"sense_key":"lead%2:42:03::","synset_id":"go.v.25"},{"index":12,"sense_key":"lead%2:38:02::","synset_id":"precede.v.04"},{"index":13,"sense_key":"lead%2:42:02::","synset_id":"run.v.23"},{"index":14,"sense_key":"lead%2:32:01::","synset_id":"moderate.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"lead%2:32:00::","call_ids":["senseval3.d001.s035.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"lead%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1443,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0036559999999999995,"input_uncached_usd":0.0028859999999999997,"input_cached_usd":0.0,"output_usd":0.0007700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5774639169685543}
{"item_id":"senseval3.d001.s035.t013","gold_sense_keys":["republican%1:18:01::"],"candidates":[{"index":1,"sense_key":"republican%1:18:01::","synset_id":"republican.n.01"},{"index":2,"sense_key":"republican%1:18:00::","synset_id":"republican.n.02"},{"index":3,"sense_key":"republican%1:17:00::","synset_id":"republican.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"republican%1:18:01::","call_ids":["senseval3.d001.s035.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"republican%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":763,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0016460000000000001,"input_uncached_usd":0.001526,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5833067919593304}
{"item_id":"senseval3.d001.s035.t014","gold_sense_keys":["narrow%5:00:00:limited:00"],"candidates":[{"index":1,"sense_key":"narrow%3:00:00::","synset_id":"narrow.a.01"},{"index":2,"sense_key":"narrow%5:00:00:limited:00","synset_id":"narrow.s.01"},{"index":3,"sense_key":"narrow%3:00:06::","synset_id":"narrow-minded.a.02"},{"index":4,"sense_key":"narrow%3:00:02::","synset_id":"narrow.a.04"},{"index":5,"sense_key":"narrow%5:00:00:careful:00","synset_id":"minute.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"narrow%5:00:00:limited:00","call_ids":["senseval3.d001.s035.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"narrow%5:00:00:limited:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":914,"cached_input_tokens":0,"output_tokens":108,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002908,"input_uncached_usd":0.001828,"input_cached_usd":0.0,"output_usd":0.00108,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8406133750686422}
{"item_id":"senseval3.d001.s035.t015","gold_sense_keys":["constituent%1:18:00::"],"candidates":[{"index":1,"sense_key":"constituent%1:06:00::","synset_id":"component.n.03"},{"index":2,"sense_key":"constituent%1:18:00::","synset_id":"constituent.n.02"},{"index":3,"sense_key":"constituent%1:24:00::","synset_id":"part.n.01"},{"index":4,"sense_key":"constituent%1:10:00::","synset_id":"constituent.n.04"},{"index":5,"sense_key":"constituent%1:09:00::","synset_id":"component.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"constituent%1:18:00::","call_ids":["senseval3.d001.s035.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"constituent%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1010,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0026,"input_uncached_usd":0.00202,"input_cached_usd":0.0,"output_usd":0.00058,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4129700829507783}
{"item_id":"senseval3.d001.s035.t016","gold_sense_keys":["concern%1:09:00::"],"candidates":[{"index":1,"sense_key":"concern%1:09:00::","synset_id":"concern.n.01"},{"index":2,"sense_key":"concern%1:12:01::","synset_id":"concern.n.02"},{"index":3,"sense_key":"concern%1:12:00::","synset_id":"concern.n.03"},{"index":4,"sense_key":"concern%1:09:01::","synset_id":"concern.n.04"},{"index":5,"sense_key":"concern%1:14:00::","synset_id":"business.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"concern%1:09:00::","call_ids":["senseval3.d001.s035.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"concern%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":954,"cached_input_tokens":0,"output_tokens":69,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002598,"input_uncached_usd":0.001908,"input_cached_usd":0.0,"output_usd":0.0006900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8706033329945058}
{"item_id":"senseval3.d001.s036.t002","gold_sense_keys":["assumption%1:09:00::","assumption%1:10:00::"],"candidates":[{"index":1,"sense_key":"assumption%1:10:00::","synset_id":"premise.n.01"},{"index":2,"sense_key":"assumption%1:09:00::","synset_id":"assumption.n.02"},{"index":3,"sense_key":"assumption%1:04:00::","synset_id":"assumption.n.03"},{"index":4,"sense_key":"assumption%1:28:00::","synset_id":"assumption.n.04"},{"index":5,"sense_key":"assumption%1:11:00::","synset_id":"assumption.n.05"},{"index":6,"sense_key":"assumption%1:07:00::","synset_id":"presumption.n.03"},{"index":7,"sense_key":"assumption%1:04:01::","synset_id":"assumption.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"assumption%1:09:00::","call_ids":["senseval3.d001.s036.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"assumption%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1133,"cached_input_tokens":0,"output_tokens":168,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003946,"input_uncached_usd":0.002266,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2807307499460876}
{"item_id":"senseval3.d001.s036.t003","gold_sense_keys":["valid%3:00:00::"],"candidates":[{"index":1,"sense_key":"valid%3:00:00::","synset_id":"valid.a.01"},{"index":2,"sense_key":"valid%5:00:00:unexpired:00","synset_id":"valid.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"valid%3:00:00::","call_ids":["senseval3.d001.s036.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"valid%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":701,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0015220000000000001,"input_uncached_usd":0.001402,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.46789812494535}
{"item_id":"senseval3.d001.s036.t004","gold_sense_keys":["democrat%1:18:01::"],"candidates":[{"index":1,"sense_key":"democrat%1:18:01::","synset_id":"democrat.n.01"},{"index":2,"sense_key":"democrat%1:18:00::","synset_id":"democrat.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"democrat%1:18:01::","call_ids":["senseval3.d001.s036.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"democrat%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":681,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001482,"input_uncached_usd":0.001362,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.376088667078875}
{"item_id":"senseval3.d001.s036.t005","gold_sense_keys":["have%2:30:01::","have%2:40:00::","have%2:42:00::","have%2:42:12::"],"candidates":[{"index":1,"sense_key":"have%2:40:00::","synset_id":"have.v.01"},{"index":2,"sense_key":"have%2:42:00::","synset_id":"have.v.02"},{"index":3,"sense_key":"have%2:39:06::","synset_id":"experience.v.03"},{"index":4,"sense_key":"have%2:40:04::","synset_id":"own.v.01"},{"index":5,"sense_key":"have%2:30:00::","synset_id":"get.v.03"},{"index":6,"sense_key":"have%2:34:00::","synset_id":"consume.v.02"},{"index":7,"sense_key":"have%2:41:00::","synset_id":"have.v.07"},{"index":8,"sense_key":"have%2:36:00::","synset_id":"hold.v.03"},{"index":9,"sense_key":"have%2:40:01::","synset_id":"have.v.09"},{"index":10,"sense_key":"have%2:42:12::","synset_id":"have.v.10"},{"index":11,"sense_key":"have%2:30:01::","synset_id":"have.v.11"},{"index":12,"sense_key":"have%2:29:02::","synset_id":"have.v.12"},{"index":13,"sense_key":"have%2:32:00::","synset_id":"induce.v.02"},{"index":14,"sense_key":"have%2:40:05::","synset_id":"accept.v.02"},{"index":15,"sense_key":"have%2:40:03::","synset_id":"receive.v.01"},{"index":16,"sense_key":"have%2:29:05::","synset_id":"suffer.v.02"},{"index":17,"sense_key":"have%2:40:02::","synset_id":"have.v.17"},{"index":18,"sense_key":"have%2:29:00::","synset_id":"give_birth.v.01"},{"index":19,"sense_key":"have%2:35:00::","synset_id":"take.v.35"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"have%2:40:00::","call_ids":["senseval3.d001.s036.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"have%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1771,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004222,"input_uncached_usd":0.003542,"input_cached_usd":0.0,"output_usd":0.00068,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5426015830598772}
{"item_id":"senseval3.d001.s036.t006","gold_sense_keys":["attendance%1:04:00::"],"candidates":[{"index":1,"sense_key":"attendance%1:04:00::","synset_id":"attendance.n.01"},{"index":2,"sense_key":"attendance%1:28:00::","synset_id":"attendance.n.02"},{"index":3,"sense_key":"attendance%1:14:00::","synset_id":"attendance.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"attendance%1:28:00::","call_ids":["senseval3.d001.s036.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"attendance%1:28:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":752,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002064,"input_uncached_usd":0.001504,"input_cached_usd":0.0,"output_usd":0.0005600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":4.681814500014298}
{"item_id":"senseval3.d001.s036.t007","gold_sense_keys":["rate%1:24:00::"],"candidates":[{"index":1,"sense_key":"rate%1:28:00::","synset_id":"rate.n.01"},{"index":2,"sense_key":"rate%1:21:00::","synset_id":"rate.n.02"},{"index":3,"sense_key":"rate%1:07:00::","synset_id":"pace.n.03"},{"index":4,"sense_key":"rate%1:24:00::","synset_id":"rate.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"rate%1:24:00::","call_ids":["senseval3.d001.s036.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"rate%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":872,"cached_input_tokens":0,"output_tokens":105,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002794,"input_uncached_usd":0.0017439999999999999,"input_cached_usd":0.0,"output_usd":0.0010500000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5926424589706585}
{"item_id":"senseval3.d001.s036.t009","gold_sense_keys":["vote%1:04:00::"],"candidates":[{"index":1,"sense_key":"vote%1:04:00::","synset_id":"vote.n.01"},{"index":2,"sense_key":"vote%1:04:01::","synset_id":"vote.n.02"},{"index":3,"sense_key":"vote%1:07:00::","synset_id":"right_to_vote.n.01"},{"index":4,"sense_key":"vote%1:14:00::","synset_id":"vote.n.04"},{"index":5,"sense_key":"vote%1:09:00::","synset_id":"vote.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"vote%1:04:00::","call_ids":["senseval3.d001.s036.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"vote%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":933,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0025859999999999998,"input_uncached_usd":0.0018659999999999998,"input_cached_usd":0.0,"output_usd":0.00072,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4548866669647396}
{"item_id":"senseval3.d001.s036.t010","gold_sense_keys":["republican%1:18:01::"],"candidates":[{"index":1,"sense_key":"republican%1:18:01::","synset_id":"republican.n.01"},{"index":2,"sense_key":"republican%1:18:00::","synset_id":"republican.n.02"},{"index":3,"sense_key":"republican%1:17:00::","synset_id":"republican.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"republican%1:18:01::","call_ids":["senseval3.d001.s036.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"republican%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":749,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001618,"input_uncached_usd":0.001498,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9736210419796407}
{"item_id":"senseval3.d001.s036.t011","gold_sense_keys":["extent%1:26:00::"],"candidates":[{"index":1,"sense_key":"extent%1:26:00::","synset_id":"extent.n.01"},{"index":2,"sense_key":"extent%1:07:00::","synset_id":"extent.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"extent%1:26:00::","call_ids":["senseval3.d001.s036.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"extent%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":714,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0020080000000000002,"input_uncached_usd":0.001428,"input_cached_usd":0.0,"output_usd":0.00058,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7980649590026587}
{"item_id":"senseval3.d001.s036.t013","gold_sense_keys":["vote%1:04:00::"],"candidates":[{"index":1,"sense_key":"vote%1:04:00::","synset_id":"vote.n.01"},{"index":2,"sense_key":"vote%1:04:01::","synset_id":"vote.n.02"},{"index":3,"sense_key":"vote%1:07:00::","synset_id":"right_to_vote.n.01"},{"index":4,"sense_key":"vote%1:14:00::","synset_id":"vote.n.04"},{"index":5,"sense_key":"vote%1:09:00::","synset_id":"vote.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"vote%1:04:00::","call_ids":["senseval3.d001.s036.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"vote%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":933,"cached_input_tokens":0,"output_tokens":107,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002936,"input_uncached_usd":0.0018659999999999998,"input_cached_usd":0.0,"output_usd":0.00107,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.691321499994956}
{"item_id":"senseval3.d001.s036.t014","gold_sense_keys":["reflect%2:39:01::"],"candidates":[{"index":1,"sense_key":"reflect%2:39:01::","synset_id":"reflect.v.01"},{"index":2,"sense_key":"reflect%2:31:00::","synset_id":"chew_over.v.01"},{"index":3,"sense_key":"reflect%2:39:00::","synset_id":"reflect.v.03"},{"index":4,"sense_key":"reflect%2:43:00::","synset_id":"reflect.v.04"},{"index":5,"sense_key":"reflect%2:39:02::","synset_id":"reflect.v.05"},{"index":6,"sense_key":"reflect%2:32:01::","synset_id":"reflect.v.06"},{"index":7,"sense_key":"reflect%2:32:00::","synset_id":"reflect.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"reflect%2:39:01::","call_ids":["senseval3.d001.s036.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"reflect%2:39:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1023,"cached_input_tokens":0,"output_tokens":112,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0031660000000000004,"input_uncached_usd":0.002046,"input_cached_usd":0.0,"output_usd":0.0011200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0527924170019105}
{"item_id":"senseval3.d001.s036.t015","gold_sense_keys":["national%3:00:00::","national%3:00:01::","national%3:01:00::","national%5:00:00:domestic:00"],"candidates":[{"index":1,"sense_key":"national%3:01:00::","synset_id":"national.a.01"},{"index":2,"sense_key":"national%3:00:00::","synset_id":"national.a.02"},{"index":3,"sense_key":"national%3:00:01::","synset_id":"national.a.03"},{"index":4,"sense_key":"national%5:00:00:public:00","synset_id":"national.s.01"},{"index":5,"sense_key":"national%5:00:00:domestic:00","synset_id":"home.s.01"},{"index":6,"sense_key":"national%3:01:02::","synset_id":"national.a.06"},{"index":7,"sense_key":"national%3:01:01::","synset_id":"national.a.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"national%3:00:01::","call_ids":["senseval3.d001.s036.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"national%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":965,"cached_input_tokens":0,"output_tokens":83,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00276,"input_uncached_usd":0.0019299999999999999,"input_cached_usd":0.0,"output_usd":0.0008300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5604071249254048}
{"item_id":"senseval3.d001.s036.t016","gold_sense_keys":["policy%1:09:00::"],"candidates":[{"index":1,"sense_key":"policy%1:09:00::","synset_id":"policy.n.01"},{"index":2,"sense_key":"policy%1:10:00::","synset_id":"policy.n.02"},{"index":3,"sense_key":"policy%1:10:02::","synset_id":"policy.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"policy%1:09:00::","call_ids":["senseval3.d001.s036.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"policy%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":791,"cached_input_tokens":0,"output_tokens":110,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002682,"input_uncached_usd":0.0015819999999999999,"input_cached_usd":0.0,"output_usd":0.0011,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.678778124973178}
{"item_id":"senseval3.d001.s036.t017","gold_sense_keys":["making%1:04:00::"],"candidates":[{"index":1,"sense_key":"making%1:04:00::","synset_id":"devising.n.01"},{"index":2,"sense_key":"making%1:07:00::","synset_id":"qualification.n.01"},{"index":3,"sense_key":"making%1:06:00::","synset_id":"making.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"making%1:04:00::","call_ids":["senseval3.d001.s036.t017__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"making%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":810,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00174,"input_uncached_usd":0.00162,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1020947910146788}
{"item_id":"senseval3.d001.s036.t020","gold_sense_keys":["vote%1:04:00::"],"candidates":[{"index":1,"sense_key":"vote%1:04:00::","synset_id":"vote.n.01"},{"index":2,"sense_key":"vote%1:04:01::","synset_id":"vote.n.02"},{"index":3,"sense_key":"vote%1:07:00::","synset_id":"right_to_vote.n.01"},{"index":4,"sense_key":"vote%1:14:00::","synset_id":"vote.n.04"},{"index":5,"sense_key":"vote%1:09:00::","synset_id":"vote.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"vote%1:04:00::","call_ids":["senseval3.d001.s036.t020__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"vote%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":933,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002526,"input_uncached_usd":0.0018659999999999998,"input_cached_usd":0.0,"output_usd":0.0006600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.4317650420125574}
{"item_id":"senseval3.d001.s036.t021","gold_sense_keys":["time%1:28:05::"],"candidates":[{"index":1,"sense_key":"time%1:11:00::","synset_id":"time.n.01"},{"index":2,"sense_key":"time%1:28:05::","synset_id":"time.n.02"},{"index":3,"sense_key":"time%1:28:00::","synset_id":"time.n.03"},{"index":4,"sense_key":"time%1:28:06::","synset_id":"time.n.04"},{"index":5,"sense_key":"time%1:03:00::","synset_id":"time.n.05"},{"index":6,"sense_key":"time%1:11:01::","synset_id":"time.n.06"},{"index":7,"sense_key":"time%1:28:03::","synset_id":"clock_time.n.01"},{"index":8,"sense_key":"time%1:28:04::","synset_id":"fourth_dimension.n.01"},{"index":9,"sense_key":"time%1:07:01::","synset_id":"meter.n.04"},{"index":10,"sense_key":"time%1:28:01::","synset_id":"prison_term.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"time%1:28:05::","call_ids":["senseval3.d001.s036.t021__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"time%1:28:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1173,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002986,"input_uncached_usd":0.002346,"input_cached_usd":0.0,"output_usd":0.00064,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8627595000434667}
{"item_id":"senseval3.d001.s036.t023","gold_sense_keys":["devote%2:40:00::"],"candidates":[{"index":1,"sense_key":"devote%2:32:00::","synset_id":"give.v.18"},{"index":2,"sense_key":"devote%2:31:00::","synset_id":"give.v.10"},{"index":3,"sense_key":"devote%2:40:00::","synset_id":"devote.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"devote%2:31:00::","call_ids":["senseval3.d001.s036.t023__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"devote%2:31:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":786,"cached_input_tokens":0,"output_tokens":255,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004122,"input_uncached_usd":0.0015719999999999998,"input_cached_usd":0.0,"output_usd":0.00255,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.876308750011958}
{"item_id":"senseval3.d001.s036.t024","gold_sense_keys":["local%3:00:01::"],"candidates":[{"index":1,"sense_key":"local%3:00:01::","synset_id":"local.a.01"},{"index":2,"sense_key":"local%3:01:01::","synset_id":"local.a.02"},{"index":3,"sense_key":"local%3:00:02::","synset_id":"local.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"local%3:00:01::","call_ids":["senseval3.d001.s036.t024__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"local%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":766,"cached_input_tokens":0,"output_tokens":154,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003072,"input_uncached_usd":0.001532,"input_cached_usd":0.0,"output_usd":0.0015400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.170304915984161}
{"item_id":"senseval3.d001.s037.t001","gold_sense_keys":["indicate%2:32:01::","indicate%2:32:02::"],"candidates":[{"index":1,"sense_key":"indicate%2:32:01::","synset_id":"bespeak.v.01"},{"index":2,"sense_key":"indicate%2:32:03::","synset_id":"indicate.v.02"},{"index":3,"sense_key":"indicate%2:32:00::","synset_id":"indicate.v.03"},{"index":4,"sense_key":"indicate%2:32:02::","synset_id":"argue.v.03"},{"index":5,"sense_key":"indicate%2:32:04::","synset_id":"indicate.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"indicate%2:32:02::","call_ids":["senseval3.d001.s037.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"indicate%2:32:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":903,"cached_input_tokens":0,"output_tokens":128,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003086,"input_uncached_usd":0.0018059999999999999,"input_cached_usd":0.0,"output_usd":0.00128,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.938540208968334}
{"item_id":"senseval3.d001.s037.t002","gold_sense_keys":["case%1:26:00::"],"candidates":[{"index":1,"sense_key":"case%1:11:00::","synset_id":"case.n.01"},{"index":2,"sense_key":"case%1:26:00::","synset_id":"event.n.02"},{"index":3,"sense_key":"case%1:04:00::","synset_id":"lawsuit.n.01"},{"index":4,"sense_key":"case%1:09:00::","synset_id":"case.n.04"},{"index":5,"sense_key":"case%1:06:00::","synset_id":"case.n.05"},{"index":6,"sense_key":"case%1:18:03::","synset_id":"case.n.06"},{"index":7,"sense_key":"case%1:18:02::","synset_id":"subject.n.06"},{"index":8,"sense_key":"case%1:10:02::","synset_id":"case.n.08"},{"index":9,"sense_key":"case%1:10:01::","synset_id":"case.n.09"},{"index":10,"sense_key":"case%1:23:00::","synset_id":"case.n.10"},{"index":11,"sense_key":"case%1:10:00::","synset_id":"case.n.11"},{"index":12,"sense_key":"case%1:26:02::","synset_id":"case.n.12"},{"index":13,"sense_key":"case%1:18:04::","synset_id":"character.n.05"},{"index":14,"sense_key":"case%1:10:03::","synset_id":"font.n.01"},{"index":15,"sense_key":"case%1:08:00::","synset_id":"sheath.n.02"},{"index":16,"sense_key":"case%1:06:06::","synset_id":"shell.n.08"},{"index":17,"sense_key":"case%1:06:07::","synset_id":"casing.n.03"},{"index":18,"sense_key":"case%1:06:03::","synset_id":"case.n.18"},{"index":19,"sense_key":"case%1:06:02::","synset_id":"case.n.19"},{"index":20,"sense_key":"case%1:06:01::","synset_id":"case.n.20"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"case%1:11:00::","call_ids":["senseval3.d001.s037.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"case%1:11:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1810,"cached_input_tokens":0,"output_tokens":129,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00491,"input_uncached_usd":0.00362,"input_cached_usd":0.0,"output_usd":0.0012900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9001676250481978}
{"item_id":"senseval3.d001.s038.t000","gold_sense_keys":["democratic%3:01:00::"],"candidates":[{"index":1,"sense_key":"democratic%3:00:00::","synset_id":"democratic.a.01"},{"index":2,"sense_key":"democratic%3:01:00::","synset_id":"democratic.a.02"},{"index":3,"sense_key":"democratic%5:00:00:common:01","synset_id":"democratic.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"democratic%3:01:00::","call_ids":["senseval3.d001.s038.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"democratic%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":812,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001744,"input_uncached_usd":0.001624,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1554317919071764}
{"item_id":"senseval3.d001.s038.t001","gold_sense_keys":["attendance%1:04:00::"],"candidates":[{"index":1,"sense_key":"attendance%1:04:00::","synset_id":"attendance.n.01"},{"index":2,"sense_key":"attendance%1:28:00::","synset_id":"attendance.n.02"},{"index":3,"sense_key":"attendance%1:14:00::","synset_id":"attendance.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"attendance%1:28:00::","call_ids":["senseval3.d001.s038.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"attendance%1:28:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":763,"cached_input_tokens":0,"output_tokens":52,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002046,"input_uncached_usd":0.001526,"input_cached_usd":0.0,"output_usd":0.0005200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.054262459045276}
{"item_id":"senseval3.d001.s038.t002","gold_sense_keys":["rate%1:24:00::"],"candidates":[{"index":1,"sense_key":"rate%1:28:00::","synset_id":"rate.n.01"},{"index":2,"sense_key":"rate%1:21:00::","synset_id":"rate.n.02"},{"index":3,"sense_key":"rate%1:07:00::","synset_id":"pace.n.03"},{"index":4,"sense_key":"rate%1:24:00::","synset_id":"rate.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"rate%1:24:00::","call_ids":["senseval3.d001.s038.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"rate%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":883,"cached_input_tokens":0,"output_tokens":78,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002546,"input_uncached_usd":0.001766,"input_cached_usd":0.0,"output_usd":0.0007800000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3589958749944344}
{"item_id":"senseval3.d001.s038.t004","gold_sense_keys":["republican%3:01:01::"],"candidates":[{"index":1,"sense_key":"republican%3:01:01::","synset_id":"republican.a.01"},{"index":2,"sense_key":"republican%5:00:00:democratic:00","synset_id":"republican.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"republican%3:01:01::","call_ids":["senseval3.d001.s038.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"republican%3:01:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":802,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001724,"input_uncached_usd":0.001604,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1560104589443654}
{"item_id":"senseval3.d001.s038.t005","gold_sense_keys":["attendance%1:04:00::"],"candidates":[{"index":1,"sense_key":"attendance%1:04:00::","synset_id":"attendance.n.01"},{"index":2,"sense_key":"attendance%1:28:00::","synset_id":"attendance.n.02"},{"index":3,"sense_key":"attendance%1:14:00::","synset_id":"attendance.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"attendance%1:28:00::","call_ids":["senseval3.d001.s038.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"attendance%1:28:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":763,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002156,"input_uncached_usd":0.001526,"input_cached_usd":0.0,"output_usd":0.00063,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.857806415995583}
{"item_id":"senseval3.d001.s038.t006","gold_sense_keys":["rate%1:24:00::"],"candidates":[{"index":1,"sense_key":"rate%1:28:00::","synset_id":"rate.n.01"},{"index":2,"sense_key":"rate%1:21:00::","synset_id":"rate.n.02"},{"index":3,"sense_key":"rate%1:07:00::","synset_id":"pace.n.03"},{"index":4,"sense_key":"rate%1:24:00::","synset_id":"rate.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"rate%1:24:00::","call_ids":["senseval3.d001.s038.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"rate%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":883,"cached_input_tokens":0,"output_tokens":102,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0027860000000000003,"input_uncached_usd":0.001766,"input_cached_usd":0.0,"output_usd":0.00102,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1512898330111057}
{"item_id":"senseval3.d001.s039.t000","gold_sense_keys":["show%2:32:00::","show%2:32:03::","show%2:36:00::"],"candidates":[{"index":1,"sense_key":"show%2:39:02::","synset_id":"show.v.01"},{"index":2,"sense_key":"show%2:31:00::","synset_id":"prove.v.02"},{"index":3,"sense_key":"show%2:32:00::","synset_id":"testify.v.02"},{"index":4,"sense_key":"show%2:39:00::","synset_id":"show.v.04"},{"index":5,"sense_key":"show%2:36:00::","synset_id":"picture.v.02"},{"index":6,"sense_key":"show%2:32:02::","synset_id":"express.v.01"},{"index":7,"sense_key":"show%2:32:01::","synset_id":"indicate.v.02"},{"index":8,"sense_key":"show%2:39:01::","synset_id":"show.v.08"},{"index":9,"sense_key":"show%2:32:04::","synset_id":"read.v.08"},{"index":10,"sense_key":"show%2:32:03::","synset_id":"show.v.10"},{"index":11,"sense_key":"show%2:38:00::","synset_id":"usher.v.01"},{"index":12,"sense_key":"show%2:33:00::","synset_id":"show.v.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"show%2:32:03::","call_ids":["senseval3.d001.s039.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"show%2:32:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1490,"cached_input_tokens":0,"output_tokens":181,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00479,"input_uncached_usd":0.00298,"input_cached_usd":0.0,"output_usd":0.0018100000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.3749905000440776}
{"item_id":"senseval3.d001.s039.t001","gold_sense_keys":["table%1:14:00::"],"candidates":[{"index":1,"sense_key":"table%1:14:00::","synset_id":"table.n.01"},{"index":2,"sense_key":"table%1:06:01::","synset_id":"table.n.02"},{"index":3,"sense_key":"table%1:06:02::","synset_id":"table.n.03"},{"index":4,"sense_key":"table%1:17:00::","synset_id":"mesa.n.01"},{"index":5,"sense_key":"table%1:14:01::","synset_id":"table.n.05"},{"index":6,"sense_key":"table%1:13:01::","synset_id":"board.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"table%1:14:00::","call_ids":["senseval3.d001.s039.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"table%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1026,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002172,"input_uncached_usd":0.002052,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5449574590893462}
{"item_id":"senseval3.d001.s039.t002","gold_sense_keys":["democrat%1:18:01::"],"candidates":[{"index":1,"sense_key":"democrat%1:18:01::","synset_id":"democrat.n.01"},{"index":2,"sense_key":"democrat%1:18:00::","synset_id":"democrat.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"democrat%1:18:01::","call_ids":["senseval3.d001.s039.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"democrat%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":754,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001628,"input_uncached_usd":0.001508,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8326073749922216}
{"item_id":"senseval3.d001.s039.t004","gold_sense_keys":["proportion%1:24:01::"],"candidates":[{"index":1,"sense_key":"proportion%1:24:00::","synset_id":"proportion.n.01"},{"index":2,"sense_key":"proportion%1:07:00::","synset_id":"proportion.n.02"},{"index":3,"sense_key":"proportion%1:25:00::","synset_id":"symmetry.n.02"},{"index":4,"sense_key":"proportion%1:24:01::","synset_id":"proportion.n.04"},{"index":5,"sense_key":"proportion%1:07:01::","synset_id":"proportion.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"proportion%1:24:00::","call_ids":["senseval3.d001.s039.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"proportion%1:24:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1031,"cached_input_tokens":0,"output_tokens":174,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0038020000000000003,"input_uncached_usd":0.002062,"input_cached_usd":0.0,"output_usd":0.0017400000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0320548329036683}
{"item_id":"senseval3.d001.s039.t005","gold_sense_keys":["personal%3:00:00::","personal%5:00:00:subjective:00"],"candidates":[{"index":1,"sense_key":"personal%3:00:00::","synset_id":"personal.a.01"},{"index":2,"sense_key":"personal%5:00:00:subjective:00","synset_id":"personal.s.01"},{"index":3,"sense_key":"personal%3:01:01::","synset_id":"personal.a.03"},{"index":4,"sense_key":"personal%5:00:00:physical:00","synset_id":"personal.s.02"},{"index":5,"sense_key":"personal%3:01:00::","synset_id":"personal.a.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"personal%5:00:00:subjective:00","call_ids":["senseval3.d001.s039.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"personal%5:00:00:subjective:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":897,"cached_input_tokens":0,"output_tokens":100,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002794,"input_uncached_usd":0.001794,"input_cached_usd":0.0,"output_usd":0.001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.587134875007905}
{"item_id":"senseval3.d001.s039.t006","gold_sense_keys":["staff%1:14:01::"],"candidates":[{"index":1,"sense_key":"staff%1:14:01::","synset_id":"staff.n.01"},{"index":2,"sense_key":"staff%1:06:00::","synset_id":"staff.n.02"},{"index":3,"sense_key":"staff%1:14:00::","synset_id":"staff.n.03"},{"index":4,"sense_key":"staff%1:27:00::","synset_id":"staff.n.04"},{"index":5,"sense_key":"staff%1:10:01::","synset_id":"staff.n.05"},{"index":6,"sense_key":"staff%1:10:00::","synset_id":"staff.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"staff%1:14:01::","call_ids":["senseval3.d001.s039.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"staff%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1026,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002172,"input_uncached_usd":0.002052,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4738452500896528}
{"item_id":"senseval3.d001.s039.t008","gold_sense_keys":["office%1:06:00::"],"candidates":[{"index":1,"sense_key":"office%1:06:00::","synset_id":"office.n.01"},{"index":2,"sense_key":"office%1:14:01::","synset_id":"agency.n.01"},{"index":3,"sense_key":"office%1:04:01::","synset_id":"function.n.03"},{"index":4,"sense_key":"office%1:26:00::","synset_id":"office.n.04"},{"index":5,"sense_key":"office%1:14:00::","synset_id":"office.n.05"},{"index":6,"sense_key":"office%1:04:00::","synset_id":"office.n.06"},{"index":7,"sense_key":"office%1:04:02::","synset_id":"position.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"office%1:06:00::","call_ids":["senseval3.d001.s039.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"office%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1125,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00283,"input_uncached_usd":0.00225,"input_cached_usd":0.0,"output_usd":0.00058,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2823433330049738}
{"item_id":"senseval3.d001.s039.t009","gold_sense_keys":["local%3:00:01::"],"candidates":[{"index":1,"sense_key":"local%3:00:01::","synset_id":"local.a.01"},{"index":2,"sense_key":"local%3:01:01::","synset_id":"local.a.02"},{"index":3,"sense_key":"local%3:00:02::","synset_id":"local.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"local%3:00:01::","call_ids":["senseval3.d001.s039.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"local%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":839,"cached_input_tokens":0,"output_tokens":139,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0030680000000000004,"input_uncached_usd":0.001678,"input_cached_usd":0.0,"output_usd":0.0013900000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8301507500000298}
{"item_id":"senseval3.d001.s039.t011","gold_sense_keys":["duty%1:04:02::"],"candidates":[{"index":1,"sense_key":"duty%1:04:00::","synset_id":"duty.n.01"},{"index":2,"sense_key":"duty%1:04:02::","synset_id":"duty.n.02"},{"index":3,"sense_key":"duty%1:21:00::","synset_id":"duty.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"duty%1:04:02::","call_ids":["senseval3.d001.s039.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"duty%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":909,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002548,"input_uncached_usd":0.001818,"input_cached_usd":0.0,"output_usd":0.0007300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5079368330771104}
{"item_id":"senseval3.d001.s039.t013","gold_sense_keys":["national%3:00:00::","national%3:00:01::","national%3:01:00::","national%5:00:00:domestic:00"],"candidates":[{"index":1,"sense_key":"national%3:01:00::","synset_id":"national.a.01"},{"index":2,"sense_key":"national%3:00:00::","synset_id":"national.a.02"},{"index":3,"sense_key":"national%3:00:01::","synset_id":"national.a.03"},{"index":4,"sense_key":"national%5:00:00:public:00","synset_id":"national.s.01"},{"index":5,"sense_key":"national%5:00:00:domestic:00","synset_id":"home.s.01"},{"index":6,"sense_key":"national%3:01:02::","synset_id":"national.a.06"},{"index":7,"sense_key":"national%3:01:01::","synset_id":"national.a.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"national%3:00:01::","call_ids":["senseval3.d001.s039.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"national%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1038,"cached_input_tokens":0,"output_tokens":103,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003106,"input_uncached_usd":0.0020759999999999997,"input_cached_usd":0.0,"output_usd":0.00103,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6760006659897044}
{"item_id":"senseval3.d001.s039.t014","gold_sense_keys":["policy%1:09:00::"],"candidates":[{"index":1,"sense_key":"policy%1:09:00::","synset_id":"policy.n.01"},{"index":2,"sense_key":"policy%1:10:00::","synset_id":"policy.n.02"},{"index":3,"sense_key":"policy%1:10:02::","synset_id":"policy.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"policy%1:09:00::","call_ids":["senseval3.d001.s039.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"policy%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":864,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001848,"input_uncached_usd":0.001728,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0644440420437604}
{"item_id":"senseval3.d001.s039.t015","gold_sense_keys":["making%1:04:00::"],"candidates":[{"index":1,"sense_key":"making%1:04:00::","synset_id":"devising.n.01"},{"index":2,"sense_key":"making%1:07:00::","synset_id":"qualification.n.01"},{"index":3,"sense_key":"making%1:06:00::","synset_id":"making.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"making%1:04:00::","call_ids":["senseval3.d001.s039.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"making%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":883,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001886,"input_uncached_usd":0.001766,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2276562500046566}
{"item_id":"senseval3.d001.s039.t016","gold_sense_keys":["activity%1:04:00::"],"candidates":[{"index":1,"sense_key":"activity%1:04:00::","synset_id":"activity.n.01"},{"index":2,"sense_key":"activity%1:26:00::","synset_id":"action.n.02"},{"index":3,"sense_key":"activity%1:22:00::","synset_id":"bodily_process.n.01"},{"index":4,"sense_key":"activity%1:26:01::","synset_id":"activity.n.04"},{"index":5,"sense_key":"activity%1:22:01::","synset_id":"natural_process.n.01"},{"index":6,"sense_key":"activity%1:07:00::","synset_id":"activeness.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"activity%1:04:00::","call_ids":["senseval3.d001.s039.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"activity%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1035,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0027199999999999998,"input_uncached_usd":0.00207,"input_cached_usd":0.0,"output_usd":0.0006500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5302861250238493}
{"item_id":"senseval3.d001.s039.t018","gold_sense_keys":["relative%5:00:00:proportionate:00"],"candidates":[{"index":1,"sense_key":"relative%3:00:00::","synset_id":"relative.a.01"},{"index":2,"sense_key":"relative%5:00:00:proportionate:00","synset_id":"proportional.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"relative%5:00:00:proportionate:00","call_ids":["senseval3.d001.s039.t018__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"relative%5:00:00:proportionate:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":823,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002196,"input_uncached_usd":0.001646,"input_cached_usd":0.0,"output_usd":0.00055,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1468667909502983}
{"item_id":"senseval3.d001.s039.t019","gold_sense_keys":["office%1:06:00::"],"candidates":[{"index":1,"sense_key":"office%1:06:00::","synset_id":"office.n.01"},{"index":2,"sense_key":"office%1:14:01::","synset_id":"agency.n.01"},{"index":3,"sense_key":"office%1:04:01::","synset_id":"function.n.03"},{"index":4,"sense_key":"office%1:26:00::","synset_id":"office.n.04"},{"index":5,"sense_key":"office%1:14:00::","synset_id":"office.n.05"},{"index":6,"sense_key":"office%1:04:00::","synset_id":"office.n.06"},{"index":7,"sense_key":"office%1:04:02::","synset_id":"position.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"office%1:06:00::","call_ids":["senseval3.d001.s039.t019__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"office%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1125,"cached_input_tokens":0,"output_tokens":52,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00277,"input_uncached_usd":0.00225,"input_cached_usd":0.0,"output_usd":0.0005200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6786547909723595}
{"item_id":"senseval3.d001.s040.t000","gold_sense_keys":["examination%1:04:00::"],"candidates":[{"index":1,"sense_key":"examination%1:04:00::","synset_id":"examination.n.01"},{"index":2,"sense_key":"examination%1:10:00::","synset_id":"examination.n.02"},{"index":3,"sense_key":"examination%1:10:01::","synset_id":"interrogation.n.03"},{"index":4,"sense_key":"examination%1:09:03::","synset_id":"examen.n.01"},{"index":5,"sense_key":"examination%1:04:02::","synset_id":"examination.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"examination%1:04:00::","call_ids":["senseval3.d001.s040.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"examination%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1028,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002176,"input_uncached_usd":0.002056,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2589279590174556}
{"item_id":"senseval3.d001.s040.t001","gold_sense_keys":["change%1:04:00::","change%1:11:00::","change%1:19:00::","change%1:24:00::"],"candidates":[{"index":1,"sense_key":"change%1:11:00::","synset_id":"change.n.01"},{"index":2,"sense_key":"change%1:24:00::","synset_id":"change.n.02"},{"index":3,"sense_key":"change%1:04:00::","synset_id":"change.n.03"},{"index":4,"sense_key":"change%1:19:00::","synset_id":"change.n.04"},{"index":5,"sense_key":"change%1:21:02::","synset_id":"change.n.05"},{"index":6,"sense_key":"change%1:06:01::","synset_id":"change.n.06"},{"index":7,"sense_key":"change%1:06:00::","synset_id":"change.n.07"},{"index":8,"sense_key":"change%1:21:01::","synset_id":"change.n.08"},{"index":9,"sense_key":"change%1:21:03::","synset_id":"change.n.09"},{"index":10,"sense_key":"change%1:07:00::","synset_id":"variety.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"change%1:19:00::","call_ids":["senseval3.d001.s040.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"change%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1329,"cached_input_tokens":0,"output_tokens":151,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004168,"input_uncached_usd":0.0026579999999999998,"input_cached_usd":0.0,"output_usd":0.00151,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.012360041961074}
{"item_id":"senseval3.d001.s040.t002","gold_sense_keys":["personal%3:00:00::","personal%5:00:00:subjective:00"],"candidates":[{"index":1,"sense_key":"personal%3:00:00::","synset_id":"personal.a.01"},{"index":2,"sense_key":"personal%5:00:00:subjective:00","synset_id":"personal.s.01"},{"index":3,"sense_key":"personal%3:01:01::","synset_id":"personal.a.03"},{"index":4,"sense_key":"personal%5:00:00:physical:00","synset_id":"personal.s.02"},{"index":5,"sense_key":"personal%3:01:00::","synset_id":"personal.a.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"personal%5:00:00:subjective:00","call_ids":["senseval3.d001.s040.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"personal%5:00:00:subjective:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":966,"cached_input_tokens":0,"output_tokens":83,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002762,"input_uncached_usd":0.001932,"input_cached_usd":0.0,"output_usd":0.0008300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6230697089340538}
{"item_id":"senseval3.d001.s040.t003","gold_sense_keys":["decision%1:09:00::"],"candidates":[{"index":1,"sense_key":"decision%1:04:00::","synset_id":"decision.n.01"},{"index":2,"sense_key":"decision%1:09:00::","synset_id":"decision.n.02"},{"index":3,"sense_key":"decision%1:11:02::","synset_id":"decision.n.03"},{"index":4,"sense_key":"decision%1:11:01::","synset_id":"decision.n.04"},{"index":5,"sense_key":"decision%1:07:00::","synset_id":"decisiveness.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"decision%1:04:00::","call_ids":["senseval3.d001.s040.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"decision%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1083,"cached_input_tokens":0,"output_tokens":158,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003746,"input_uncached_usd":0.002166,"input_cached_usd":0.0,"output_usd":0.00158,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.167183958925307}
{"item_id":"senseval3.d001.s040.t004","gold_sense_keys":["control%1:04:00::","control%1:07:00::","control%1:24:00::"],"candidates":[{"index":1,"sense_key":"control%1:07:00::","synset_id":"control.n.01"},{"index":2,"sense_key":"control%1:24:00::","synset_id":"control.n.02"},{"index":3,"sense_key":"control%1:04:02::","synset_id":"control.n.03"},{"index":4,"sense_key":"control%1:09:02::","synset_id":"control_condition.n.01"},{"index":5,"sense_key":"control%1:04:00::","synset_id":"control.n.05"},{"index":6,"sense_key":"control%1:26:00::","synset_id":"dominance.n.02"},{"index":7,"sense_key":"control%1:07:01::","synset_id":"restraint.n.02"},{"index":8,"sense_key":"control%1:09:01::","synset_id":"command.n.06"},{"index":9,"sense_key":"control%1:06:00::","synset_id":"control.n.09"},{"index":10,"sense_key":"control%1:18:00::","synset_id":"control.n.10"},{"index":11,"sense_key":"control%1:10:00::","synset_id":"control.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"control%1:07:00::","call_ids":["senseval3.d001.s040.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"control%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1454,"cached_input_tokens":0,"output_tokens":141,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004318,"input_uncached_usd":0.002908,"input_cached_usd":0.0,"output_usd":0.00141,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.4923656249884516}
{"item_id":"senseval3.d001.s040.t005","gold_sense_keys":["body%1:14:00::"],"candidates":[{"index":1,"sense_key":"body%1:08:00::","synset_id":"body.n.01"},{"index":2,"sense_key":"body%1:14:00::","synset_id":"body.n.02"},{"index":3,"sense_key":"body%1:08:02::","synset_id":"body.n.03"},{"index":4,"sense_key":"body%1:17:00::","synset_id":"body.n.04"},{"index":5,"sense_key":"body%1:08:01::","synset_id":"torso.n.01"},{"index":6,"sense_key":"body%1:14:02::","synset_id":"body.n.06"},{"index":7,"sense_key":"body%1:07:00::","synset_id":"consistency.n.01"},{"index":8,"sense_key":"body%1:10:00::","synset_id":"body.n.08"},{"index":9,"sense_key":"body%1:07:01::","synset_id":"body.n.09"},{"index":10,"sense_key":"body%1:06:01::","synset_id":"soundbox.n.01"},{"index":11,"sense_key":"body%1:06:00::","synset_id":"body.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"body%1:14:00::","call_ids":["senseval3.d001.s040.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"body%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1362,"cached_input_tokens":0,"output_tokens":45,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0031739999999999997,"input_uncached_usd":0.002724,"input_cached_usd":0.0,"output_usd":0.00045000000000000004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.420643291901797}
{"item_id":"senseval3.d001.s040.t007","gold_sense_keys":["party%1:14:01::"],"candidates":[{"index":1,"sense_key":"party%1:14:01::","synset_id":"party.n.01"},{"index":2,"sense_key":"party%1:14:00::","synset_id":"party.n.02"},{"index":3,"sense_key":"party%1:14:02::","synset_id":"party.n.03"},{"index":4,"sense_key":"party%1:11:00::","synset_id":"party.n.04"},{"index":5,"sense_key":"party%1:18:00::","synset_id":"party.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"party%1:14:01::","call_ids":["senseval3.d001.s040.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"party%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1026,"cached_input_tokens":0,"output_tokens":51,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002562,"input_uncached_usd":0.002052,"input_cached_usd":0.0,"output_usd":0.00051,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7751718749059364}
{"item_id":"senseval3.d001.s040.t008","gold_sense_keys":["reveal%2:39:00::"],"candidates":[{"index":1,"sense_key":"reveal%2:39:00::","synset_id":"uncover.v.01"},{"index":2,"sense_key":"reveal%2:32:00::","synset_id":"unwrap.v.02"},{"index":3,"sense_key":"reveal%2:39:04::","synset_id":"reveal.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"reveal%2:32:00::","call_ids":["senseval3.d001.s040.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"reveal%2:32:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":984,"cached_input_tokens":0,"output_tokens":91,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002878,"input_uncached_usd":0.0019679999999999997,"input_cached_usd":0.0,"output_usd":0.0009100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.008957125013694}
{"item_id":"senseval3.d001.s040.t009","gold_sense_keys":["personal%3:00:00::","personal%5:00:00:subjective:00"],"candidates":[{"index":1,"sense_key":"personal%3:00:00::","synset_id":"personal.a.01"},{"index":2,"sense_key":"personal%5:00:00:subjective:00","synset_id":"personal.s.01"},{"index":3,"sense_key":"personal%3:01:01::","synset_id":"personal.a.03"},{"index":4,"sense_key":"personal%5:00:00:physical:00","synset_id":"personal.s.02"},{"index":5,"sense_key":"personal%3:01:00::","synset_id":"personal.a.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"personal%5:00:00:subjective:00","call_ids":["senseval3.d001.s040.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"personal%5:00:00:subjective:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":966,"cached_input_tokens":0,"output_tokens":114,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003072,"input_uncached_usd":0.001932,"input_cached_usd":0.0,"output_usd":0.0011400000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.816304916050285}
{"item_id":"senseval3.d001.s040.t010","gold_sense_keys":["difference%1:07:00::"],"candidates":[{"index":1,"sense_key":"difference%1:07:00::","synset_id":"difference.n.01"},{"index":2,"sense_key":"difference%1:11:00::","synset_id":"deviation.n.01"},{"index":3,"sense_key":"difference%1:10:00::","synset_id":"dispute.n.01"},{"index":4,"sense_key":"difference%1:24:00::","synset_id":"difference.n.04"},{"index":5,"sense_key":"difference%1:23:00::","synset_id":"remainder.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"difference%1:07:00::","call_ids":["senseval3.d001.s040.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"difference%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1047,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002214,"input_uncached_usd":0.002094,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.182643083971925}
{"item_id":"senseval3.d001.s040.t011","gold_sense_keys":["note%2:32:00::"],"candidates":[{"index":1,"sense_key":"note%2:32:00::","synset_id":"note.v.01"},{"index":2,"sense_key":"note%2:39:00::","synset_id":"notice.v.02"},{"index":3,"sense_key":"note%2:39:02::","synset_id":"note.v.03"},{"index":4,"sense_key":"note%2:32:01::","synset_id":"note.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"note%2:32:00::","call_ids":["senseval3.d001.s040.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"note%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":982,"cached_input_tokens":0,"output_tokens":83,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002794,"input_uncached_usd":0.001964,"input_cached_usd":0.0,"output_usd":0.0008300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2785671249730512}
{"item_id":"senseval3.d001.s040.t012","gold_sense_keys":["table%1:14:00::"],"candidates":[{"index":1,"sense_key":"table%1:14:00::","synset_id":"table.n.01"},{"index":2,"sense_key":"table%1:06:01::","synset_id":"table.n.02"},{"index":3,"sense_key":"table%1:06:02::","synset_id":"table.n.03"},{"index":4,"sense_key":"table%1:17:00::","synset_id":"mesa.n.01"},{"index":5,"sense_key":"table%1:14:01::","synset_id":"table.n.05"},{"index":6,"sense_key":"table%1:13:01::","synset_id":"board.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"table%1:14:00::","call_ids":["senseval3.d001.s040.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"table%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1095,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00231,"input_uncached_usd":0.00219,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0790335410274565}
{"item_id":"senseval3.d001.s040.t013","gold_sense_keys":["attribute%2:31:00::"],"candidates":[{"index":1,"sense_key":"attribute%2:31:00::","synset_id":"impute.v.01"},{"index":2,"sense_key":"attribute%2:31:01::","synset_id":"assign.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"attribute%2:31:00::","call_ids":["senseval3.d001.s040.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"attribute%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":891,"cached_input_tokens":0,"output_tokens":113,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002912,"input_uncached_usd":0.001782,"input_cached_usd":0.0,"output_usd":0.0011300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.51634366705548}
{"item_id":"senseval3.d001.s040.t015","gold_sense_keys":["control%1:07:00::"],"candidates":[{"index":1,"sense_key":"control%1:07:00::","synset_id":"control.n.01"},{"index":2,"sense_key":"control%1:24:00::","synset_id":"control.n.02"},{"index":3,"sense_key":"control%1:04:02::","synset_id":"control.n.03"},{"index":4,"sense_key":"control%1:09:02::","synset_id":"control_condition.n.01"},{"index":5,"sense_key":"control%1:04:00::","synset_id":"control.n.05"},{"index":6,"sense_key":"control%1:26:00::","synset_id":"dominance.n.02"},{"index":7,"sense_key":"control%1:07:01::","synset_id":"restraint.n.02"},{"index":8,"sense_key":"control%1:09:01::","synset_id":"command.n.06"},{"index":9,"sense_key":"control%1:06:00::","synset_id":"control.n.09"},{"index":10,"sense_key":"control%1:18:00::","synset_id":"control.n.10"},{"index":11,"sense_key":"control%1:10:00::","synset_id":"control.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"control%1:07:00::","call_ids":["senseval3.d001.s040.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"control%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1454,"cached_input_tokens":0,"output_tokens":196,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004868000000000001,"input_uncached_usd":0.002908,"input_cached_usd":0.0,"output_usd":0.0019600000000000004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":4.291069541010074}
{"item_id":"senseval3.d001.s040.t016","gold_sense_keys":["democrat%1:18:01::"],"candidates":[{"index":1,"sense_key":"democrat%1:18:01::","synset_id":"democrat.n.01"},{"index":2,"sense_key":"democrat%1:18:00::","synset_id":"democrat.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"democrat%1:18:01::","call_ids":["senseval3.d001.s040.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"democrat%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":823,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001766,"input_uncached_usd":0.001646,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1592769590206444}
{"item_id":"senseval3.d001.s040.t017","gold_sense_keys":["exercise%2:34:00::"],"candidates":[{"index":1,"sense_key":"exercise%2:34:00::","synset_id":"exert.v.01"},{"index":2,"sense_key":"exercise%2:41:00::","synset_id":"practice.v.01"},{"index":3,"sense_key":"exercise%2:29:01::","synset_id":"exercise.v.03"},{"index":4,"sense_key":"exercise%2:29:00::","synset_id":"exercise.v.04"},{"index":5,"sense_key":"exercise%2:31:00::","synset_id":"drill.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"exercise%2:34:00::","call_ids":["senseval3.d001.s040.t017__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"exercise%2:34:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1031,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002182,"input_uncached_usd":0.002062,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.170374249923043}
{"item_id":"senseval3.d001.s040.t019","gold_sense_keys":["status%1:26:00::"],"candidates":[{"index":1,"sense_key":"status%1:26:00::","synset_id":"status.n.01"},{"index":2,"sense_key":"status%1:26:01::","synset_id":"condition.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"status%1:26:00::","call_ids":["senseval3.d001.s040.t019__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"status%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":877,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002314,"input_uncached_usd":0.001754,"input_cached_usd":0.0,"output_usd":0.0005600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.800082332920283}
{"item_id":"senseval3.d001.s040.t020","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":2,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":3,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"other%3:00:00::","call_ids":["senseval3.d001.s040.t020__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":979,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002078,"input_uncached_usd":0.001958,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9021172909997404}
{"item_id":"senseval3.d001.s040.t021","gold_sense_keys":["resource%1:07:00::"],"candidates":[{"index":1,"sense_key":"resource%1:21:00::","synset_id":"resource.n.01"},{"index":2,"sense_key":"resource%1:07:00::","synset_id":"resource.n.02"},{"index":3,"sense_key":"resource%1:09:00::","synset_id":"resource.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"resource%1:07:00::","call_ids":["senseval3.d001.s040.t021__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"resource%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":912,"cached_input_tokens":0,"output_tokens":135,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0031739999999999997,"input_uncached_usd":0.0018239999999999999,"input_cached_usd":0.0,"output_usd":0.00135,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.840984708047472}
{"item_id":"senseval3.d001.s040.t022","gold_sense_keys":["committee%1:14:00::"],"candidates":[{"index":1,"sense_key":"committee%1:14:00::","synset_id":"committee.n.01"},{"index":2,"sense_key":"committee%1:14:01::","synset_id":"committee.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"committee%1:14:00::","call_ids":["senseval3.d001.s040.t022__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"committee%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":864,"cached_input_tokens":0,"output_tokens":75,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002478,"input_uncached_usd":0.001728,"input_cached_usd":0.0,"output_usd":0.00075,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.220824124990031}
{"item_id":"senseval3.d001.s040.t023","gold_sense_keys":["staff%1:14:01::"],"candidates":[{"index":1,"sense_key":"staff%1:14:01::","synset_id":"staff.n.01"},{"index":2,"sense_key":"staff%1:06:00::","synset_id":"staff.n.02"},{"index":3,"sense_key":"staff%1:14:00::","synset_id":"staff.n.03"},{"index":4,"sense_key":"staff%1:27:00::","synset_id":"staff.n.04"},{"index":5,"sense_key":"staff%1:10:01::","synset_id":"staff.n.05"},{"index":6,"sense_key":"staff%1:10:00::","synset_id":"staff.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"staff%1:14:01::","call_ids":["senseval3.d001.s040.t023__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"staff%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1094,"cached_input_tokens":0,"output_tokens":83,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003018,"input_uncached_usd":0.002188,"input_cached_usd":0.0,"output_usd":0.0008300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5773896670434624}
{"item_id":"senseval3.d001.s041.t001","gold_sense_keys":["piece%1:06:00::"],"candidates":[{"index":1,"sense_key":"piece%1:06:00::","synset_id":"piece.n.01"},{"index":2,"sense_key":"piece%1:06:05::","synset_id":"piece.n.02"},{"index":3,"sense_key":"piece%1:17:00::","synset_id":"part.n.03"},{"index":4,"sense_key":"piece%1:10:01::","synset_id":"musical_composition.n.01"},{"index":5,"sense_key":"piece%1:11:00::","synset_id":"piece.n.05"},{"index":6,"sense_key":"piece%1:10:02::","synset_id":"piece.n.06"},{"index":7,"sense_key":"piece%1:06:03::","synset_id":"firearm.n.01"},{"index":8,"sense_key":"piece%1:13:00::","synset_id":"piece.n.08"},{"index":9,"sense_key":"piece%1:07:00::","synset_id":"piece.n.09"},{"index":10,"sense_key":"piece%1:06:04::","synset_id":"objet_d'art.n.01"},{"index":11,"sense_key":"piece%1:28:00::","synset_id":"while.n.01"},{"index":12,"sense_key":"piece%1:21:00::","synset_id":"slice.n.01"},{"index":13,"sense_key":"piece%1:06:02::","synset_id":"man.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"piece%1:06:05::","call_ids":["senseval3.d001.s041.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"piece%1:06:05::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1529,"cached_input_tokens":0,"output_tokens":281,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.005868,"input_uncached_usd":0.003058,"input_cached_usd":0.0,"output_usd":0.0028100000000000004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.546902709058486}
{"item_id":"senseval3.d001.s041.t002","gold_sense_keys":["evidence%1:09:00::","evidence%1:10:01::"],"candidates":[{"index":1,"sense_key":"evidence%1:09:00::","synset_id":"evidence.n.01"},{"index":2,"sense_key":"evidence%1:10:01::","synset_id":"evidence.n.02"},{"index":3,"sense_key":"evidence%1:10:00::","synset_id":"evidence.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"evidence%1:09:00::","call_ids":["senseval3.d001.s041.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"evidence%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":924,"cached_input_tokens":0,"output_tokens":143,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0032779999999999997,"input_uncached_usd":0.0018479999999999998,"input_cached_usd":0.0,"output_usd":0.00143,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.3148941659601405}
{"item_id":"senseval3.d001.s041.t003","gold_sense_keys":["hold%2:42:00::"],"candidates":[{"index":1,"sense_key":"hold%2:42:00::","synset_id":"keep.v.01"},{"index":2,"sense_key":"hold%2:35:00::","synset_id":"hold.v.02"},{"index":3,"sense_key":"hold%2:36:00::","synset_id":"hold.v.03"},{"index":4,"sense_key":"hold%2:40:00::","synset_id":"have.v.01"},{"index":5,"sense_key":"hold%2:31:01::","synset_id":"deem.v.01"},{"index":6,"sense_key":"hold%2:37:00::","synset_id":"harbor.v.01"},{"index":7,"sense_key":"hold%2:35:03::","synset_id":"restrain.v.03"},{"index":8,"sense_key":"hold%2:40:02::","synset_id":"retain.v.03"},{"index":9,"sense_key":"hold%2:40:04::","synset_id":"bear.v.11"},{"index":10,"sense_key":"hold%2:35:01::","synset_id":"hold.v.10"},{"index":11,"sense_key":"hold%2:42:13::","synset_id":"hold.v.11"},{"index":12,"sense_key":"hold%2:42:05::","synset_id":"accommodate.v.04"},{"index":13,"sense_key":"hold%2:42:01::","synset_id":"hold.v.13"},{"index":14,"sense_key":"hold%2:35:02::","synset_id":"hold.v.14"},{"index":15,"sense_key":"hold%2:42:02::","synset_id":"prevail.v.02"},{"index":16,"sense_key":"hold%2:31:08::","synset_id":"hold.v.16"},{"index":17,"sense_key":"hold%2:42:07::","synset_id":"hold.v.17"},{"index":18,"sense_key":"hold%2:42:14::","synset_id":"contain.v.05"},{"index":19,"sense_key":"hold%2:41:00::","synset_id":"reserve.v.04"},{"index":20,"sense_key":"hold%2:33:05::","synset_id":"defend.v.03"},{"index":21,"sense_key":"hold%2:32:13::","synset_id":"oblige.v.02"},{"index":22,"sense_key":"hold%2:31:04::","synset_id":"hold.v.22"},{"index":23,"sense_key":"hold%2:31:10::","synset_id":"hold.v.23"},{"index":24,"sense_key":"hold%2:42:08::","synset_id":"defy.v.01"},{"index":25,"sense_key":"hold%2:42:04::","synset_id":"apply.v.02"},{"index":26,"sense_key":"hold%2:42:12::","synset_id":"hold.v.26"},{"index":27,"sense_key":"hold%2:41:15::","synset_id":"control.v.02"},{"index":28,"sense_key":"hold%2:41:13::","synset_id":"hold.v.28"},{"index":29,"sense_key":"hold%2:41:14::","synset_id":"hold.v.29"},{"index":30,"sense_key":"hold%2:38:05::","synset_id":"halt.v.01"},{"index":31,"sense_key":"hold%2:35:09::","synset_id":"hold.v.31"},{"index":32,"sense_key":"hold%2:34:00::","synset_id":"carry.v.33"},{"index":33,"sense_key":"hold%2:33:00::","synset_id":"hold.v.33"},{"index":34,"sense_key":"hold%2:32:11::","synset_id":"declare.v.04"},{"index":35,"sense_key":"hold%2:32:15::","synset_id":"agree.v.01"},{"index":36,"sense_key":"hold%2:29:04::","synset_id":"hold.v.36"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"hold%2:42:00::","call_ids":["senseval3.d001.s041.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"hold%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":3078,"cached_input_tokens":0,"output_tokens":80,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.006956,"input_uncached_usd":0.006156,"input_cached_usd":0.0,"output_usd":0.0008,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7542065830202773}
{"item_id":"senseval3.d001.s041.t004","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":2,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":3,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"other%3:00:00::","call_ids":["senseval3.d001.s041.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":969,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002058,"input_uncached_usd":0.0019379999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1646109169814736}
{"item_id":"senseval3.d001.s041.t005","gold_sense_keys":["factor%1:11:00::"],"candidates":[{"index":1,"sense_key":"factor%1:11:00::","synset_id":"factor.n.01"},{"index":2,"sense_key":"factor%1:09:00::","synset_id":"component.n.01"},{"index":3,"sense_key":"factor%1:23:00::","synset_id":"divisor.n.01"},{"index":4,"sense_key":"factor%1:18:00::","synset_id":"agent.n.04"},{"index":5,"sense_key":"factor%1:23:01::","synset_id":"factor.n.05"},{"index":6,"sense_key":"factor%1:09:01::","synset_id":"factor.n.06"},{"index":7,"sense_key":"factor%1:08:00::","synset_id":"gene.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"factor%1:11:00::","call_ids":["senseval3.d001.s041.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"factor%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1169,"cached_input_tokens":0,"output_tokens":96,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0032979999999999997,"input_uncached_usd":0.002338,"input_cached_usd":0.0,"output_usd":0.0009600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.101467166095972}
{"item_id":"senseval3.d001.s041.t006","gold_sense_keys":["constant%5:00:00:invariable:00"],"candidates":[{"index":1,"sense_key":"constant%5:00:00:invariable:00","synset_id":"changeless.s.01"},{"index":2,"sense_key":"constant%3:00:00::","synset_id":"constant.a.02"},{"index":3,"sense_key":"constant%5:00:00:continuous:01","synset_id":"ceaseless.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"constant%5:00:00:invariable:00","call_ids":["senseval3.d001.s041.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"constant%5:00:00:invariable:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":973,"cached_input_tokens":0,"output_tokens":89,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002836,"input_uncached_usd":0.0019459999999999998,"input_cached_usd":0.0,"output_usd":0.0008900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7051245839102194}
{"item_id":"senseval3.d001.s041.t007","gold_sense_keys":["incumbency%1:04:00::","incumbency%1:28:00::"],"candidates":[{"index":1,"sense_key":"incumbency%1:28:00::","synset_id":"tenure.n.01"},{"index":2,"sense_key":"incumbency%1:04:01::","synset_id":"incumbency.n.02"},{"index":3,"sense_key":"incumbency%1:04:00::","synset_id":"incumbency.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"incumbency%1:04:00::","call_ids":["senseval3.d001.s041.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"incumbency%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":853,"cached_input_tokens":0,"output_tokens":87,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002576,"input_uncached_usd":0.0017059999999999998,"input_cached_usd":0.0,"output_usd":0.0008700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3515540000516921}
{"item_id":"senseval3.d001.s041.t008","gold_sense_keys":["advantage%1:07:01::"],"candidates":[{"index":1,"sense_key":"advantage%1:07:00::","synset_id":"advantage.n.01"},{"index":2,"sense_key":"advantage%1:23:00::","synset_id":"advantage.n.02"},{"index":3,"sense_key":"advantage%1:07:01::","synset_id":"advantage.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"advantage%1:07:00::","call_ids":["senseval3.d001.s041.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"advantage%1:07:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":903,"cached_input_tokens":0,"output_tokens":87,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002676,"input_uncached_usd":0.0018059999999999999,"input_cached_usd":0.0,"output_usd":0.0008700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8079165830276906}
{"item_id":"senseval3.d001.s041.t009","gold_sense_keys":["regional%5:00:00:territorial:00"],"candidates":[{"index":1,"sense_key":"regional%3:01:00::","synset_id":"regional.a.01"},{"index":2,"sense_key":"regional%5:00:00:territorial:00","synset_id":"regional.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"regional%5:00:00:territorial:00","call_ids":["senseval3.d001.s041.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"regional%5:00:00:territorial:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":819,"cached_input_tokens":0,"output_tokens":163,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003268,"input_uncached_usd":0.001638,"input_cached_usd":0.0,"output_usd":0.0016300000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.24910654197447}
{"item_id":"senseval3.d001.s041.t010","gold_sense_keys":["factor%1:11:00::"],"candidates":[{"index":1,"sense_key":"factor%1:11:00::","synset_id":"factor.n.01"},{"index":2,"sense_key":"factor%1:09:00::","synset_id":"component.n.01"},{"index":3,"sense_key":"factor%1:23:00::","synset_id":"divisor.n.01"},{"index":4,"sense_key":"factor%1:18:00::","synset_id":"agent.n.04"},{"index":5,"sense_key":"factor%1:23:01::","synset_id":"factor.n.05"},{"index":6,"sense_key":"factor%1:09:01::","synset_id":"factor.n.06"},{"index":7,"sense_key":"factor%1:08:00::","synset_id":"gene.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"factor%1:11:00::","call_ids":["senseval3.d001.s041.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"factor%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1169,"cached_input_tokens":0,"output_tokens":136,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003698,"input_uncached_usd":0.002338,"input_cached_usd":0.0,"output_usd":0.00136,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1310030419845134}
{"item_id":"senseval3.d001.s041.t011","gold_sense_keys":["difference%1:23:00::"],"candidates":[{"index":1,"sense_key":"difference%1:07:00::","synset_id":"difference.n.01"},{"index":2,"sense_key":"difference%1:11:00::","synset_id":"deviation.n.01"},{"index":3,"sense_key":"difference%1:10:00::","synset_id":"dispute.n.01"},{"index":4,"sense_key":"difference%1:24:00::","synset_id":"difference.n.04"},{"index":5,"sense_key":"difference%1:23:00::","synset_id":"remainder.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"difference%1:23:00::","call_ids":["senseval3.d001.s041.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"difference%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1037,"cached_input_tokens":0,"output_tokens":157,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003644,"input_uncached_usd":0.002074,"input_cached_usd":0.0,"output_usd":0.0015700000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0753117919666693}
{"item_id":"senseval3.d001.s041.t012","gold_sense_keys":["popular%5:00:00:democratic:00"],"candidates":[{"index":1,"sense_key":"popular%3:00:00::","synset_id":"popular.a.01"},{"index":2,"sense_key":"popular%5:00:00:democratic:00","synset_id":"popular.s.01"},{"index":3,"sense_key":"popular%5:00:01:common:01","synset_id":"democratic.s.01"},{"index":4,"sense_key":"popular%5:00:00:nonclassical:00","synset_id":"popular.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"popular%5:00:00:democratic:00","call_ids":["senseval3.d001.s041.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"popular%5:00:00:democratic:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":986,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0020919999999999997,"input_uncached_usd":0.001972,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8516528329346329}
{"item_id":"senseval3.d001.s041.t013","gold_sense_keys":["vote%1:04:00::"],"candidates":[{"index":1,"sense_key":"vote%1:04:00::","synset_id":"vote.n.01"},{"index":2,"sense_key":"vote%1:04:01::","synset_id":"vote.n.02"},{"index":3,"sense_key":"vote%1:07:00::","synset_id":"right_to_vote.n.01"},{"index":4,"sense_key":"vote%1:14:00::","synset_id":"vote.n.04"},{"index":5,"sense_key":"vote%1:09:00::","synset_id":"vote.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"vote%1:04:00::","call_ids":["senseval3.d001.s041.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"vote%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1065,"cached_input_tokens":0,"output_tokens":161,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0037400000000000003,"input_uncached_usd":0.00213,"input_cached_usd":0.0,"output_usd":0.00161,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2254039170220494}
{"item_id":"senseval3.d001.s041.t014","gold_sense_keys":["republican%3:01:01::"],"candidates":[{"index":1,"sense_key":"republican%3:01:01::","synset_id":"republican.a.01"},{"index":2,"sense_key":"republican%5:00:00:democratic:00","synset_id":"republican.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"republican%3:01:01::","call_ids":["senseval3.d001.s041.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"republican%3:01:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":923,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001966,"input_uncached_usd":0.001846,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4676227500895038}
{"item_id":"senseval3.d001.s041.t015","gold_sense_keys":["presidential%3:01:00::"],"candidates":[{"index":1,"sense_key":"presidential%3:01:00::","synset_id":"presidential.a.01"},{"index":2,"sense_key":"presidential%3:00:00::","synset_id":"presidential.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"presidential%3:01:00::","call_ids":["senseval3.d001.s041.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"presidential%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":831,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001782,"input_uncached_usd":0.0016619999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3227326250635087}
{"item_id":"senseval3.d001.s041.t017","gold_sense_keys":["candidate%1:18:01::"],"candidates":[{"index":1,"sense_key":"candidate%1:18:01::","synset_id":"campaigner.n.01"},{"index":2,"sense_key":"candidate%1:18:00::","synset_id":"candidate.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"candidate%1:18:01::","call_ids":["senseval3.d001.s041.t017__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"candidate%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":838,"cached_input_tokens":0,"output_tokens":52,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002196,"input_uncached_usd":0.001676,"input_cached_usd":0.0,"output_usd":0.0005200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5250657920259982}
{"item_id":"senseval3.d001.s041.t018","gold_sense_keys":["state%1:15:01::"],"candidates":[{"index":1,"sense_key":"state%1:15:01::","synset_id":"state.n.01"},{"index":2,"sense_key":"state%1:03:00::","synset_id":"state.n.02"},{"index":3,"sense_key":"state%1:14:01::","synset_id":"state.n.03"},{"index":4,"sense_key":"state%1:14:00::","synset_id":"state.n.04"},{"index":5,"sense_key":"state%1:26:02::","synset_id":"state_of_matter.n.01"},{"index":6,"sense_key":"state%1:26:01::","synset_id":"state.n.06"},{"index":7,"sense_key":"state%1:15:00::","synset_id":"country.n.02"},{"index":8,"sense_key":"state%1:14:02::","synset_id":"department_of_state.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"state%1:15:01::","call_ids":["senseval3.d001.s041.t018__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"state%1:15:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1266,"cached_input_tokens":0,"output_tokens":52,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003052,"input_uncached_usd":0.002532,"input_cached_usd":0.0,"output_usd":0.0005200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5250842079985887}
{"item_id":"senseval3.d001.s041.t019","gold_sense_keys":["conduct%2:41:00::"],"candidates":[{"index":1,"sense_key":"conduct%2:41:00::","synset_id":"conduct.v.01"},{"index":2,"sense_key":"conduct%2:36:00::","synset_id":"conduct.v.02"},{"index":3,"sense_key":"conduct%2:41:01::","synset_id":"behave.v.02"},{"index":4,"sense_key":"conduct%2:38:01::","synset_id":"lead.v.01"},{"index":5,"sense_key":"conduct%2:38:00::","synset_id":"impart.v.03"},{"index":6,"sense_key":"conduct%2:36:04::","synset_id":"conduct.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"conduct%2:41:00::","call_ids":["senseval3.d001.s041.t019__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"conduct%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1169,"cached_input_tokens":0,"output_tokens":76,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003098,"input_uncached_usd":0.002338,"input_cached_usd":0.0,"output_usd":0.00076,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7933813750278205}
{"item_id":"senseval3.d001.s041.t020","gold_sense_keys":["election%1:04:01::"],"candidates":[{"index":1,"sense_key":"election%1:04:01::","synset_id":"election.n.01"},{"index":2,"sense_key":"election%1:04:00::","synset_id":"election.n.02"},{"index":3,"sense_key":"election%1:26:00::","synset_id":"election.n.03"},{"index":4,"sense_key":"election%1:09:00::","synset_id":"election.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"election%1:04:01::","call_ids":["senseval3.d001.s041.t020__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"election%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":936,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001992,"input_uncached_usd":0.001872,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4089060840196908}
{"item_id":"senseval3.d001.s041.t021","gold_sense_keys":["turn_out%2:42:00::","turn_out%2:42:01::"],"candidates":[{"index":1,"sense_key":"turn_out%2:42:00::","synset_id":"prove.v.01"},{"index":2,"sense_key":"turn_out%2:42:01::","synset_id":"turn_out.v.02"},{"index":3,"sense_key":"turn_out%2:36:01::","synset_id":"turn_out.v.03"},{"index":4,"sense_key":"turn_out%2:42:02::","synset_id":"turn_out.v.04"},{"index":5,"sense_key":"turn_out%2:30:00::","synset_id":"turn_out.v.05"},{"index":6,"sense_key":"turn_out%2:36:00::","synset_id":"bear.v.05"},{"index":7,"sense_key":"turn_out%2:35:00::","synset_id":"eject.v.01"},{"index":8,"sense_key":"turn_out%2:41:03::","synset_id":"turn_out.v.08"},{"index":9,"sense_key":"turn_out%2:40:03::","synset_id":"turn_out.v.09"},{"index":10,"sense_key":"turn_out%2:38:00::","synset_id":"turn_out.v.10"},{"index":11,"sense_key":"turn_out%2:35:01::","synset_id":"switch_off.v.01"},{"index":12,"sense_key":"turn_out%2:29:00::","synset_id":"get_up.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"turn_out%2:42:00::","call_ids":["senseval3.d001.s041.t021__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"turn_out%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1484,"cached_input_tokens":0,"output_tokens":174,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004708,"input_uncached_usd":0.0029679999999999997,"input_cached_usd":0.0,"output_usd":0.0017400000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.995080125052482}
{"item_id":"senseval3.d001.s041.t022","gold_sense_keys":["positive%5:00:00:plus:00"],"candidates":[{"index":1,"sense_key":"positive%3:00:01::","synset_id":"positive.a.01"},{"index":2,"sense_key":"positive%5:00:00:certain:02","synset_id":"convinced.s.01"},{"index":3,"sense_key":"positive%5:00:00:advantageous:00","synset_id":"plus.s.01"},{"index":4,"sense_key":"positive%3:00:04::","synset_id":"positive.a.04"},{"index":5,"sense_key":"positive%5:00:00:formal:01","synset_id":"positive.s.03"},{"index":6,"sense_key":"positive%5:00:00:undeniable:00","synset_id":"incontrovertible.s.01"},{"index":7,"sense_key":"positive%3:01:00::","synset_id":"positivist.a.01"},{"index":8,"sense_key":"positive%3:00:05::","synset_id":"positive.a.08"},{"index":9,"sense_key":"positive%5:00:00:plus:00","synset_id":"positive.s.05"},{"index":10,"sense_key":"positive%5:00:00:charged:00","synset_id":"positive.s.06"},{"index":11,"sense_key":"positive%5:00:00:confident:00","synset_id":"cocksure.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"positive%3:00:05::","call_ids":["senseval3.d001.s041.t022__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"positive%3:00:05::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1453,"cached_input_tokens":0,"output_tokens":110,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004006,"input_uncached_usd":0.0029059999999999997,"input_cached_usd":0.0,"output_usd":0.0011,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7689169580116868}
{"item_id":"senseval3.d001.s041.t023","gold_sense_keys":["function%1:24:00::"],"candidates":[{"index":1,"sense_key":"function%1:24:00::","synset_id":"function.n.01"},{"index":2,"sense_key":"function%1:07:00::","synset_id":"function.n.02"},{"index":3,"sense_key":"function%1:04:00::","synset_id":"function.n.03"},{"index":4,"sense_key":"function%1:24:01::","synset_id":"function.n.04"},{"index":5,"sense_key":"function%1:14:00::","synset_id":"function.n.05"},{"index":6,"sense_key":"function%1:11:00::","synset_id":"affair.n.03"},{"index":7,"sense_key":"function%1:10:00::","synset_id":"routine.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"function%1:24:01::","call_ids":["senseval3.d001.s041.t023__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"function%1:24:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1200,"cached_input_tokens":0,"output_tokens":114,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0035399999999999997,"input_uncached_usd":0.0024,"input_cached_usd":0.0,"output_usd":0.0011400000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8044986249879003}
{"item_id":"senseval3.d001.s041.t027","gold_sense_keys":["burden%1:09:01::"],"candidates":[{"index":1,"sense_key":"burden%1:09:01::","synset_id":"burden.n.01"},{"index":2,"sense_key":"burden%1:06:01::","synset_id":"load.n.01"},{"index":3,"sense_key":"burden%1:10:00::","synset_id":"effect.n.04"},{"index":4,"sense_key":"burden%1:09:00::","synset_id":"burden.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"burden%1:09:01::","call_ids":["senseval3.d001.s041.t027__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"burden%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":946,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002012,"input_uncached_usd":0.001892,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.034974417067133}
{"item_id":"senseval3.d001.s041.t028","gold_sense_keys":["state%1:15:01::"],"candidates":[{"index":1,"sense_key":"state%1:15:01::","synset_id":"state.n.01"},{"index":2,"sense_key":"state%1:03:00::","synset_id":"state.n.02"},{"index":3,"sense_key":"state%1:14:01::","synset_id":"state.n.03"},{"index":4,"sense_key":"state%1:14:00::","synset_id":"state.n.04"},{"index":5,"sense_key":"state%1:26:02::","synset_id":"state_of_matter.n.01"},{"index":6,"sense_key":"state%1:26:01::","synset_id":"state.n.06"},{"index":7,"sense_key":"state%1:15:00::","synset_id":"country.n.02"},{"index":8,"sense_key":"state%1:14:02::","synset_id":"department_of_state.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"state%1:15:01::","call_ids":["senseval3.d001.s041.t028__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"state%1:15:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1266,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003192,"input_uncached_usd":0.002532,"input_cached_usd":0.0,"output_usd":0.0006600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4735413340386003}
{"item_id":"senseval3.d001.s041.t029","gold_sense_keys":["progressive%3:00:02::"],"candidates":[{"index":1,"sense_key":"progressive%3:00:01::","synset_id":"progressive.a.01"},{"index":2,"sense_key":"progressive%5:00:00:liberal:00","synset_id":"progressive.s.01"},{"index":3,"sense_key":"progressive%3:00:02::","synset_id":"progressive.a.03"},{"index":4,"sense_key":"progressive%5:00:00:increasing:00","synset_id":"progressive.s.02"},{"index":5,"sense_key":"progressive%5:00:00:ordered:00","synset_id":"progressive.s.03"},{"index":6,"sense_key":"progressive%5:00:00:active:02","synset_id":"progressive.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"progressive%3:00:02::","call_ids":["senseval3.d001.s041.t029__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"progressive%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1041,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002202,"input_uncached_usd":0.002082,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2968141249148175}
{"item_id":"senseval3.d001.s041.t031","gold_sense_keys":["hit%2:35:05::"],"candidates":[{"index":1,"sense_key":"hit%2:35:03::","synset_id":"hit.v.01"},{"index":2,"sense_key":"hit%2:35:01::","synset_id":"hit.v.02"},{"index":3,"sense_key":"hit%2:35:00::","synset_id":"hit.v.03"},{"index":4,"sense_key":"hit%2:38:00::","synset_id":"reach.v.01"},{"index":5,"sense_key":"hit%2:35:05::","synset_id":"hit.v.05"},{"index":6,"sense_key":"hit%2:33:01::","synset_id":"shoot.v.01"},{"index":7,"sense_key":"hit%2:40:05::","synset_id":"stumble.v.03"},{"index":8,"sense_key":"hit%2:33:02::","synset_id":"score.v.01"},{"index":9,"sense_key":"hit%2:39:00::","synset_id":"hit.v.09"},{"index":10,"sense_key":"hit%2:33:00::","synset_id":"strike.v.04"},{"index":11,"sense_key":"hit%2:41:00::","synset_id":"murder.v.01"},{"index":12,"sense_key":"hit%2:38:02::","synset_id":"hit.v.12"},{"index":13,"sense_key":"hit%2:38:07::","synset_id":"reach.v.02"},{"index":14,"sense_key":"hit%2:35:02::","synset_id":"strike.v.10"},{"index":15,"sense_key":"hit%2:34:00::","synset_id":"hit.v.15"},{"index":16,"sense_key":"hit%2:33:03::","synset_id":"hit.v.16"},{"index":17,"sense_key":"hit%2:32:12::","synset_id":"hit.v.17"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"hit%2:35:05::","call_ids":["senseval3.d001.s041.t031__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"hit%2:35:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1816,"cached_input_tokens":0,"output_tokens":111,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004742,"input_uncached_usd":0.003632,"input_cached_usd":0.0,"output_usd":0.00111,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.650272374972701}
{"item_id":"senseval3.d001.s041.t032","gold_sense_keys":["high%3:00:02::"],"candidates":[{"index":1,"sense_key":"high%3:00:02::","synset_id":"high.a.01"},{"index":2,"sense_key":"high%3:00:01::","synset_id":"high.a.02"},{"index":3,"sense_key":"high%5:00:00:superior:01","synset_id":"eminent.s.01"},{"index":4,"sense_key":"high%3:00:03::","synset_id":"high.a.04"},{"index":5,"sense_key":"high%5:00:00:elated:00","synset_id":"high.s.02"},{"index":6,"sense_key":"high%5:00:00:malodorous:00","synset_id":"gamey.s.02"},{"index":7,"sense_key":"high%5:00:00:intoxicated:00","synset_id":"high.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"high%3:00:02::","call_ids":["senseval3.d001.s041.t032__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"high%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1144,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002968,"input_uncached_usd":0.002288,"input_cached_usd":0.0,"output_usd":0.00068,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9888734999112785}
{"item_id":"senseval3.d001.s041.t034","gold_sense_keys":["state%1:15:01::"],"candidates":[{"index":1,"sense_key":"state%1:15:01::","synset_id":"state.n.01"},{"index":2,"sense_key":"state%1:03:00::","synset_id":"state.n.02"},{"index":3,"sense_key":"state%1:14:01::","synset_id":"state.n.03"},{"index":4,"sense_key":"state%1:14:00::","synset_id":"state.n.04"},{"index":5,"sense_key":"state%1:26:02::","synset_id":"state_of_matter.n.01"},{"index":6,"sense_key":"state%1:26:01::","synset_id":"state.n.06"},{"index":7,"sense_key":"state%1:15:00::","synset_id":"country.n.02"},{"index":8,"sense_key":"state%1:14:02::","synset_id":"department_of_state.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"state%1:15:01::","call_ids":["senseval3.d001.s041.t034__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"state%1:15:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1266,"cached_input_tokens":0,"output_tokens":86,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003392,"input_uncached_usd":0.002532,"input_cached_usd":0.0,"output_usd":0.0008600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.978330791927874}
{"item_id":"senseval3.d001.s042.t000","gold_sense_keys":["put%2:32:00::"],"candidates":[{"index":1,"sense_key":"put%2:35:00::","synset_id":"put.v.01"},{"index":2,"sense_key":"put%2:35:05::","synset_id":"put.v.02"},{"index":3,"sense_key":"put%2:32:00::","synset_id":"frame.v.04"},{"index":4,"sense_key":"put%2:34:00::","synset_id":"put.v.04"},{"index":5,"sense_key":"put%2:40:00::","synset_id":"invest.v.01"},{"index":6,"sense_key":"put%2:31:13::","synset_id":"place.v.11"},{"index":7,"sense_key":"put%2:39:13::","synset_id":"put.v.07"},{"index":8,"sense_key":"put%2:36:13::","synset_id":"put.v.08"},{"index":9,"sense_key":"put%2:31:00::","synset_id":"arrange.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"put%2:32:00::","call_ids":["senseval3.d001.s042.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"put%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1195,"cached_input_tokens":0,"output_tokens":81,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0031999999999999997,"input_uncached_usd":0.0023899999999999998,"input_cached_usd":0.0,"output_usd":0.0008100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6463164170272648}
{"item_id":"senseval3.d001.s042.t001","gold_sense_keys":["candidate%1:18:01::"],"candidates":[{"index":1,"sense_key":"candidate%1:18:01::","synset_id":"campaigner.n.01"},{"index":2,"sense_key":"candidate%1:18:00::","synset_id":"candidate.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"candidate%1:18:01::","call_ids":["senseval3.d001.s042.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"candidate%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":787,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001694,"input_uncached_usd":0.0015739999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9997888749931008}
{"item_id":"senseval3.d001.s042.t002","gold_sense_keys":["president%1:04:04::"],"candidates":[{"index":1,"sense_key":"president%1:18:01::","synset_id":"president.n.01"},{"index":2,"sense_key":"president%1:18:04::","synset_id":"president_of_the_united_states.n.01"},{"index":3,"sense_key":"president%1:18:00::","synset_id":"president.n.03"},{"index":4,"sense_key":"president%1:18:03::","synset_id":"president.n.04"},{"index":5,"sense_key":"president%1:18:02::","synset_id":"president.n.05"},{"index":6,"sense_key":"president%1:04:04::","synset_id":"president_of_the_united_states.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"president%1:04:04::","call_ids":["senseval3.d001.s042.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"president%1:04:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":996,"cached_input_tokens":0,"output_tokens":119,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003182,"input_uncached_usd":0.001992,"input_cached_usd":0.0,"output_usd":0.00119,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6690064169233665}
{"item_id":"senseval3.d001.s042.t003","gold_sense_keys":["look_on%2:31:00::"],"candidates":[{"index":1,"sense_key":"look_on%2:39:00::","synset_id":"watch.v.04"},{"index":2,"sense_key":"look_on%2:31:00::","synset_id":"think_of.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"look_on%2:31:00::","call_ids":["senseval3.d001.s042.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"look_on%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":825,"cached_input_tokens":0,"output_tokens":82,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00247,"input_uncached_usd":0.00165,"input_cached_usd":0.0,"output_usd":0.0008200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.656811958993785}
{"item_id":"senseval3.d001.s042.t005","gold_sense_keys":["republican%3:01:01::"],"candidates":[{"index":1,"sense_key":"republican%3:01:01::","synset_id":"republican.a.01"},{"index":2,"sense_key":"republican%5:00:00:democratic:00","synset_id":"republican.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"republican%3:01:01::","call_ids":["senseval3.d001.s042.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"republican%3:01:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":872,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001864,"input_uncached_usd":0.0017439999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4889608329394832}
{"item_id":"senseval3.d001.s042.t006","gold_sense_keys":["candidate%1:18:01::"],"candidates":[{"index":1,"sense_key":"candidate%1:18:01::","synset_id":"campaigner.n.01"},{"index":2,"sense_key":"candidate%1:18:00::","synset_id":"candidate.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"candidate%1:18:01::","call_ids":["senseval3.d001.s042.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"candidate%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":787,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001694,"input_uncached_usd":0.0015739999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.6317037920234725}
{"item_id":"senseval3.d001.s042.t009","gold_sense_keys":["severe%5:00:02:nonindulgent:00"],"candidates":[{"index":1,"sense_key":"severe%5:00:00:intense:00","synset_id":"severe.s.01"},{"index":2,"sense_key":"severe%5:00:00:strong:00","synset_id":"hard.s.01"},{"index":3,"sense_key":"severe%5:00:00:plain:01","synset_id":"austere.s.01"},{"index":4,"sense_key":"severe%5:00:02:nonindulgent:00","synset_id":"severe.s.04"},{"index":5,"sense_key":"severe%5:00:00:critical:03","synset_id":"dangerous.s.01"},{"index":6,"sense_key":"severe%5:00:01:bad:00","synset_id":"severe.s.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"severe%5:00:02:nonindulgent:00","call_ids":["senseval3.d001.s042.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"severe%5:00:02:nonindulgent:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1121,"cached_input_tokens":0,"output_tokens":125,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0034920000000000003,"input_uncached_usd":0.002242,"input_cached_usd":0.0,"output_usd":0.00125,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0674915000563487}
{"item_id":"senseval3.d001.s043.t000","gold_sense_keys":["appear%2:39:00::"],"candidates":[{"index":1,"sense_key":"appear%2:39:00::","synset_id":"look.v.02"},{"index":2,"sense_key":"appear%2:30:00::","synset_id":"appear.v.02"},{"index":3,"sense_key":"appear%2:30:01::","synset_id":"appear.v.03"},{"index":4,"sense_key":"appear%2:39:01::","synset_id":"appear.v.04"},{"index":5,"sense_key":"appear%2:30:02::","synset_id":"appear.v.05"},{"index":6,"sense_key":"appear%2:36:00::","synset_id":"appear.v.06"},{"index":7,"sense_key":"appear%2:41:03::","synset_id":"appear.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"appear%2:39:00::","call_ids":["senseval3.d001.s043.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"appear%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1182,"cached_input_tokens":0,"output_tokens":159,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003954,"input_uncached_usd":0.0023639999999999998,"input_cached_usd":0.0,"output_usd":0.00159,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2339873749297112}
{"item_id":"senseval3.d001.s043.t001","gold_sense_keys":["take%2:30:01::"],"candidates":[{"index":1,"sense_key":"take%2:41:04::","synset_id":"take.v.01"},{"index":2,"sense_key":"take%2:40:06::","synset_id":"take.v.02"},{"index":3,"sense_key":"take%2:38:09::","synset_id":"lead.v.01"},{"index":4,"sense_key":"take%2:35:00::","synset_id":"take.v.04"},{"index":5,"sense_key":"take%2:30:01::","synset_id":"assume.v.03"},{"index":6,"sense_key":"take%2:31:07::","synset_id":"take.v.06"},{"index":7,"sense_key":"take%2:38:10::","synset_id":"bring.v.01"},{"index":8,"sense_key":"take%2:40:00::","synset_id":"take.v.08"},{"index":9,"sense_key":"take%2:38:02::","synset_id":"take.v.09"},{"index":10,"sense_key":"take%2:31:01::","synset_id":"choose.v.01"},{"index":11,"sense_key":"take%2:40:05::","synset_id":"accept.v.02"},{"index":12,"sense_key":"take%2:41:00::","synset_id":"fill.v.04"},{"index":13,"sense_key":"take%2:31:03::","synset_id":"consider.v.03"},{"index":14,"sense_key":"take%2:42:00::","synset_id":"necessitate.v.01"},{"index":15,"sense_key":"take%2:39:00::","synset_id":"take.v.15"},{"index":16,"sense_key":"take%2:32:02::","synset_id":"film.v.01"},{"index":17,"sense_key":"take%2:30:00::","synset_id":"remove.v.01"},{"index":18,"sense_key":"take%2:34:00::","synset_id":"consume.v.02"},{"index":19,"sense_key":"take%2:31:09::","synset_id":"take.v.19"},{"index":20,"sense_key":"take%2:40:15::","synset_id":"take.v.20"},{"index":21,"sense_key":"take%2:40:01::","synset_id":"take.v.21"},{"index":22,"sense_key":"take%2:38:00::","synset_id":"assume.v.05"},{"index":23,"sense_key":"take%2:40:07::","synset_id":"accept.v.05"},{"index":24,"sense_key":"take%2:30:08::","synset_id":"take.v.24"},{"index":25,"sense_key":"take%2:31:02::","synset_id":"learn.v.04"},{"index":26,"sense_key":"take%2:32:00::","synset_id":"claim.v.05"},{"index":27,"sense_key":"take%2:38:05::","synset_id":"take.v.27"},{"index":28,"sense_key":"take%2:33:09::","synset_id":"aim.v.01"},{"index":29,"sense_key":"take%2:30:09::","synset_id":"take.v.29"},{"index":30,"sense_key":"take%2:42:10::","synset_id":"carry.v.02"},{"index":31,"sense_key":"take%2:40:03::","synset_id":"lease.v.04"},{"index":32,"sense_key":"take%2:40:04::","synset_id":"subscribe.v.05"},{"index":33,"sense_key":"take%2:40:02::","synset_id":"take.v.33"},{"index":34,"sense_key":"take%2:40:08::","synset_id":"take.v.34"},{"index":35,"sense_key":"take%2:35:14::","synset_id":"take.v.35"},{"index":36,"sense_key":"take%2:32:09::","synset_id":"claim.v.04"},{"index":37,"sense_key":"take%2:42:03::","synset_id":"accept.v.08"},{"index":38,"sense_key":"take%2:42:15::","synset_id":"contain.v.05"},{"index":39,"sense_key":"take%2:41:13::","synset_id":"take.v.39"},{"index":40,"sense_key":"take%2:38:11::","synset_id":"drive.v.16"},{"index":41,"sense_key":"take%2:33:08::","synset_id":"take.v.41"},{"index":42,"sense_key":"take%2:29:08::","synset_id":"contract.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"take%2:30:01::","call_ids":["senseval3.d001.s043.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"take%2:30:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":3294,"cached_input_tokens":0,"output_tokens":96,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.007548,"input_uncached_usd":0.006588,"input_cached_usd":0.0,"output_usd":0.0009600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5961454999633133}
{"item_id":"senseval3.d001.s043.t002","gold_sense_keys":["same%3:00:00::"],"candidates":[{"index":1,"sense_key":"same%3:00:02::","synset_id":"same.a.01"},{"index":2,"sense_key":"same%3:00:00::","synset_id":"same.a.02"},{"index":3,"sense_key":"same%3:00:04::","synset_id":"like.a.02"},{"index":4,"sense_key":"same%5:00:00:unchanged:00","synset_id":"same.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"same%3:00:00::","call_ids":["senseval3.d001.s043.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"same%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":914,"cached_input_tokens":0,"output_tokens":141,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003238,"input_uncached_usd":0.001828,"input_cached_usd":0.0,"output_usd":0.00141,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.5824603339424357}
{"item_id":"senseval3.d001.s043.t007","gold_sense_keys":["level%1:26:01::"],"candidates":[{"index":1,"sense_key":"level%1:07:00::","synset_id":"degree.n.01"},{"index":2,"sense_key":"level%1:26:01::","synset_id":"grade.n.02"},{"index":3,"sense_key":"level%1:26:00::","synset_id":"degree.n.02"},{"index":4,"sense_key":"level%1:07:01::","synset_id":"level.n.04"},{"index":5,"sense_key":"level%1:06:00::","synset_id":"level.n.05"},{"index":6,"sense_key":"level%1:06:03::","synset_id":"horizontal_surface.n.01"},{"index":7,"sense_key":"level%1:09:00::","synset_id":"level.n.07"},{"index":8,"sense_key":"level%1:06:01::","synset_id":"floor.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"level%1:26:01::","call_ids":["senseval3.d001.s043.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"level%1:26:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1215,"cached_input_tokens":0,"output_tokens":232,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00475,"input_uncached_usd":0.00243,"input_cached_usd":0.0,"output_usd":0.00232,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.813739583012648}
{"item_id":"senseval3.d001.s043.t008","gold_sense_keys":["federal%5:00:00:national:01"],"candidates":[{"index":1,"sense_key":"federal%5:00:00:national:01","synset_id":"federal.s.01"},{"index":2,"sense_key":"federal%3:01:02::","synset_id":"federal.a.02"},{"index":3,"sense_key":"federal%5:00:00:northern:02","synset_id":"union.s.01"},{"index":4,"sense_key":"federal%3:00:00::","synset_id":"federal.a.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"federal%5:00:00:national:01","call_ids":["senseval3.d001.s043.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"federal%5:00:00:national:01","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":976,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002552,"input_uncached_usd":0.001952,"input_cached_usd":0.0,"output_usd":0.0006000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9990289590787143}
{"item_id":"senseval3.d001.s043.t009","gold_sense_keys":["level%1:26:01::"],"candidates":[{"index":1,"sense_key":"level%1:07:00::","synset_id":"degree.n.01"},{"index":2,"sense_key":"level%1:26:01::","synset_id":"grade.n.02"},{"index":3,"sense_key":"level%1:26:00::","synset_id":"degree.n.02"},{"index":4,"sense_key":"level%1:07:01::","synset_id":"level.n.04"},{"index":5,"sense_key":"level%1:06:00::","synset_id":"level.n.05"},{"index":6,"sense_key":"level%1:06:03::","synset_id":"horizontal_surface.n.01"},{"index":7,"sense_key":"level%1:09:00::","synset_id":"level.n.07"},{"index":8,"sense_key":"level%1:06:01::","synset_id":"floor.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"level%1:26:01::","call_ids":["senseval3.d001.s043.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"level%1:26:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1215,"cached_input_tokens":0,"output_tokens":213,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00456,"input_uncached_usd":0.00243,"input_cached_usd":0.0,"output_usd":0.0021300000000000004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.548429499962367}
{"item_id":"senseval3.d001.s044.t003","gold_sense_keys":["line%1:09:00::"],"candidates":[{"index":1,"sense_key":"line%1:14:03::","synset_id":"line.n.01"},{"index":2,"sense_key":"line%1:10:01::","synset_id":"line.n.02"},{"index":3,"sense_key":"line%1:14:01::","synset_id":"line.n.03"},{"index":4,"sense_key":"line%1:25:00::","synset_id":"line.n.04"},{"index":5,"sense_key":"line%1:10:02::","synset_id":"line.n.05"},{"index":6,"sense_key":"line%1:19:00::","synset_id":"line.n.06"},{"index":7,"sense_key":"line%1:15:02::","synset_id":"line.n.07"},{"index":8,"sense_key":"line%1:09:01::","synset_id":"argumentation.n.02"},{"index":9,"sense_key":"line%1:06:01::","synset_id":"cable.n.02"},{"index":10,"sense_key":"line%1:14:02::","synset_id":"course.n.02"},{"index":11,"sense_key":"line%1:15:00::","synset_id":"line.n.11"},{"index":12,"sense_key":"line%1:25:01::","synset_id":"wrinkle.n.01"},{"index":13,"sense_key":"line%1:06:06::","synset_id":"pipeline.n.02"},{"index":14,"sense_key":"line%1:06:03::","synset_id":"line.n.14"},{"index":15,"sense_key":"line%1:06:07::","synset_id":"telephone_line.n.02"},{"index":16,"sense_key":"line%1:04:01::","synset_id":"line.n.16"},{"index":17,"sense_key":"line%1:14:00::","synset_id":"lineage.n.01"},{"index":18,"sense_key":"line%1:06:00::","synset_id":"line.n.18"},{"index":19,"sense_key":"line%1:04:00::","synset_id":"occupation.n.01"},{"index":20,"sense_key":"line%1:15:01::","synset_id":"line.n.20"},{"index":21,"sense_key":"line%1:10:05::","synset_id":"channel.n.05"},{"index":22,"sense_key":"line%1:06:08::","synset_id":"line.n.22"},{"index":23,"sense_key":"line%1:06:05::","synset_id":"line.n.23"},{"index":24,"sense_key":"line%1:23:00::","synset_id":"agate_line.n.01"},{"index":25,"sense_key":"line%1:21:00::","synset_id":"credit_line.n.01"},{"index":26,"sense_key":"line%1:10:03::","synset_id":"tune.n.01"},{"index":27,"sense_key":"line%1:10:06::","synset_id":"line.n.27"},{"index":28,"sense_key":"line%1:10:04::","synset_id":"note.n.02"},{"index":29,"sense_key":"line%1:09:00::","synset_id":"line.n.29"},{"index":30,"sense_key":"line%1:06:09::","synset_id":"production_line.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":29,"chosen_sense_key":"line%1:09:00::","call_ids":["senseval3.d001.s044.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":29,"predicted_sense_key":"line%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2577,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0058839999999999995,"input_uncached_usd":0.005154,"input_cached_usd":0.0,"output_usd":0.0007300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9703463329933584}
{"item_id":"senseval3.d001.s044.t004","gold_sense_keys":["reverse%1:24:00::"],"candidates":[{"index":1,"sense_key":"reverse%1:24:00::","synset_id":"reverse.n.01"},{"index":2,"sense_key":"reverse%1:06:00::","synset_id":"reverse.n.02"},{"index":3,"sense_key":"reverse%1:11:01::","synset_id":"reverse.n.03"},{"index":4,"sense_key":"reverse%1:06:02::","synset_id":"reverse.n.04"},{"index":5,"sense_key":"reverse%1:04:00::","synset_id":"reverse.n.05"},{"index":6,"sense_key":"reverse%1:04:01::","synset_id":"reversion.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"reverse%1:24:00::","call_ids":["senseval3.d001.s044.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"reverse%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1096,"cached_input_tokens":0,"output_tokens":99,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003182,"input_uncached_usd":0.002192,"input_cached_usd":0.0,"output_usd":0.00099,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5574152918998152}
{"item_id":"senseval3.d001.s045.t001","gold_sense_keys":["investigation%1:04:00::"],"candidates":[{"index":1,"sense_key":"investigation%1:09:00::","synset_id":"probe.n.01"},{"index":2,"sense_key":"investigation%1:04:00::","synset_id":"investigation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"investigation%1:04:00::","call_ids":["senseval3.d001.s045.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"investigation%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":844,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0022979999999999997,"input_uncached_usd":0.0016879999999999998,"input_cached_usd":0.0,"output_usd":0.0006100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5508439589757472}
{"item_id":"senseval3.d001.s045.t003","gold_sense_keys":["hold%2:42:00::"],"candidates":[{"index":1,"sense_key":"hold%2:42:00::","synset_id":"keep.v.01"},{"index":2,"sense_key":"hold%2:35:00::","synset_id":"hold.v.02"},{"index":3,"sense_key":"hold%2:36:00::","synset_id":"hold.v.03"},{"index":4,"sense_key":"hold%2:40:00::","synset_id":"have.v.01"},{"index":5,"sense_key":"hold%2:31:01::","synset_id":"deem.v.01"},{"index":6,"sense_key":"hold%2:37:00::","synset_id":"harbor.v.01"},{"index":7,"sense_key":"hold%2:35:03::","synset_id":"restrain.v.03"},{"index":8,"sense_key":"hold%2:40:02::","synset_id":"retain.v.03"},{"index":9,"sense_key":"hold%2:40:04::","synset_id":"bear.v.11"},{"index":10,"sense_key":"hold%2:35:01::","synset_id":"hold.v.10"},{"index":11,"sense_key":"hold%2:42:13::","synset_id":"hold.v.11"},{"index":12,"sense_key":"hold%2:42:05::","synset_id":"accommodate.v.04"},{"index":13,"sense_key":"hold%2:42:01::","synset_id":"hold.v.13"},{"index":14,"sense_key":"hold%2:35:02::","synset_id":"hold.v.14"},{"index":15,"sense_key":"hold%2:42:02::","synset_id":"prevail.v.02"},{"index":16,"sense_key":"hold%2:31:08::","synset_id":"hold.v.16"},{"index":17,"sense_key":"hold%2:42:07::","synset_id":"hold.v.17"},{"index":18,"sense_key":"hold%2:42:14::","synset_id":"contain.v.05"},{"index":19,"sense_key":"hold%2:41:00::","synset_id":"reserve.v.04"},{"index":20,"sense_key":"hold%2:33:05::","synset_id":"defend.v.03"},{"index":21,"sense_key":"hold%2:32:13::","synset_id":"oblige.v.02"},{"index":22,"sense_key":"hold%2:31:04::","synset_id":"hold.v.22"},{"index":23,"sense_key":"hold%2:31:10::","synset_id":"hold.v.23"},{"index":24,"sense_key":"hold%2:42:08::","synset_id":"defy.v.01"},{"index":25,"sense_key":"hold%2:42:04::","synset_id":"apply.v.02"},{"index":26,"sense_key":"hold%2:42:12::","synset_id":"hold.v.26"},{"index":27,"sense_key":"hold%2:41:15::","synset_id":"control.v.02"},{"index":28,"sense_key":"hold%2:41:13::","synset_id":"hold.v.28"},{"index":29,"sense_key":"hold%2:41:14::","synset_id":"hold.v.29"},{"index":30,"sense_key":"hold%2:38:05::","synset_id":"halt.v.01"},{"index":31,"sense_key":"hold%2:35:09::","synset_id":"hold.v.31"},{"index":32,"sense_key":"hold%2:34:00::","synset_id":"carry.v.33"},{"index":33,"sense_key":"hold%2:33:00::","synset_id":"hold.v.33"},{"index":34,"sense_key":"hold%2:32:11::","synset_id":"declare.v.04"},{"index":35,"sense_key":"hold%2:32:15::","synset_id":"agree.v.01"},{"index":36,"sense_key":"hold%2:29:04::","synset_id":"hold.v.36"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"hold%2:42:00::","call_ids":["senseval3.d001.s045.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"hold%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":3067,"cached_input_tokens":0,"output_tokens":88,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.007013999999999999,"input_uncached_usd":0.006134,"input_cached_usd":0.0,"output_usd":0.00088,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9948139580665156}
{"item_id":"senseval3.d001.s045.t004","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":2,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":3,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"other%3:00:00::","call_ids":["senseval3.d001.s045.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":959,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002038,"input_uncached_usd":0.001918,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9538034999277443}
{"item_id":"senseval3.d001.s045.t005","gold_sense_keys":["factor%1:11:00::"],"candidates":[{"index":1,"sense_key":"factor%1:11:00::","synset_id":"factor.n.01"},{"index":2,"sense_key":"factor%1:09:00::","synset_id":"component.n.01"},{"index":3,"sense_key":"factor%1:23:00::","synset_id":"divisor.n.01"},{"index":4,"sense_key":"factor%1:18:00::","synset_id":"agent.n.04"},{"index":5,"sense_key":"factor%1:23:01::","synset_id":"factor.n.05"},{"index":6,"sense_key":"factor%1:09:01::","synset_id":"factor.n.06"},{"index":7,"sense_key":"factor%1:08:00::","synset_id":"gene.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"factor%1:11:00::","call_ids":["senseval3.d001.s045.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"factor%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1159,"cached_input_tokens":0,"output_tokens":172,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004038,"input_uncached_usd":0.0023179999999999997,"input_cached_usd":0.0,"output_usd":0.0017200000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.275530625018291}
{"item_id":"senseval3.d001.s045.t006","gold_sense_keys":["constant%5:00:00:invariable:00"],"candidates":[{"index":1,"sense_key":"constant%5:00:00:invariable:00","synset_id":"changeless.s.01"},{"index":2,"sense_key":"constant%3:00:00::","synset_id":"constant.a.02"},{"index":3,"sense_key":"constant%5:00:00:continuous:01","synset_id":"ceaseless.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"constant%5:00:00:invariable:00","call_ids":["senseval3.d001.s045.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"constant%5:00:00:invariable:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":963,"cached_input_tokens":0,"output_tokens":48,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002406,"input_uncached_usd":0.001926,"input_cached_usd":0.0,"output_usd":0.00048000000000000007,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2870055419625714}
{"item_id":"senseval3.d001.s045.t007","gold_sense_keys":["difference%1:23:00::"],"candidates":[{"index":1,"sense_key":"difference%1:07:00::","synset_id":"difference.n.01"},{"index":2,"sense_key":"difference%1:11:00::","synset_id":"deviation.n.01"},{"index":3,"sense_key":"difference%1:10:00::","synset_id":"dispute.n.01"},{"index":4,"sense_key":"difference%1:24:00::","synset_id":"difference.n.04"},{"index":5,"sense_key":"difference%1:23:00::","synset_id":"remainder.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"difference%1:23:00::","call_ids":["senseval3.d001.s045.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"difference%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1027,"cached_input_tokens":0,"output_tokens":187,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003924,"input_uncached_usd":0.002054,"input_cached_usd":0.0,"output_usd":0.0018700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.245269624982029}
{"item_id":"senseval3.d001.s045.t008","gold_sense_keys":["state%1:15:01::"],"candidates":[{"index":1,"sense_key":"state%1:15:01::","synset_id":"state.n.01"},{"index":2,"sense_key":"state%1:03:00::","synset_id":"state.n.02"},{"index":3,"sense_key":"state%1:14:01::","synset_id":"state.n.03"},{"index":4,"sense_key":"state%1:14:00::","synset_id":"state.n.04"},{"index":5,"sense_key":"state%1:26:02::","synset_id":"state_of_matter.n.01"},{"index":6,"sense_key":"state%1:26:01::","synset_id":"state.n.06"},{"index":7,"sense_key":"state%1:15:00::","synset_id":"country.n.02"},{"index":8,"sense_key":"state%1:14:02::","synset_id":"department_of_state.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"state%1:15:01::","call_ids":["senseval3.d001.s045.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"state%1:15:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1257,"cached_input_tokens":0,"output_tokens":85,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003364,"input_uncached_usd":0.0025139999999999997,"input_cached_usd":0.0,"output_usd":0.0008500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.376000874908641}
{"item_id":"senseval3.d001.s045.t009","gold_sense_keys":["major%3:00:01::","major%3:00:06::"],"candidates":[{"index":1,"sense_key":"major%3:00:06::","synset_id":"major.a.01"},{"index":2,"sense_key":"major%3:00:02::","synset_id":"major.a.02"},{"index":3,"sense_key":"major%3:00:01::","synset_id":"major.a.03"},{"index":4,"sense_key":"major%3:00:05::","synset_id":"major.a.04"},{"index":5,"sense_key":"major%3:00:04::","synset_id":"major.a.05"},{"index":6,"sense_key":"major%3:00:07::","synset_id":"major.a.06"},{"index":7,"sense_key":"major%3:00:03::","synset_id":"major.a.07"},{"index":8,"sense_key":"major%5:00:00:senior:00","synset_id":"major.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"major%3:00:06::","call_ids":["senseval3.d001.s045.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"major%3:00:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1077,"cached_input_tokens":0,"output_tokens":164,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0037940000000000005,"input_uncached_usd":0.002154,"input_cached_usd":0.0,"output_usd":0.0016400000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2226534159854054}
{"item_id":"senseval3.d001.s045.t010","gold_sense_keys":["party%1:14:01::"],"candidates":[{"index":1,"sense_key":"party%1:14:01::","synset_id":"party.n.01"},{"index":2,"sense_key":"party%1:14:00::","synset_id":"party.n.02"},{"index":3,"sense_key":"party%1:14:02::","synset_id":"party.n.03"},{"index":4,"sense_key":"party%1:11:00::","synset_id":"party.n.04"},{"index":5,"sense_key":"party%1:18:00::","synset_id":"party.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"party%1:14:01::","call_ids":["senseval3.d001.s045.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"party%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1006,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0021319999999999998,"input_uncached_usd":0.002012,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5231796250445768}
{"item_id":"senseval3.d001.s045.t011","gold_sense_keys":["vote%1:09:00::"],"candidates":[{"index":1,"sense_key":"vote%1:04:00::","synset_id":"vote.n.01"},{"index":2,"sense_key":"vote%1:04:01::","synset_id":"vote.n.02"},{"index":3,"sense_key":"vote%1:07:00::","synset_id":"right_to_vote.n.01"},{"index":4,"sense_key":"vote%1:14:00::","synset_id":"vote.n.04"},{"index":5,"sense_key":"vote%1:09:00::","synset_id":"vote.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"vote%1:09:00::","call_ids":["senseval3.d001.s045.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"vote%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1055,"cached_input_tokens":0,"output_tokens":262,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00473,"input_uncached_usd":0.00211,"input_cached_usd":0.0,"output_usd":0.0026200000000000004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.675125041976571}
{"item_id":"senseval3.d001.s045.t012","gold_sense_keys":["go%2:42:06::"],"candidates":[{"index":1,"sense_key":"go%2:38:00::","synset_id":"travel.v.01"},{"index":2,"sense_key":"go%2:41:00::","synset_id":"go.v.02"},{"index":3,"sense_key":"go%2:38:01::","synset_id":"go.v.03"},{"index":4,"sense_key":"go%2:30:04::","synset_id":"become.v.01"},{"index":5,"sense_key":"go%2:42:06::","synset_id":"go.v.05"},{"index":6,"sense_key":"go%2:42:02::","synset_id":"run.v.05"},{"index":7,"sense_key":"go%2:42:00::","synset_id":"run.v.03"},{"index":8,"sense_key":"go%2:30:01::","synset_id":"proceed.v.04"},{"index":9,"sense_key":"go%2:30:08::","synset_id":"go.v.09"},{"index":10,"sense_key":"go%2:42:07::","synset_id":"go.v.10"},{"index":11,"sense_key":"go%2:39:00::","synset_id":"sound.v.02"},{"index":12,"sense_key":"go%2:35:00::","synset_id":"function.v.01"},{"index":13,"sense_key":"go%2:34:00::","synset_id":"run_low.v.01"},{"index":14,"sense_key":"go%2:30:02::","synset_id":"move.v.13"},{"index":15,"sense_key":"go%2:42:14::","synset_id":"survive.v.01"},{"index":16,"sense_key":"go%2:42:03::","synset_id":"go.v.16"},{"index":17,"sense_key":"go%2:30:00::","synset_id":"die.v.01"},{"index":18,"sense_key":"go%2:42:04::","synset_id":"belong.v.03"},{"index":19,"sense_key":"go%2:42:10::","synset_id":"go.v.19"},{"index":20,"sense_key":"go%2:38:09::","synset_id":"start.v.09"},{"index":21,"sense_key":"go%2:33:00::","synset_id":"move.v.15"},{"index":22,"sense_key":"go%2:42:13::","synset_id":"go.v.22"},{"index":23,"sense_key":"go%2:42:12::","synset_id":"go.v.23"},{"index":24,"sense_key":"go%2:42:11::","synset_id":"blend.v.02"},{"index":25,"sense_key":"go%2:42:08::","synset_id":"go.v.25"},{"index":26,"sense_key":"go%2:42:05::","synset_id":"fit.v.02"},{"index":27,"sense_key":"go%2:35:13::","synset_id":"rifle.v.02"},{"index":28,"sense_key":"go%2:34:09::","synset_id":"go.v.28"},{"index":29,"sense_key":"go%2:31:12::","synset_id":"plump.v.04"},{"index":30,"sense_key":"go%2:30:03::","synset_id":"fail.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"go%2:42:06::","call_ids":["senseval3.d001.s045.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"go%2:42:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2627,"cached_input_tokens":0,"output_tokens":89,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.006144,"input_uncached_usd":0.005254,"input_cached_usd":0.0,"output_usd":0.0008900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9808530409354717}
{"item_id":"senseval3.d001.s045.t013","gold_sense_keys":["republican%3:01:01::"],"candidates":[{"index":1,"sense_key":"republican%3:01:01::","synset_id":"republican.a.01"},{"index":2,"sense_key":"republican%5:00:00:democratic:00","synset_id":"republican.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"republican%3:01:01::","call_ids":["senseval3.d001.s045.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"republican%3:01:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":913,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001946,"input_uncached_usd":0.001826,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8233497499022633}
{"item_id":"senseval3.d001.s045.t015","gold_sense_keys":["candidate%1:18:01::"],"candidates":[{"index":1,"sense_key":"candidate%1:18:01::","synset_id":"campaigner.n.01"},{"index":2,"sense_key":"candidate%1:18:00::","synset_id":"candidate.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"candidate%1:18:01::","call_ids":["senseval3.d001.s045.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"candidate%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":828,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001776,"input_uncached_usd":0.001656,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0395185829838738}
{"item_id":"senseval3.d001.s045.t016","gold_sense_keys":["republican%3:01:01::"],"candidates":[{"index":1,"sense_key":"republican%3:01:01::","synset_id":"republican.a.01"},{"index":2,"sense_key":"republican%5:00:00:democratic:00","synset_id":"republican.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"republican%3:01:01::","call_ids":["senseval3.d001.s045.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"republican%3:01:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":913,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001946,"input_uncached_usd":0.001826,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.7243344999151304}
{"item_id":"senseval3.d001.s045.t017","gold_sense_keys":["share%1:04:00::"],"candidates":[{"index":1,"sense_key":"share%1:21:00::","synset_id":"share.n.01"},{"index":2,"sense_key":"share%1:21:01::","synset_id":"share.n.02"},{"index":3,"sense_key":"share%1:04:00::","synset_id":"parcel.n.02"},{"index":4,"sense_key":"share%1:04:01::","synset_id":"contribution.n.01"},{"index":5,"sense_key":"share%1:06:00::","synset_id":"plowshare.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"share%1:21:00::","call_ids":["senseval3.d001.s045.t017__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"share%1:21:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1069,"cached_input_tokens":0,"output_tokens":222,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004358,"input_uncached_usd":0.0021379999999999997,"input_cached_usd":0.0,"output_usd":0.00222,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.4372830000938848}
{"item_id":"senseval3.d001.s045.t019","gold_sense_keys":["house%1:14:02::"],"candidates":[{"index":1,"sense_key":"house%1:06:00::","synset_id":"house.n.01"},{"index":2,"sense_key":"house%1:14:01::","synset_id":"firm.n.01"},{"index":3,"sense_key":"house%1:14:04::","synset_id":"house.n.03"},{"index":4,"sense_key":"house%1:14:03::","synset_id":"house.n.04"},{"index":5,"sense_key":"house%1:14:02::","synset_id":"house.n.05"},{"index":6,"sense_key":"house%1:14:00::","synset_id":"house.n.06"},{"index":7,"sense_key":"house%1:04:00::","synset_id":"house.n.07"},{"index":8,"sense_key":"house%1:15:00::","synset_id":"sign_of_the_zodiac.n.01"},{"index":9,"sense_key":"house%1:14:06::","synset_id":"house.n.09"},{"index":10,"sense_key":"house%1:14:05::","synset_id":"family.n.01"},{"index":11,"sense_key":"house%1:06:02::","synset_id":"theater.n.01"},{"index":12,"sense_key":"house%1:06:01::","synset_id":"house.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"house%1:14:02::","call_ids":["senseval3.d001.s045.t019__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"house%1:14:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1372,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002864,"input_uncached_usd":0.002744,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1788191669620574}
{"item_id":"senseval3.d001.s045.t020","gold_sense_keys":["positive%5:00:00:plus:00"],"candidates":[{"index":1,"sense_key":"positive%3:00:01::","synset_id":"positive.a.01"},{"index":2,"sense_key":"positive%5:00:00:certain:02","synset_id":"convinced.s.01"},{"index":3,"sense_key":"positive%5:00:00:advantageous:00","synset_id":"plus.s.01"},{"index":4,"sense_key":"positive%3:00:04::","synset_id":"positive.a.04"},{"index":5,"sense_key":"positive%5:00:00:formal:01","synset_id":"positive.s.03"},{"index":6,"sense_key":"positive%5:00:00:undeniable:00","synset_id":"incontrovertible.s.01"},{"index":7,"sense_key":"positive%3:01:00::","synset_id":"positivist.a.01"},{"index":8,"sense_key":"positive%3:00:05::","synset_id":"positive.a.08"},{"index":9,"sense_key":"positive%5:00:00:plus:00","synset_id":"positive.s.05"},{"index":10,"sense_key":"positive%5:00:00:charged:00","synset_id":"positive.s.06"},{"index":11,"sense_key":"positive%5:00:00:confident:00","synset_id":"cocksure.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"positive%3:00:05::","call_ids":["senseval3.d001.s045.t020__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"positive%3:00:05::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1443,"cached_input_tokens":0,"output_tokens":131,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004196,"input_uncached_usd":0.0028859999999999997,"input_cached_usd":0.0,"output_usd":0.0013100000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.4318310419330373}
{"item_id":"senseval3.d001.s045.t021","gold_sense_keys":["function%1:24:00::"],"candidates":[{"index":1,"sense_key":"function%1:24:00::","synset_id":"function.n.01"},{"index":2,"sense_key":"function%1:07:00::","synset_id":"function.n.02"},{"index":3,"sense_key":"function%1:04:00::","synset_id":"function.n.03"},{"index":4,"sense_key":"function%1:24:01::","synset_id":"function.n.04"},{"index":5,"sense_key":"function%1:14:00::","synset_id":"function.n.05"},{"index":6,"sense_key":"function%1:11:00::","synset_id":"affair.n.03"},{"index":7,"sense_key":"function%1:10:00::","synset_id":"routine.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"function%1:24:01::","call_ids":["senseval3.d001.s045.t021__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"function%1:24:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1190,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0024999999999999996,"input_uncached_usd":0.0023799999999999997,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4322374999755993}
{"item_id":"senseval3.d001.s045.t022","gold_sense_keys":["state%1:15:01::"],"candidates":[{"index":1,"sense_key":"state%1:15:01::","synset_id":"state.n.01"},{"index":2,"sense_key":"state%1:03:00::","synset_id":"state.n.02"},{"index":3,"sense_key":"state%1:14:01::","synset_id":"state.n.03"},{"index":4,"sense_key":"state%1:14:00::","synset_id":"state.n.04"},{"index":5,"sense_key":"state%1:26:02::","synset_id":"state_of_matter.n.01"},{"index":6,"sense_key":"state%1:26:01::","synset_id":"state.n.06"},{"index":7,"sense_key":"state%1:15:00::","synset_id":"country.n.02"},{"index":8,"sense_key":"state%1:14:02::","synset_id":"department_of_state.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"state%1:15:01::","call_ids":["senseval3.d001.s045.t022__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"state%1:15:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1256,"cached_input_tokens":0,"output_tokens":147,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003982,"input_uncached_usd":0.002512,"input_cached_usd":0.0,"output_usd":0.0014700000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.539588832994923}
{"item_id":"senseval3.d001.s046.t001","gold_sense_keys":["federal%5:00:00:national:01"],"candidates":[{"index":1,"sense_key":"federal%5:00:00:national:01","synset_id":"federal.s.01"},{"index":2,"sense_key":"federal%3:01:02::","synset_id":"federal.a.02"},{"index":3,"sense_key":"federal%5:00:00:northern:02","synset_id":"union.s.01"},{"index":4,"sense_key":"federal%3:00:00::","synset_id":"federal.a.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"federal%5:00:00:national:01","call_ids":["senseval3.d001.s046.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"federal%5:00:00:national:01","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":917,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0025139999999999997,"input_uncached_usd":0.001834,"input_cached_usd":0.0,"output_usd":0.00068,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8055051660630852}
{"item_id":"senseval3.d001.s046.t004","gold_sense_keys":["level%1:09:00::"],"candidates":[{"index":1,"sense_key":"level%1:07:00::","synset_id":"degree.n.01"},{"index":2,"sense_key":"level%1:26:01::","synset_id":"grade.n.02"},{"index":3,"sense_key":"level%1:26:00::","synset_id":"degree.n.02"},{"index":4,"sense_key":"level%1:07:01::","synset_id":"level.n.04"},{"index":5,"sense_key":"level%1:06:00::","synset_id":"level.n.05"},{"index":6,"sense_key":"level%1:06:03::","synset_id":"horizontal_surface.n.01"},{"index":7,"sense_key":"level%1:09:00::","synset_id":"level.n.07"},{"index":8,"sense_key":"level%1:06:01::","synset_id":"floor.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"level%1:26:01::","call_ids":["senseval3.d001.s046.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"level%1:26:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1157,"cached_input_tokens":0,"output_tokens":128,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003594,"input_uncached_usd":0.002314,"input_cached_usd":0.0,"output_usd":0.00128,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.05290833406616}
{"item_id":"senseval3.d001.s046.t005","gold_sense_keys":["part%1:24:00::"],"candidates":[{"index":1,"sense_key":"part%1:24:00::","synset_id":"part.n.01"},{"index":2,"sense_key":"part%1:06:00::","synset_id":"part.n.02"},{"index":3,"sense_key":"part%1:17:00::","synset_id":"part.n.03"},{"index":4,"sense_key":"part%1:09:02::","synset_id":"part.n.04"},{"index":5,"sense_key":"part%1:15:00::","synset_id":"region.n.01"},{"index":6,"sense_key":"part%1:04:01::","synset_id":"function.n.03"},{"index":7,"sense_key":"part%1:09:01::","synset_id":"character.n.04"},{"index":8,"sense_key":"part%1:21:00::","synset_id":"share.n.01"},{"index":9,"sense_key":"part%1:09:00::","synset_id":"part.n.09"},{"index":10,"sense_key":"part%1:08:01::","synset_id":"part.n.10"},{"index":11,"sense_key":"part%1:10:00::","synset_id":"part.n.11"},{"index":12,"sense_key":"part%1:04:03::","synset_id":"contribution.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"part%1:24:00::","call_ids":["senseval3.d001.s046.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"part%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1444,"cached_input_tokens":0,"output_tokens":131,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004198,"input_uncached_usd":0.002888,"input_cached_usd":0.0,"output_usd":0.0013100000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.019104624981992}
{"item_id":"senseval3.d001.s046.t006","gold_sense_keys":["irrational%3:00:00::"],"candidates":[{"index":1,"sense_key":"irrational%3:00:00::","synset_id":"irrational.a.01"},{"index":2,"sense_key":"irrational%3:01:00::","synset_id":"irrational.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"irrational%3:00:00::","call_ids":["senseval3.d001.s046.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"irrational%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":732,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0015840000000000001,"input_uncached_usd":0.001464,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1141455000033602}
{"item_id":"senseval3.d001.s046.t007","gold_sense_keys":["behavior%1:04:00::"],"candidates":[{"index":1,"sense_key":"behavior%1:04:00::","synset_id":"behavior.n.01"},{"index":2,"sense_key":"behavior%1:26:00::","synset_id":"behavior.n.02"},{"index":3,"sense_key":"behavior%1:07:00::","synset_id":"demeanor.n.01"},{"index":4,"sense_key":"behavior%1:04:01::","synset_id":"behavior.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"behavior%1:04:00::","call_ids":["senseval3.d001.s046.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"behavior%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":900,"cached_input_tokens":0,"output_tokens":237,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00417,"input_uncached_usd":0.0018,"input_cached_usd":0.0,"output_usd":0.00237,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.376059541013092}
{"item_id":"senseval3.d001.s046.t009","gold_sense_keys":["display%2:39:00::"],"candidates":[{"index":1,"sense_key":"display%2:39:00::","synset_id":"expose.v.03"},{"index":2,"sense_key":"display%2:41:00::","synset_id":"display.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"display%2:39:00::","call_ids":["senseval3.d001.s046.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"display%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":765,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00165,"input_uncached_usd":0.00153,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1756758749252185}
{"item_id":"senseval3.d001.s046.t012","gold_sense_keys":["rational%3:00:00::"],"candidates":[{"index":1,"sense_key":"rational%3:00:00::","synset_id":"rational.a.01"},{"index":2,"sense_key":"rational%5:00:00:mental:00","synset_id":"intellectual.s.01"},{"index":3,"sense_key":"rational%3:01:00::","synset_id":"rational.a.03"},{"index":4,"sense_key":"rational%5:00:00:intellectual:00","synset_id":"rational.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"rational%3:00:00::","call_ids":["senseval3.d001.s046.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"rational%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":877,"cached_input_tokens":0,"output_tokens":92,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002674,"input_uncached_usd":0.001754,"input_cached_usd":0.0,"output_usd":0.00092,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3861992920283228}
{"item_id":"senseval3.d002.s000.t000","gold_sense_keys":["long%3:00:02::"],"candidates":[{"index":1,"sense_key":"long%3:00:02::","synset_id":"long.a.01"},{"index":2,"sense_key":"long%3:00:01::","synset_id":"long.a.02"},{"index":3,"sense_key":"long%5:00:00:tall:00","synset_id":"long.s.01"},{"index":4,"sense_key":"long%3:00:00::","synset_id":"retentive.a.01"},{"index":5,"sense_key":"long%3:00:05::","synset_id":"long.a.05"},{"index":6,"sense_key":"long%3:00:04::","synset_id":"long.a.06"},{"index":7,"sense_key":"long%5:00:00:unsound:00","synset_id":"long.s.02"},{"index":8,"sense_key":"long%5:00:00:provident:00","synset_id":"farseeing.s.02"},{"index":9,"sense_key":"long%5:00:00:abundant:00","synset_id":"long.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"long%3:00:02::","call_ids":["senseval3.d002.s000.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"long%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":870,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00186,"input_uncached_usd":0.00174,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5756924169836566}
{"item_id":"senseval3.d002.s000.t002","gold_sense_keys":["night%1:28:00::"],"candidates":[{"index":1,"sense_key":"night%1:28:00::","synset_id":"night.n.01"},{"index":2,"sense_key":"night%1:28:01::","synset_id":"night.n.02"},{"index":3,"sense_key":"night%1:28:04::","synset_id":"night.n.03"},{"index":4,"sense_key":"night%1:28:02::","synset_id":"night.n.04"},{"index":5,"sense_key":"night%1:26:00::","synset_id":"night.n.05"},{"index":6,"sense_key":"night%1:28:05::","synset_id":"night.n.06"},{"index":7,"sense_key":"night%1:28:03::","synset_id":"night.n.07"},{"index":8,"sense_key":"night%1:18:00::","synset_id":"nox.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"night%1:28:02::","call_ids":["senseval3.d002.s000.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"night%1:28:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":649,"cached_input_tokens":0,"output_tokens":168,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0029779999999999997,"input_uncached_usd":0.001298,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.574435583082959}
{"item_id":"senseval3.d002.s000.t003","gold_sense_keys":["devastate%2:30:00::"],"candidates":[{"index":1,"sense_key":"devastate%2:30:00::","synset_id":"lay_waste_to.v.01"},{"index":2,"sense_key":"devastate%2:30:01::","synset_id":"devastate.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"devastate%2:30:00::","call_ids":["senseval3.d002.s000.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"devastate%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":428,"cached_input_tokens":0,"output_tokens":49,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001346,"input_uncached_usd":0.000856,"input_cached_usd":0.0,"output_usd":0.0004900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1906928749522194}
{"item_id":"senseval3.d002.s000.t004","gold_sense_keys":["earthquake%1:11:00::"],"candidates":[{"index":1,"sense_key":"earthquake%1:11:00::","synset_id":"earthquake.n.01"},{"index":2,"sense_key":"earthquake%1:26:00::","synset_id":"earthquake.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"earthquake%1:11:00::","call_ids":["senseval3.d002.s000.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"earthquake%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":428,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.000976,"input_uncached_usd":0.000856,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0481917499564588}
{"item_id":"senseval3.d002.s000.t005","gold_sense_keys":["resident%1:18:00::"],"candidates":[{"index":1,"sense_key":"resident%1:18:00::","synset_id":"resident.n.01"},{"index":2,"sense_key":"resident%1:18:01::","synset_id":"house_physician.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"resident%1:18:00::","call_ids":["senseval3.d002.s000.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"resident%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":450,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00102,"input_uncached_usd":0.0009,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9036219579866156}
{"item_id":"senseval3.d002.s000.t006","gold_sense_keys":["search%2:35:00::","search%2:39:00::"],"candidates":[{"index":1,"sense_key":"search%2:35:00::","synset_id":"search.v.01"},{"index":2,"sense_key":"search%2:39:00::","synset_id":"search.v.02"},{"index":3,"sense_key":"search%2:31:00::","synset_id":"research.v.02"},{"index":4,"sense_key":"search%2:35:01::","synset_id":"search.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"search%2:35:00::","call_ids":["senseval3.d002.s000.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"search%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":555,"cached_input_tokens":0,"output_tokens":196,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0030700000000000002,"input_uncached_usd":0.0011099999999999999,"input_cached_usd":0.0,"output_usd":0.0019600000000000004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.5810354999266565}
{"item_id":"senseval3.d002.s000.t007","gold_sense_keys":["comfort%1:04:00::"],"candidates":[{"index":1,"sense_key":"comfort%1:26:00::","synset_id":"comfort.n.01"},{"index":2,"sense_key":"comfort%1:12:00::","synset_id":"comfort.n.02"},{"index":3,"sense_key":"comfort%1:04:00::","synset_id":"consolation.n.02"},{"index":4,"sense_key":"comfort%1:26:01::","synset_id":"ease.n.02"},{"index":5,"sense_key":"comfort%1:26:02::","synset_id":"comfort.n.05"},{"index":6,"sense_key":"comfort%1:06:00::","synset_id":"quilt.n.01"},{"index":7,"sense_key":"comfort%1:04:01::","synset_id":"comfort.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"comfort%1:04:00::","call_ids":["senseval3.d002.s000.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"comfort%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":748,"cached_input_tokens":0,"output_tokens":112,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0026160000000000003,"input_uncached_usd":0.001496,"input_cached_usd":0.0,"output_usd":0.0011200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7232517499942333}
{"item_id":"senseval3.d002.s000.t008","gold_sense_keys":["solace%1:12:00::","solace%1:26:00::"],"candidates":[{"index":1,"sense_key":"solace%1:12:00::","synset_id":"consolation.n.01"},{"index":2,"sense_key":"solace%1:26:00::","synset_id":"solace.n.02"},{"index":3,"sense_key":"solace%1:04:00::","synset_id":"consolation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"solace%1:26:00::","call_ids":["senseval3.d002.s000.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"solace%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":463,"cached_input_tokens":0,"output_tokens":288,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0038060000000000004,"input_uncached_usd":0.000926,"input_cached_usd":0.0,"output_usd":0.00288,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.3588435830315575}
{"item_id":"senseval3.d002.s001.t001","gold_sense_keys":["screen%1:06:05::"],"candidates":[{"index":1,"sense_key":"screen%1:06:06::","synset_id":"screen.n.01"},{"index":2,"sense_key":"screen%1:06:04::","synset_id":"blind.n.03"},{"index":3,"sense_key":"screen%1:06:05::","synset_id":"screen.n.03"},{"index":4,"sense_key":"screen%1:06:01::","synset_id":"screen.n.04"},{"index":5,"sense_key":"screen%1:06:00::","synset_id":"screen.n.05"},{"index":6,"sense_key":"screen%1:14:00::","synset_id":"filmdom.n.01"},{"index":7,"sense_key":"screen%1:06:02::","synset_id":"sieve.n.01"},{"index":8,"sense_key":"screen%1:06:07::","synset_id":"screen_door.n.01"},{"index":9,"sense_key":"screen%1:06:03::","synset_id":"screen.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"screen%1:06:05::","call_ids":["senseval3.d002.s001.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"screen%1:06:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":922,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001964,"input_uncached_usd":0.001844,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2585357079515234}
{"item_id":"senseval3.d002.s002.t001","gold_sense_keys":["make%2:38:10::"],"candidates":[{"index":1,"sense_key":"make%2:41:00::","synset_id":"make.v.01"},{"index":2,"sense_key":"make%2:30:00::","synset_id":"make.v.02"},{"index":3,"sense_key":"make%2:36:00::","synset_id":"make.v.03"},{"index":4,"sense_key":"make%2:32:00::","synset_id":"induce.v.02"},{"index":5,"sense_key":"make%2:36:08::","synset_id":"cause.v.01"},{"index":6,"sense_key":"make%2:36:01::","synset_id":"produce.v.02"},{"index":7,"sense_key":"make%2:31:00::","synset_id":"draw.v.04"},{"index":8,"sense_key":"make%2:36:15::","synset_id":"make.v.08"},{"index":9,"sense_key":"make%2:36:09::","synset_id":"create.v.05"},{"index":10,"sense_key":"make%2:40:01::","synset_id":"gain.v.08"},{"index":11,"sense_key":"make%2:36:05::","synset_id":"do.v.08"},{"index":12,"sense_key":"make%2:42:05::","synset_id":"form.v.02"},{"index":13,"sense_key":"make%2:38:02::","synset_id":"reach.v.07"},{"index":14,"sense_key":"make%2:42:00::","synset_id":"make.v.14"},{"index":15,"sense_key":"make%2:36:11::","synset_id":"make.v.15"},{"index":16,"sense_key":"make%2:36:12::","synset_id":"make.v.16"},{"index":17,"sense_key":"make%2:36:13::","synset_id":"construct.v.01"},{"index":18,"sense_key":"make%2:30:15::","synset_id":"make.v.18"},{"index":19,"sense_key":"make%2:29:08::","synset_id":"make.v.19"},{"index":20,"sense_key":"make%2:41:03::","synset_id":"name.v.03"},{"index":21,"sense_key":"make%2:40:02::","synset_id":"have.v.17"},{"index":22,"sense_key":"make%2:38:00::","synset_id":"reach.v.01"},{"index":23,"sense_key":"make%2:31:13::","synset_id":"lay_down.v.01"},{"index":24,"sense_key":"make%2:41:13::","synset_id":"make.v.24"},{"index":25,"sense_key":"make%2:41:01::","synset_id":"make.v.25"},{"index":26,"sense_key":"make%2:36:04::","synset_id":"hold.v.03"},{"index":27,"sense_key":"make%2:30:02::","synset_id":"make.v.27"},{"index":28,"sense_key":"make%2:38:05::","synset_id":"take.v.27"},{"index":29,"sense_key":"make%2:29:00::","synset_id":"stool.v.04"},{"index":30,"sense_key":"make%2:42:09::","synset_id":"make.v.30"},{"index":31,"sense_key":"make%2:42:13::","synset_id":"make.v.31"},{"index":32,"sense_key":"make%2:42:12::","synset_id":"make.v.32"},{"index":33,"sense_key":"make%2:42:11::","synset_id":"make.v.33"},{"index":34,"sense_key":"make%2:42:15::","synset_id":"make.v.34"},{"index":35,"sense_key":"make%2:39:12::","synset_id":"make.v.35"},{"index":36,"sense_key":"make%2:38:10::","synset_id":"make.v.36"},{"index":37,"sense_key":"make%2:38:15::","synset_id":"make.v.37"},{"index":38,"sense_key":"make%2:36:10::","synset_id":"make.v.38"},{"index":39,"sense_key":"make%2:36:07::","synset_id":"cook.v.02"},{"index":40,"sense_key":"make%2:35:13::","synset_id":"seduce.v.01"},{"index":41,"sense_key":"make%2:32:14::","synset_id":"make.v.41"},{"index":42,"sense_key":"make%2:32:09::","synset_id":"make.v.42"},{"index":43,"sense_key":"make%2:31:10::","synset_id":"make.v.43"},{"index":44,"sense_key":"make%2:31:12::","synset_id":"make.v.44"},{"index":45,"sense_key":"make%2:30:10::","synset_id":"make.v.45"},{"index":46,"sense_key":"make%2:30:14::","synset_id":"make.v.46"},{"index":47,"sense_key":"make%2:30:12::","synset_id":"make.v.47"},{"index":48,"sense_key":"make%2:29:06::","synset_id":"make.v.48"},{"index":49,"sense_key":"make%2:29:01::","synset_id":"make.v.49"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":36,"chosen_sense_key":"make%2:38:10::","call_ids":["senseval3.d002.s002.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":36,"predicted_sense_key":"make%2:38:10::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":3063,"cached_input_tokens":0,"output_tokens":92,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.007045999999999999,"input_uncached_usd":0.0061259999999999995,"input_cached_usd":0.0,"output_usd":0.00092,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0933070000028238}
{"item_id":"senseval3.d002.s002.t003","gold_sense_keys":["computer%1:06:00::"],"candidates":[{"index":1,"sense_key":"computer%1:06:00::","synset_id":"computer.n.01"},{"index":2,"sense_key":"computer%1:18:00::","synset_id":"calculator.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"computer%1:06:00::","call_ids":["senseval3.d002.s002.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"computer%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":523,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0011660000000000002,"input_uncached_usd":0.001046,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2029655410442501}
{"item_id":"senseval3.d002.s002.t007","gold_sense_keys":["link%2:35:00::"],"candidates":[{"index":1,"sense_key":"link%2:31:00::","synset_id":"associate.v.01"},{"index":2,"sense_key":"link%2:35:00::","synset_id":"connect.v.01"},{"index":3,"sense_key":"link%2:42:01::","synset_id":"connect.v.03"},{"index":4,"sense_key":"link%2:35:02::","synset_id":"yoke.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"link%2:35:00::","call_ids":["senseval3.d002.s002.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"link%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":679,"cached_input_tokens":0,"output_tokens":154,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002898,"input_uncached_usd":0.0013579999999999998,"input_cached_usd":0.0,"output_usd":0.0015400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9572352499235421}
{"item_id":"senseval3.d002.s002.t008","gold_sense_keys":["computer%1:06:00::"],"candidates":[{"index":1,"sense_key":"computer%1:06:00::","synset_id":"computer.n.01"},{"index":2,"sense_key":"computer%1:18:00::","synset_id":"calculator.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"computer%1:06:00::","call_ids":["senseval3.d002.s002.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"computer%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":523,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0011660000000000002,"input_uncached_usd":0.001046,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0954505000263453}
{"item_id":"senseval3.d002.s003.t000","gold_sense_keys":["vivid%5:00:00:clear:00","vivid%5:00:00:realistic:00"],"candidates":[{"index":1,"sense_key":"vivid%5:00:00:realistic:00","synset_id":"graphic.s.03"},{"index":2,"sense_key":"vivid%5:00:00:clear:00","synset_id":"vivid.s.02"},{"index":3,"sense_key":"vivid%5:00:00:colorful:00","synset_id":"bright.s.01"},{"index":4,"sense_key":"vivid%5:00:00:saturated:03","synset_id":"intense.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"vivid%5:00:00:realistic:00","call_ids":["senseval3.d002.s003.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"vivid%5:00:00:realistic:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":687,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001944,"input_uncached_usd":0.001374,"input_cached_usd":0.0,"output_usd":0.0005700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2860380839556456}
{"item_id":"senseval3.d002.s003.t002","gold_sense_keys":["come%2:38:02::"],"candidates":[{"index":1,"sense_key":"come%2:38:00::","synset_id":"come.v.01"},{"index":2,"sense_key":"come%2:38:04::","synset_id":"arrive.v.01"},{"index":3,"sense_key":"come%2:30:01::","synset_id":"come.v.03"},{"index":4,"sense_key":"come%2:30:03::","synset_id":"come.v.04"},{"index":5,"sense_key":"come%2:42:04::","synset_id":"come.v.05"},{"index":6,"sense_key":"come%2:30:02::","synset_id":"come.v.06"},{"index":7,"sense_key":"come%2:42:13::","synset_id":"issue_forth.v.01"},{"index":8,"sense_key":"come%2:42:09::","synset_id":"hail.v.02"},{"index":9,"sense_key":"come%2:42:03::","synset_id":"come.v.09"},{"index":10,"sense_key":"come%2:42:05::","synset_id":"come.v.10"},{"index":11,"sense_key":"come%2:38:03::","synset_id":"come.v.11"},{"index":12,"sense_key":"come%2:42:00::","synset_id":"fall.v.04"},{"index":13,"sense_key":"come%2:30:13::","synset_id":"come.v.13"},{"index":14,"sense_key":"come%2:42:12::","synset_id":"total.v.01"},{"index":15,"sense_key":"come%2:42:11::","synset_id":"come.v.15"},{"index":16,"sense_key":"come%2:38:02::","synset_id":"come.v.16"},{"index":17,"sense_key":"come%2:31:14::","synset_id":"occur.v.02"},{"index":18,"sense_key":"come%2:42:15::","synset_id":"derive.v.05"},{"index":19,"sense_key":"come%2:42:02::","synset_id":"do.v.04"},{"index":20,"sense_key":"come%2:39:00::","synset_id":"come.v.20"},{"index":21,"sense_key":"come%2:31:13::","synset_id":"come.v.21"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":16,"chosen_sense_key":"come%2:38:02::","call_ids":["senseval3.d002.s003.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":16,"predicted_sense_key":"come%2:38:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1600,"cached_input_tokens":0,"output_tokens":181,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00501,"input_uncached_usd":0.0031999999999999997,"input_cached_usd":0.0,"output_usd":0.0018100000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0479849580442533}
{"item_id":"senseval3.d002.s003.t004","gold_sense_keys":["electronic%3:01:00::"],"candidates":[{"index":1,"sense_key":"electronic%3:01:00::","synset_id":"electronic.a.01"},{"index":2,"sense_key":"electronic%3:01:01::","synset_id":"electronic.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"electronic%3:01:00::","call_ids":["senseval3.d002.s003.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"electronic%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":535,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00119,"input_uncached_usd":0.00107,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3176066660089418}
{"item_id":"senseval3.d002.s004.t001","gold_sense_keys":["subscriber%1:18:01::"],"candidates":[{"index":1,"sense_key":"subscriber%1:18:02::","synset_id":"subscriber.n.01"},{"index":2,"sense_key":"subscriber%1:18:01::","synset_id":"subscriber.n.02"},{"index":3,"sense_key":"subscriber%1:18:00::","synset_id":"subscriber.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"subscriber%1:18:01::","call_ids":["senseval3.d002.s004.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"subscriber%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":632,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001384,"input_uncached_usd":0.001264,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0340729999588802}
{"item_id":"senseval3.d002.s004.t002","gold_sense_keys":["live%2:42:08::"],"candidates":[{"index":1,"sense_key":"live%2:42:08::","synset_id":"populate.v.01"},{"index":2,"sense_key":"live%2:42:06::","synset_id":"live.v.02"},{"index":3,"sense_key":"live%2:42:04::","synset_id":"survive.v.01"},{"index":4,"sense_key":"live%2:42:01::","synset_id":"exist.v.02"},{"index":5,"sense_key":"live%2:42:00::","synset_id":"be.v.11"},{"index":6,"sense_key":"live%2:31:00::","synset_id":"know.v.05"},{"index":7,"sense_key":"live%2:42:07::","synset_id":"live.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"live%2:42:08::","call_ids":["senseval3.d002.s004.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"live%2:42:08::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":946,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002012,"input_uncached_usd":0.001892,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5198854169575498}
{"item_id":"senseval3.d002.s005.t002","gold_sense_keys":["hour%1:28:00::"],"candidates":[{"index":1,"sense_key":"hour%1:28:00::","synset_id":"hour.n.01"},{"index":2,"sense_key":"hour%1:28:01::","synset_id":"hour.n.02"},{"index":3,"sense_key":"hour%1:28:02::","synset_id":"hour.n.03"},{"index":4,"sense_key":"hour%1:07:00::","synset_id":"hour.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"hour%1:28:00::","call_ids":["senseval3.d002.s005.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"hour%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":698,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001516,"input_uncached_usd":0.0013959999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2268666250165552}
{"item_id":"senseval3.d002.s005.t003","gold_sense_keys":["come_up%2:35:00::"],"candidates":[{"index":1,"sense_key":"come_up%2:36:03::","synset_id":"come_up.v.01"},{"index":2,"sense_key":"come_up%2:30:00::","synset_id":"arise.v.04"},{"index":3,"sense_key":"come_up%2:38:02::","synset_id":"come.v.01"},{"index":4,"sense_key":"come_up%2:38:01::","synset_id":"surface.v.01"},{"index":5,"sense_key":"come_up%2:42:00::","synset_id":"arise.v.02"},{"index":6,"sense_key":"come_up%2:38:03::","synset_id":"rise.v.01"},{"index":7,"sense_key":"come_up%2:42:01::","synset_id":"come_up.v.07"},{"index":8,"sense_key":"come_up%2:35:00::","synset_id":"go_on.v.05"},{"index":9,"sense_key":"come_up%2:40:00::","synset_id":"line_up.v.02"},{"index":10,"sense_key":"come_up%2:38:00::","synset_id":"rise.v.16"},{"index":11,"sense_key":"come_up%2:35:02::","synset_id":"scrape.v.05"},{"index":12,"sense_key":"come_up%2:35:01::","synset_id":"muster.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"come_up%2:35:00::","call_ids":["senseval3.d002.s005.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"come_up%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1236,"cached_input_tokens":0,"output_tokens":79,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0032619999999999997,"input_uncached_usd":0.002472,"input_cached_usd":0.0,"output_usd":0.00079,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8695628750137985}
{"item_id":"senseval3.d002.s005.t004","gold_sense_keys":["teem%2:42:00::"],"candidates":[{"index":1,"sense_key":"teem%2:42:00::","synset_id":"teem.v.01"},{"index":2,"sense_key":"teem%2:38:00::","synset_id":"pour.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"teem%2:42:00::","call_ids":["senseval3.d002.s005.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"teem%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":634,"cached_input_tokens":0,"output_tokens":100,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002268,"input_uncached_usd":0.001268,"input_cached_usd":0.0,"output_usd":0.001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6892347499961033}
{"item_id":"senseval3.d002.s005.t005","gold_sense_keys":["emotional%3:00:00::"],"candidates":[{"index":1,"sense_key":"emotional%3:00:02::","synset_id":"emotional.a.01"},{"index":2,"sense_key":"emotional%3:00:00::","synset_id":"emotional.a.02"},{"index":3,"sense_key":"emotional%3:01:00::","synset_id":"emotional.a.03"},{"index":4,"sense_key":"emotional%5:00:00:agitated:00","synset_id":"aroused.s.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"emotional%3:00:00::","call_ids":["senseval3.d002.s005.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"emotional%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":714,"cached_input_tokens":0,"output_tokens":146,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002888,"input_uncached_usd":0.001428,"input_cached_usd":0.0,"output_usd":0.0014600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.84077470889315}
{"item_id":"senseval3.d002.s005.t007","gold_sense_keys":["report%1:10:00::"],"candidates":[{"index":1,"sense_key":"report%1:10:03::","synset_id":"report.n.01"},{"index":2,"sense_key":"report%1:10:01::","synset_id":"report.n.02"},{"index":3,"sense_key":"report%1:10:00::","synset_id":"report.n.03"},{"index":4,"sense_key":"report%1:11:00::","synset_id":"report.n.04"},{"index":5,"sense_key":"report%1:10:04::","synset_id":"report_card.n.01"},{"index":6,"sense_key":"report%1:10:02::","synset_id":"composition.n.08"},{"index":7,"sense_key":"report%1:09:00::","synset_id":"reputation.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"report%1:10:01::","call_ids":["senseval3.d002.s005.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"report%1:10:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":930,"cached_input_tokens":0,"output_tokens":128,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00314,"input_uncached_usd":0.0018599999999999999,"input_cached_usd":0.0,"output_usd":0.00128,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.293286333908327}
{"item_id":"senseval3.d002.s006.t001","gold_sense_keys":["electronic%3:01:00::"],"candidates":[{"index":1,"sense_key":"electronic%3:01:00::","synset_id":"electronic.a.01"},{"index":2,"sense_key":"electronic%3:01:01::","synset_id":"electronic.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"electronic%3:01:00::","call_ids":["senseval3.d002.s006.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"electronic%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":582,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001284,"input_uncached_usd":0.0011639999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9099585419753566}
{"item_id":"senseval3.d002.s006.t002","gold_sense_keys":["traffic%1:10:00::"],"candidates":[{"index":1,"sense_key":"traffic%1:14:00::","synset_id":"traffic.n.01"},{"index":2,"sense_key":"traffic%1:04:01::","synset_id":"traffic.n.02"},{"index":3,"sense_key":"traffic%1:10:00::","synset_id":"traffic.n.03"},{"index":4,"sense_key":"traffic%1:04:02::","synset_id":"dealings.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"traffic%1:10:00::","call_ids":["senseval3.d002.s006.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"traffic%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":695,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00151,"input_uncached_usd":0.00139,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3422019169665873}
{"item_id":"senseval3.d002.s007.t000","gold_sense_keys":["time%1:28:03::"],"candidates":[{"index":1,"sense_key":"time%1:11:00::","synset_id":"time.n.01"},{"index":2,"sense_key":"time%1:28:05::","synset_id":"time.n.02"},{"index":3,"sense_key":"time%1:28:00::","synset_id":"time.n.03"},{"index":4,"sense_key":"time%1:28:06::","synset_id":"time.n.04"},{"index":5,"sense_key":"time%1:03:00::","synset_id":"time.n.05"},{"index":6,"sense_key":"time%1:11:01::","synset_id":"time.n.06"},{"index":7,"sense_key":"time%1:28:03::","synset_id":"clock_time.n.01"},{"index":8,"sense_key":"time%1:28:04::","synset_id":"fourth_dimension.n.01"},{"index":9,"sense_key":"time%1:07:01::","synset_id":"meter.n.04"},{"index":10,"sense_key":"time%1:28:01::","synset_id":"prison_term.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"time%1:28:03::","call_ids":["senseval3.d002.s007.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"time%1:28:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1037,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0021939999999999998,"input_uncached_usd":0.002074,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4696501670405269}
{"item_id":"senseval3.d002.s007.t001","gold_sense_keys":["time%1:28:03::"],"candidates":[{"index":1,"sense_key":"time%1:11:00::","synset_id":"time.n.01"},{"index":2,"sense_key":"time%1:28:05::","synset_id":"time.n.02"},{"index":3,"sense_key":"time%1:28:00::","synset_id":"time.n.03"},{"index":4,"sense_key":"time%1:28:06::","synset_id":"time.n.04"},{"index":5,"sense_key":"time%1:03:00::","synset_id":"time.n.05"},{"index":6,"sense_key":"time%1:11:01::","synset_id":"time.n.06"},{"index":7,"sense_key":"time%1:28:03::","synset_id":"clock_time.n.01"},{"index":8,"sense_key":"time%1:28:04::","synset_id":"fourth_dimension.n.01"},{"index":9,"sense_key":"time%1:07:01::","synset_id":"meter.n.04"},{"index":10,"sense_key":"time%1:28:01::","synset_id":"prison_term.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"time%1:28:03::","call_ids":["senseval3.d002.s007.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"time%1:28:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1038,"cached_input_tokens":0,"output_tokens":88,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002956,"input_uncached_usd":0.0020759999999999997,"input_cached_usd":0.0,"output_usd":0.00088,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.801393207977526}
{"item_id":"senseval3.d002.s007.t004","gold_sense_keys":["subscriber%1:18:01::"],"candidates":[{"index":1,"sense_key":"subscriber%1:18:02::","synset_id":"subscriber.n.01"},{"index":2,"sense_key":"subscriber%1:18:01::","synset_id":"subscriber.n.02"},{"index":3,"sense_key":"subscriber%1:18:00::","synset_id":"subscriber.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"subscriber%1:18:01::","call_ids":["senseval3.d002.s007.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"subscriber%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":636,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001862,"input_uncached_usd":0.001272,"input_cached_usd":0.0,"output_usd":0.00059,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.346002292004414}
{"item_id":"senseval3.d002.s007.t005","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":2,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":3,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":4,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":5,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":6,"sense_key":"use%2:41:03::","synset_id":"use.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%2:34:01::","call_ids":["senseval3.d002.s007.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":853,"cached_input_tokens":0,"output_tokens":91,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002616,"input_uncached_usd":0.0017059999999999998,"input_cached_usd":0.0,"output_usd":0.0009100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6525768750580028}
{"item_id":"senseval3.d002.s007.t006","gold_sense_keys":["identify%2:32:00::"],"candidates":[{"index":1,"sense_key":"identify%2:31:00::","synset_id":"identify.v.01"},{"index":2,"sense_key":"identify%2:32:00::","synset_id":"name.v.02"},{"index":3,"sense_key":"identify%2:31:04::","synset_id":"identify.v.03"},{"index":4,"sense_key":"identify%2:31:03::","synset_id":"identify.v.04"},{"index":5,"sense_key":"identify%2:31:01::","synset_id":"identify.v.05"},{"index":6,"sense_key":"identify%2:31:02::","synset_id":"identify.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"identify%2:32:00::","call_ids":["senseval3.d002.s007.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"identify%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":830,"cached_input_tokens":0,"output_tokens":78,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0024400000000000003,"input_uncached_usd":0.00166,"input_cached_usd":0.0,"output_usd":0.0007800000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.638534083031118}
{"item_id":"senseval3.d002.s011.t000","gold_sense_keys":["avenue%1:06:00::"],"candidates":[{"index":1,"sense_key":"avenue%1:04:00::","synset_id":"avenue.n.01"},{"index":2,"sense_key":"avenue%1:06:00::","synset_id":"avenue.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"avenue%1:06:00::","call_ids":["senseval3.d002.s011.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"avenue%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":504,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0016680000000000002,"input_uncached_usd":0.001008,"input_cached_usd":0.0,"output_usd":0.0006600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5776774580590427}
{"item_id":"senseval3.d002.s011.t002","gold_sense_keys":["floor%1:06:01::"],"candidates":[{"index":1,"sense_key":"floor%1:06:00::","synset_id":"floor.n.01"},{"index":2,"sense_key":"floor%1:06:01::","synset_id":"floor.n.02"},{"index":3,"sense_key":"floor%1:10:00::","synset_id":"floor.n.03"},{"index":4,"sense_key":"floor%1:17:01::","synset_id":"floor.n.04"},{"index":5,"sense_key":"floor%1:17:00::","synset_id":"floor.n.05"},{"index":6,"sense_key":"floor%1:17:02::","synset_id":"floor.n.06"},{"index":7,"sense_key":"floor%1:14:00::","synset_id":"floor.n.07"},{"index":8,"sense_key":"floor%1:07:00::","synset_id":"floor.n.08"},{"index":9,"sense_key":"floor%1:06:03::","synset_id":"floor.n.09"},{"index":10,"sense_key":"floor%1:06:02::","synset_id":"floor.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"floor%1:06:01::","call_ids":["senseval3.d002.s011.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"floor%1:06:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":966,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002052,"input_uncached_usd":0.001932,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2726460410049185}
{"item_id":"senseval3.d002.s011.t003","gold_sense_keys":["old%3:00:01::"],"candidates":[{"index":1,"sense_key":"old%3:00:02::","synset_id":"old.a.01"},{"index":2,"sense_key":"old%3:00:01::","synset_id":"old.a.02"},{"index":3,"sense_key":"old%5:00:00:familiar:00","synset_id":"old.s.01"},{"index":4,"sense_key":"old%5:00:00:experienced:00","synset_id":"old.s.02"},{"index":5,"sense_key":"old%5:00:03:past:00","synset_id":"erstwhile.s.01"},{"index":6,"sense_key":"old%5:00:00:genuine:00","synset_id":"honest-to-god.s.01"},{"index":7,"sense_key":"old%5:00:00:early:01","synset_id":"old.s.05"},{"index":8,"sense_key":"old%5:00:00:preceding:00","synset_id":"previous.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"old%3:00:01::","call_ids":["senseval3.d002.s011.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"old%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":904,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001928,"input_uncached_usd":0.001808,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3618632910074666}
{"item_id":"senseval3.d002.s011.t004","gold_sense_keys":["building%1:06:00::"],"candidates":[{"index":1,"sense_key":"building%1:06:00::","synset_id":"building.n.01"},{"index":2,"sense_key":"building%1:04:00::","synset_id":"construction.n.01"},{"index":3,"sense_key":"building%1:04:01::","synset_id":"construction.n.07"},{"index":4,"sense_key":"building%1:14:00::","synset_id":"building.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"building%1:06:00::","call_ids":["senseval3.d002.s011.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"building%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":651,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001422,"input_uncached_usd":0.001302,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4808503329986706}
{"item_id":"senseval3.d002.s011.t005","gold_sense_keys":["heart%1:08:00::"],"candidates":[{"index":1,"sense_key":"heart%1:09:00::","synset_id":"heart.n.01"},{"index":2,"sense_key":"heart%1:08:00::","synset_id":"heart.n.02"},{"index":3,"sense_key":"heart%1:07:01::","synset_id":"heart.n.03"},{"index":4,"sense_key":"heart%1:15:00::","synset_id":"center.n.01"},{"index":5,"sense_key":"heart%1:09:01::","synset_id":"kernel.n.03"},{"index":6,"sense_key":"heart%1:07:04::","synset_id":"heart.n.06"},{"index":7,"sense_key":"heart%1:25:00::","synset_id":"heart.n.07"},{"index":8,"sense_key":"heart%1:13:00::","synset_id":"heart.n.08"},{"index":9,"sense_key":"heart%1:12:04::","synset_id":"affection.n.01"},{"index":10,"sense_key":"heart%1:06:00::","synset_id":"heart.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"heart%1:08:00::","call_ids":["senseval3.d002.s011.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"heart%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1158,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003036,"input_uncached_usd":0.002316,"input_cached_usd":0.0,"output_usd":0.00072,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2989661670289934}
{"item_id":"senseval3.d002.s012.t000","gold_sense_keys":["fall%2:38:03::"],"candidates":[{"index":1,"sense_key":"fall%2:38:03::","synset_id":"fall.v.01"},{"index":2,"sense_key":"fall%2:38:00::","synset_id":"descend.v.01"},{"index":3,"sense_key":"fall%2:30:08::","synset_id":"fall.v.03"},{"index":4,"sense_key":"fall%2:42:03::","synset_id":"fall.v.04"},{"index":5,"sense_key":"fall%2:43:00::","synset_id":"precipitate.v.03"},{"index":6,"sense_key":"fall%2:41:02::","synset_id":"fall.v.06"},{"index":7,"sense_key":"fall%2:42:00::","synset_id":"fall.v.07"},{"index":8,"sense_key":"fall%2:42:04::","synset_id":"fall.v.08"},{"index":9,"sense_key":"fall%2:40:05::","synset_id":"fall.v.09"},{"index":10,"sense_key":"fall%2:30:00::","synset_id":"fall.v.10"},{"index":11,"sense_key":"fall%2:30:06::","synset_id":"decrease.v.01"},{"index":12,"sense_key":"fall%2:41:01::","synset_id":"fall.v.12"},{"index":13,"sense_key":"fall%2:41:00::","synset_id":"fall.v.13"},{"index":14,"sense_key":"fall%2:40:03::","synset_id":"fall.v.14"},{"index":15,"sense_key":"fall%2:38:01::","synset_id":"fall.v.15"},{"index":16,"sense_key":"fall%2:42:02::","synset_id":"fall.v.16"},{"index":17,"sense_key":"fall%2:41:15::","synset_id":"fall.v.17"},{"index":18,"sense_key":"fall%2:40:12::","synset_id":"fall.v.18"},{"index":19,"sense_key":"fall%2:40:09::","synset_id":"accrue.v.02"},{"index":20,"sense_key":"fall%2:40:08::","synset_id":"fall.v.20"},{"index":21,"sense_key":"fall%2:40:13::","synset_id":"fall.v.21"},{"index":22,"sense_key":"fall%2:38:11::","synset_id":"fall.v.22"},{"index":23,"sense_key":"fall%2:38:15::","synset_id":"fall.v.23"},{"index":24,"sense_key":"fall%2:38:14::","synset_id":"fall.v.24"},{"index":25,"sense_key":"fall%2:35:00::","synset_id":"hang.v.05"},{"index":26,"sense_key":"fall%2:30:11::","synset_id":"fall.v.26"},{"index":27,"sense_key":"fall%2:30:14::","synset_id":"fall.v.27"},{"index":28,"sense_key":"fall%2:30:10::","synset_id":"fall.v.28"},{"index":29,"sense_key":"fall%2:30:09::","synset_id":"fall.v.29"},{"index":30,"sense_key":"fall%2:30:12::","synset_id":"fall.v.30"},{"index":31,"sense_key":"fall%2:30:15::","synset_id":"fall.v.31"},{"index":32,"sense_key":"fall%2:30:13::","synset_id":"fall.v.32"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"fall%2:38:03::","call_ids":["senseval3.d002.s012.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"fall%2:38:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2182,"cached_input_tokens":0,"output_tokens":150,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.005863999999999999,"input_uncached_usd":0.004364,"input_cached_usd":0.0,"output_usd":0.0015,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.9207724580774084}
{"item_id":"senseval3.d002.s012.t001","gold_sense_keys":["poster%1:10:00::"],"candidates":[{"index":1,"sense_key":"poster%1:10:00::","synset_id":"poster.n.01"},{"index":2,"sense_key":"poster%1:18:00::","synset_id":"bill_poster.n.01"},{"index":3,"sense_key":"poster%1:05:00::","synset_id":"post_horse.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"poster%1:10:00::","call_ids":["senseval3.d002.s012.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"poster%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":580,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00128,"input_uncached_usd":0.00116,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1579145420109853}
{"item_id":"senseval3.d002.s012.t002","gold_sense_keys":["frame%1:06:00::"],"candidates":[{"index":1,"sense_key":"frame%1:06:02::","synset_id":"frame.n.01"},{"index":2,"sense_key":"frame%1:06:03::","synset_id":"frame.n.02"},{"index":3,"sense_key":"frame%1:08:01::","synset_id":"human_body.n.01"},{"index":4,"sense_key":"frame%1:28:00::","synset_id":"inning.n.01"},{"index":5,"sense_key":"frame%1:10:00::","synset_id":"frame.n.05"},{"index":6,"sense_key":"frame%1:10:01::","synset_id":"frame.n.06"},{"index":7,"sense_key":"frame%1:09:00::","synset_id":"frame_of_reference.n.02"},{"index":8,"sense_key":"frame%1:08:00::","synset_id":"skeletal_system.n.01"},{"index":9,"sense_key":"frame%1:06:01::","synset_id":"skeleton.n.04"},{"index":10,"sense_key":"frame%1:06:00::","synset_id":"frame.n.10"},{"index":11,"sense_key":"frame%1:04:01::","synset_id":"frame.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"frame%1:06:00::","call_ids":["senseval3.d002.s012.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"frame%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1013,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002146,"input_uncached_usd":0.002026,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2748011250514537}
{"item_id":"senseval3.d002.s012.t003","gold_sense_keys":["glass%1:27:00::"],"candidates":[{"index":1,"sense_key":"glass%1:27:00::","synset_id":"glass.n.01"},{"index":2,"sense_key":"glass%1:06:00::","synset_id":"glass.n.02"},{"index":3,"sense_key":"glass%1:23:00::","synset_id":"glass.n.03"},{"index":4,"sense_key":"glass%1:06:03::","synset_id":"field_glass.n.01"},{"index":5,"sense_key":"glass%1:06:01::","synset_id":"methamphetamine.n.01"},{"index":6,"sense_key":"glass%1:06:02::","synset_id":"looking_glass.n.01"},{"index":7,"sense_key":"glass%1:06:04::","synset_id":"glass.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"glass%1:27:00::","call_ids":["senseval3.d002.s012.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"glass%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":778,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001676,"input_uncached_usd":0.0015559999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9726955409860238}
{"item_id":"senseval3.d002.s012.t004","gold_sense_keys":["floor%1:06:00::"],"candidates":[{"index":1,"sense_key":"floor%1:06:00::","synset_id":"floor.n.01"},{"index":2,"sense_key":"floor%1:06:01::","synset_id":"floor.n.02"},{"index":3,"sense_key":"floor%1:10:00::","synset_id":"floor.n.03"},{"index":4,"sense_key":"floor%1:17:01::","synset_id":"floor.n.04"},{"index":5,"sense_key":"floor%1:17:00::","synset_id":"floor.n.05"},{"index":6,"sense_key":"floor%1:17:02::","synset_id":"floor.n.06"},{"index":7,"sense_key":"floor%1:14:00::","synset_id":"floor.n.07"},{"index":8,"sense_key":"floor%1:07:00::","synset_id":"floor.n.08"},{"index":9,"sense_key":"floor%1:06:03::","synset_id":"floor.n.09"},{"index":10,"sense_key":"floor%1:06:02::","synset_id":"floor.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"floor%1:06:00::","call_ids":["senseval3.d002.s012.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"floor%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":957,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002034,"input_uncached_usd":0.001914,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9631880829110742}
{"item_id":"senseval3.d002.s012.t006","gold_sense_keys":["open%3:00:01::"],"candidates":[{"index":1,"sense_key":"open%3:00:01::","synset_id":"open.a.01"},{"index":2,"sense_key":"open%3:00:02::","synset_id":"open.a.02"},{"index":3,"sense_key":"open%5:00:00:unprotected:00","synset_id":"exposed.s.01"},{"index":4,"sense_key":"open%5:00:00:public:00","synset_id":"open.s.02"},{"index":5,"sense_key":"open%3:00:08::","synset_id":"open.a.05"},{"index":6,"sense_key":"open%5:00:00:available:00","synset_id":"open.s.03"},{"index":7,"sense_key":"open%5:00:00:unrestricted:00","synset_id":"open.s.04"},{"index":8,"sense_key":"open%5:00:00:vulnerable:00","synset_id":"assailable.s.01"},{"index":9,"sense_key":"open%5:00:00:coarse:00","synset_id":"loose.s.06"},{"index":10,"sense_key":"open%5:00:00:unenclosed:00","synset_id":"open.s.07"},{"index":11,"sense_key":"open%3:00:04::","synset_id":"open.a.11"},{"index":12,"sense_key":"open%5:00:00:unsettled:02","synset_id":"open.s.08"},{"index":13,"sense_key":"open%5:00:00:unsealed:01","synset_id":"open.s.09"},{"index":14,"sense_key":"open%5:00:00:unconstricted:00","synset_id":"open.s.10"},{"index":15,"sense_key":"open%3:00:00::","synset_id":"receptive.a.02"},{"index":16,"sense_key":"open%3:00:07::","synset_id":"overt.a.01"},{"index":17,"sense_key":"open%5:00:00:nonunion:00","synset_id":"open.s.11"},{"index":18,"sense_key":"open%5:00:00:susceptible:00","synset_id":"capable.s.01"},{"index":19,"sense_key":"open%5:00:00:unobstructed:00","synset_id":"clear.s.02"},{"index":20,"sense_key":"open%5:00:00:ingenuous:00","synset_id":"candid.s.03"},{"index":21,"sense_key":"open%5:00:00:active:06","synset_id":"open.s.15"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"open%3:00:01::","call_ids":["senseval3.d002.s012.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"open%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1602,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0038439999999999998,"input_uncached_usd":0.003204,"input_cached_usd":0.0,"output_usd":0.00064,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8197418750496581}
{"item_id":"senseval3.d002.s012.t007","gold_sense_keys":["floor%1:06:00::"],"candidates":[{"index":1,"sense_key":"floor%1:06:00::","synset_id":"floor.n.01"},{"index":2,"sense_key":"floor%1:06:01::","synset_id":"floor.n.02"},{"index":3,"sense_key":"floor%1:10:00::","synset_id":"floor.n.03"},{"index":4,"sense_key":"floor%1:17:01::","synset_id":"floor.n.04"},{"index":5,"sense_key":"floor%1:17:00::","synset_id":"floor.n.05"},{"index":6,"sense_key":"floor%1:17:02::","synset_id":"floor.n.06"},{"index":7,"sense_key":"floor%1:14:00::","synset_id":"floor.n.07"},{"index":8,"sense_key":"floor%1:07:00::","synset_id":"floor.n.08"},{"index":9,"sense_key":"floor%1:06:03::","synset_id":"floor.n.09"},{"index":10,"sense_key":"floor%1:06:02::","synset_id":"floor.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"floor%1:06:00::","call_ids":["senseval3.d002.s012.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"floor%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":957,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002034,"input_uncached_usd":0.001914,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1092166670132428}
{"item_id":"senseval3.d002.s015.t000","gold_sense_keys":["favorite%5:00:00:loved:00"],"candidates":[{"index":1,"sense_key":"favorite%5:00:00:popular:00","synset_id":"favorite.s.01"},{"index":2,"sense_key":"favorite%5:00:00:loved:00","synset_id":"favored.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"favorite%5:00:00:loved:00","call_ids":["senseval3.d002.s015.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"favorite%5:00:00:loved:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":552,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001654,"input_uncached_usd":0.001104,"input_cached_usd":0.0,"output_usd":0.00055,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.450275499955751}
{"item_id":"senseval3.d002.s015.t001","gold_sense_keys":["wait%2:42:00::"],"candidates":[{"index":1,"sense_key":"wait%2:42:00::","synset_id":"wait.v.01"},{"index":2,"sense_key":"wait%2:42:01::","synset_id":"wait.v.02"},{"index":3,"sense_key":"wait%2:31:00::","synset_id":"expect.v.03"},{"index":4,"sense_key":"wait%2:41:00::","synset_id":"wait.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"wait%2:42:00::","call_ids":["senseval3.d002.s015.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"wait%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":679,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001478,"input_uncached_usd":0.0013579999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0049655840266496}
{"item_id":"senseval3.d002.s015.t002","gold_sense_keys":["game%1:04:03::"],"candidates":[{"index":1,"sense_key":"game%1:04:00::","synset_id":"game.n.01"},{"index":2,"sense_key":"game%1:04:03::","synset_id":"game.n.02"},{"index":3,"sense_key":"game%1:04:01::","synset_id":"game.n.03"},{"index":4,"sense_key":"game%1:05:00::","synset_id":"game.n.04"},{"index":5,"sense_key":"game%1:28:00::","synset_id":"game.n.05"},{"index":6,"sense_key":"game%1:23:00::","synset_id":"game.n.06"},{"index":7,"sense_key":"game%1:13:00::","synset_id":"game.n.07"},{"index":8,"sense_key":"game%1:09:00::","synset_id":"plot.n.01"},{"index":9,"sense_key":"game%1:06:00::","synset_id":"game.n.09"},{"index":10,"sense_key":"game%1:04:02::","synset_id":"game.n.10"},{"index":11,"sense_key":"game%1:04:04::","synset_id":"game.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"game%1:04:03::","call_ids":["senseval3.d002.s015.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"game%1:04:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":969,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002608,"input_uncached_usd":0.0019379999999999998,"input_cached_usd":0.0,"output_usd":0.00067,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.370465666987002}
{"item_id":"senseval3.d002.s015.t003","gold_sense_keys":["start%2:42:00::"],"candidates":[{"index":1,"sense_key":"start%2:30:00::","synset_id":"get_down.v.07"},{"index":2,"sense_key":"start%2:30:01::","synset_id":"begin.v.03"},{"index":3,"sense_key":"start%2:38:02::","synset_id":"depart.v.03"},{"index":4,"sense_key":"start%2:42:00::","synset_id":"begin.v.02"},{"index":5,"sense_key":"start%2:36:00::","synset_id":"originate.v.02"},{"index":6,"sense_key":"start%2:36:01::","synset_id":"start.v.06"},{"index":7,"sense_key":"start%2:38:04::","synset_id":"startle.v.02"},{"index":8,"sense_key":"start%2:38:01::","synset_id":"start.v.08"},{"index":9,"sense_key":"start%2:38:00::","synset_id":"start.v.09"},{"index":10,"sense_key":"start%2:41:00::","synset_id":"start.v.10"},{"index":11,"sense_key":"start%2:33:00::","synset_id":"start.v.11"},{"index":12,"sense_key":"start%2:42:03::","synset_id":"begin.v.07"},{"index":13,"sense_key":"start%2:41:04::","synset_id":"begin.v.08"},{"index":14,"sense_key":"start%2:38:03::","synset_id":"start.v.14"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"start%2:42:00::","call_ids":["senseval3.d002.s015.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"start%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1300,"cached_input_tokens":0,"output_tokens":242,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00502,"input_uncached_usd":0.0026,"input_cached_usd":0.0,"output_usd":0.0024200000000000003,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.5960997089277953}
{"item_id":"senseval3.d002.s016.t000","gold_sense_keys":["feel%2:39:00::"],"candidates":[{"index":1,"sense_key":"feel%2:37:00::","synset_id":"feel.v.01"},{"index":2,"sense_key":"feel%2:31:00::","synset_id":"find.v.05"},{"index":3,"sense_key":"feel%2:39:00::","synset_id":"feel.v.03"},{"index":4,"sense_key":"feel%2:29:00::","synset_id":"feel.v.04"},{"index":5,"sense_key":"feel%2:31:01::","synset_id":"feel.v.05"},{"index":6,"sense_key":"feel%2:39:01::","synset_id":"feel.v.06"},{"index":7,"sense_key":"feel%2:42:00::","synset_id":"feel.v.07"},{"index":8,"sense_key":"feel%2:39:10::","synset_id":"feel.v.08"},{"index":9,"sense_key":"feel%2:35:00::","synset_id":"feel.v.09"},{"index":10,"sense_key":"feel%2:35:01::","synset_id":"palpate.v.01"},{"index":11,"sense_key":"feel%2:40:00::","synset_id":"feel.v.11"},{"index":12,"sense_key":"feel%2:39:09::","synset_id":"feel.v.12"},{"index":13,"sense_key":"feel%2:35:11::","synset_id":"feel.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"feel%2:39:00::","call_ids":["senseval3.d002.s016.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"feel%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1221,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0031119999999999997,"input_uncached_usd":0.0024419999999999997,"input_cached_usd":0.0,"output_usd":0.00067,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.845308875082992}
{"item_id":"senseval3.d002.s016.t002","gold_sense_keys":["begin%2:42:00::","begin%2:42:04::"],"candidates":[{"index":1,"sense_key":"begin%2:30:00::","synset_id":"get_down.v.07"},{"index":2,"sense_key":"begin%2:42:00::","synset_id":"begin.v.02"},{"index":3,"sense_key":"begin%2:30:01::","synset_id":"begin.v.03"},{"index":4,"sense_key":"begin%2:32:03::","synset_id":"begin.v.04"},{"index":5,"sense_key":"begin%2:42:02::","synset_id":"begin.v.05"},{"index":6,"sense_key":"begin%2:42:04::","synset_id":"begin.v.06"},{"index":7,"sense_key":"begin%2:42:03::","synset_id":"begin.v.07"},{"index":8,"sense_key":"begin%2:41:04::","synset_id":"begin.v.08"},{"index":9,"sense_key":"begin%2:41:00::","synset_id":"begin.v.09"},{"index":10,"sense_key":"begin%2:32:04::","synset_id":"begin.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"begin%2:42:04::","call_ids":["senseval3.d002.s016.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"begin%2:42:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1144,"cached_input_tokens":0,"output_tokens":157,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0038580000000000003,"input_uncached_usd":0.002288,"input_cached_usd":0.0,"output_usd":0.0015700000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8560014999238774}
{"item_id":"senseval3.d002.s016.t003","gold_sense_keys":["glance%2:39:00::"],"candidates":[{"index":1,"sense_key":"glance%2:39:00::","synset_id":"glance.v.01"},{"index":2,"sense_key":"glance%2:38:00::","synset_id":"glance.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"glance%2:39:00::","call_ids":["senseval3.d002.s016.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"glance%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":536,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0011920000000000001,"input_uncached_usd":0.001072,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1956362499622628}
{"item_id":"senseval3.d002.s016.t004","gold_sense_keys":["table%1:14:01::"],"candidates":[{"index":1,"sense_key":"table%1:14:00::","synset_id":"table.n.01"},{"index":2,"sense_key":"table%1:06:01::","synset_id":"table.n.02"},{"index":3,"sense_key":"table%1:06:02::","synset_id":"table.n.03"},{"index":4,"sense_key":"table%1:17:00::","synset_id":"mesa.n.01"},{"index":5,"sense_key":"table%1:14:01::","synset_id":"table.n.05"},{"index":6,"sense_key":"table%1:13:01::","synset_id":"board.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"table%1:06:01::","call_ids":["senseval3.d002.s016.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"table%1:06:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":780,"cached_input_tokens":0,"output_tokens":150,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0030600000000000002,"input_uncached_usd":0.00156,"input_cached_usd":0.0,"output_usd":0.0015,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.845207541016862}
{"item_id":"senseval3.d002.s016.t005","gold_sense_keys":["next%5:00:00:close:01"],"candidates":[{"index":1,"sense_key":"next%5:00:00:succeeding:00","synset_id":"following.s.02"},{"index":2,"sense_key":"next%5:00:00:close:01","synset_id":"adjacent.s.01"},{"index":3,"sense_key":"next%5:00:00:incoming:00","synset_id":"future.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"next%5:00:00:close:01","call_ids":["senseval3.d002.s016.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"next%5:00:00:close:01","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":617,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001354,"input_uncached_usd":0.0012339999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1352381249889731}
{"item_id":"senseval3.d002.s016.t007","gold_sense_keys":["guilty%5:00:00:ashamed:00"],"candidates":[{"index":1,"sense_key":"guilty%3:00:00::","synset_id":"guilty.a.01"},{"index":2,"sense_key":"guilty%5:00:00:ashamed:00","synset_id":"guilty.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"guilty%5:00:00:ashamed:00","call_ids":["senseval3.d002.s016.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"guilty%5:00:00:ashamed:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":588,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001846,"input_uncached_usd":0.001176,"input_cached_usd":0.0,"output_usd":0.00067,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6182451250497252}
{"item_id":"senseval3.d002.s016.t009","gold_sense_keys":["mouth%2:32:01::"],"candidates":[{"index":1,"sense_key":"mouth%2:32:00::","synset_id":"talk.v.02"},{"index":2,"sense_key":"mouth%2:32:01::","synset_id":"mouth.v.02"},{"index":3,"sense_key":"mouth%2:35:00::","synset_id":"mouth.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"mouth%2:32:01::","call_ids":["senseval3.d002.s016.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"mouth%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":592,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001304,"input_uncached_usd":0.001184,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9309253749670461}
{"item_id":"senseval3.d002.s016.t010","gold_sense_keys":["word%1:10:00::"],"candidates":[{"index":1,"sense_key":"word%1:10:00::","synset_id":"word.n.01"},{"index":2,"sense_key":"word%1:10:03::","synset_id":"word.n.02"},{"index":3,"sense_key":"word%1:10:02::","synset_id":"news.n.01"},{"index":4,"sense_key":"word%1:10:06::","synset_id":"word.n.04"},{"index":5,"sense_key":"word%1:10:04::","synset_id":"discussion.n.02"},{"index":6,"sense_key":"word%1:10:01::","synset_id":"parole.n.01"},{"index":7,"sense_key":"word%1:23:00::","synset_id":"word.n.07"},{"index":8,"sense_key":"word%1:18:00::","synset_id":"son.n.02"},{"index":9,"sense_key":"word%1:10:05::","synset_id":"password.n.01"},{"index":10,"sense_key":"word%1:10:07::","synset_id":"bible.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"word%1:10:00::","call_ids":["senseval3.d002.s016.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"word%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1010,"cached_input_tokens":0,"output_tokens":48,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0025,"input_uncached_usd":0.00202,"input_cached_usd":0.0,"output_usd":0.00048000000000000007,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.363687540986575}
{"item_id":"senseval3.d002.s016.t011","gold_sense_keys":["earthquake%1:11:00::"],"candidates":[{"index":1,"sense_key":"earthquake%1:11:00::","synset_id":"earthquake.n.01"},{"index":2,"sense_key":"earthquake%1:26:00::","synset_id":"earthquake.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"earthquake%1:11:00::","call_ids":["senseval3.d002.s016.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"earthquake%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":570,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00126,"input_uncached_usd":0.00114,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5797239169478416}
{"item_id":"senseval3.d002.s017.t000","gold_sense_keys":["long%3:00:02::"],"candidates":[{"index":1,"sense_key":"long%3:00:02::","synset_id":"long.a.01"},{"index":2,"sense_key":"long%3:00:01::","synset_id":"long.a.02"},{"index":3,"sense_key":"long%5:00:00:tall:00","synset_id":"long.s.01"},{"index":4,"sense_key":"long%3:00:00::","synset_id":"retentive.a.01"},{"index":5,"sense_key":"long%3:00:05::","synset_id":"long.a.05"},{"index":6,"sense_key":"long%3:00:04::","synset_id":"long.a.06"},{"index":7,"sense_key":"long%5:00:00:unsound:00","synset_id":"long.s.02"},{"index":8,"sense_key":"long%5:00:00:provident:00","synset_id":"farseeing.s.02"},{"index":9,"sense_key":"long%5:00:00:abundant:00","synset_id":"long.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"long%3:00:02::","call_ids":["senseval3.d002.s017.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"long%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1000,"cached_input_tokens":0,"output_tokens":50,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0025,"input_uncached_usd":0.002,"input_cached_usd":0.0,"output_usd":0.0005,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.503360165981576}
{"item_id":"senseval3.d002.s017.t001","gold_sense_keys":["take%2:40:06::"],"candidates":[{"index":1,"sense_key":"take%2:41:04::","synset_id":"take.v.01"},{"index":2,"sense_key":"take%2:40:06::","synset_id":"take.v.02"},{"index":3,"sense_key":"take%2:38:09::","synset_id":"lead.v.01"},{"index":4,"sense_key":"take%2:35:00::","synset_id":"take.v.04"},{"index":5,"sense_key":"take%2:30:01::","synset_id":"assume.v.03"},{"index":6,"sense_key":"take%2:31:07::","synset_id":"take.v.06"},{"index":7,"sense_key":"take%2:38:10::","synset_id":"bring.v.01"},{"index":8,"sense_key":"take%2:40:00::","synset_id":"take.v.08"},{"index":9,"sense_key":"take%2:38:02::","synset_id":"take.v.09"},{"index":10,"sense_key":"take%2:31:01::","synset_id":"choose.v.01"},{"index":11,"sense_key":"take%2:40:05::","synset_id":"accept.v.02"},{"index":12,"sense_key":"take%2:41:00::","synset_id":"fill.v.04"},{"index":13,"sense_key":"take%2:31:03::","synset_id":"consider.v.03"},{"index":14,"sense_key":"take%2:42:00::","synset_id":"necessitate.v.01"},{"index":15,"sense_key":"take%2:39:00::","synset_id":"take.v.15"},{"index":16,"sense_key":"take%2:32:02::","synset_id":"film.v.01"},{"index":17,"sense_key":"take%2:30:00::","synset_id":"remove.v.01"},{"index":18,"sense_key":"take%2:34:00::","synset_id":"consume.v.02"},{"index":19,"sense_key":"take%2:31:09::","synset_id":"take.v.19"},{"index":20,"sense_key":"take%2:40:15::","synset_id":"take.v.20"},{"index":21,"sense_key":"take%2:40:01::","synset_id":"take.v.21"},{"index":22,"sense_key":"take%2:38:00::","synset_id":"assume.v.05"},{"index":23,"sense_key":"take%2:40:07::","synset_id":"accept.v.05"},{"index":24,"sense_key":"take%2:30:08::","synset_id":"take.v.24"},{"index":25,"sense_key":"take%2:31:02::","synset_id":"learn.v.04"},{"index":26,"sense_key":"take%2:32:00::","synset_id":"claim.v.05"},{"index":27,"sense_key":"take%2:38:05::","synset_id":"take.v.27"},{"index":28,"sense_key":"take%2:33:09::","synset_id":"aim.v.01"},{"index":29,"sense_key":"take%2:30:09::","synset_id":"take.v.29"},{"index":30,"sense_key":"take%2:42:10::","synset_id":"carry.v.02"},{"index":31,"sense_key":"take%2:40:03::","synset_id":"lease.v.04"},{"index":32,"sense_key":"take%2:40:04::","synset_id":"subscribe.v.05"},{"index":33,"sense_key":"take%2:40:02::","synset_id":"take.v.33"},{"index":34,"sense_key":"take%2:40:08::","synset_id":"take.v.34"},{"index":35,"sense_key":"take%2:35:14::","synset_id":"take.v.35"},{"index":36,"sense_key":"take%2:32:09::","synset_id":"claim.v.04"},{"index":37,"sense_key":"take%2:42:03::","synset_id":"accept.v.08"},{"index":38,"sense_key":"take%2:42:15::","synset_id":"contain.v.05"},{"index":39,"sense_key":"take%2:41:13::","synset_id":"take.v.39"},{"index":40,"sense_key":"take%2:38:11::","synset_id":"drive.v.16"},{"index":41,"sense_key":"take%2:33:08::","synset_id":"take.v.41"},{"index":42,"sense_key":"take%2:29:08::","synset_id":"contract.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"take%2:40:06::","call_ids":["senseval3.d002.s017.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"take%2:40:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":3022,"cached_input_tokens":0,"output_tokens":125,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.007294,"input_uncached_usd":0.006044,"input_cached_usd":0.0,"output_usd":0.00125,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0688983330037445}
{"item_id":"senseval3.d002.s017.t003","gold_sense_keys":["pass%2:30:01::","pass%2:38:03::"],"candidates":[{"index":1,"sense_key":"pass%2:38:00::","synset_id":"pass.v.01"},{"index":2,"sense_key":"pass%2:38:05::","synset_id":"travel_by.v.01"},{"index":3,"sense_key":"pass%2:41:02::","synset_id":"legislate.v.01"},{"index":4,"sense_key":"pass%2:38:03::","synset_id":"elapse.v.01"},{"index":5,"sense_key":"pass%2:40:00::","synset_id":"pass.v.05"},{"index":6,"sense_key":"pass%2:42:08::","synset_id":"run.v.03"},{"index":7,"sense_key":"pass%2:38:01::","synset_id":"pass.v.07"},{"index":8,"sense_key":"pass%2:30:00::","synset_id":"happen.v.01"},{"index":9,"sense_key":"pass%2:41:04::","synset_id":"pass.v.09"},{"index":10,"sense_key":"pass%2:42:00::","synset_id":"spend.v.01"},{"index":11,"sense_key":"pass%2:35:00::","synset_id":"guide.v.05"},{"index":12,"sense_key":"pass%2:32:01::","synset_id":"communicate.v.01"},{"index":13,"sense_key":"pass%2:30:01::","synset_id":"evanesce.v.01"},{"index":14,"sense_key":"pass%2:41:08::","synset_id":"pass.v.14"},{"index":15,"sense_key":"pass%2:42:04::","synset_id":"exceed.v.02"},{"index":16,"sense_key":"pass%2:41:07::","synset_id":"pass.v.16"},{"index":17,"sense_key":"pass%2:41:11::","synset_id":"pass.v.17"},{"index":18,"sense_key":"pass%2:40:02::","synset_id":"pass.v.18"},{"index":19,"sense_key":"pass%2:38:02::","synset_id":"sink.v.03"},{"index":20,"sense_key":"pass%2:40:01::","synset_id":"pass.v.20"},{"index":21,"sense_key":"pass%2:40:13::","synset_id":"fall.v.21"},{"index":22,"sense_key":"pass%2:38:04::","synset_id":"pass.v.22"},{"index":23,"sense_key":"pass%2:32:00::","synset_id":"authorize.v.01"},{"index":24,"sense_key":"pass%2:30:11::","synset_id":"die.v.01"},{"index":25,"sense_key":"pass%2:29:00::","synset_id":"excrete.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":13,"chosen_sense_key":"pass%2:30:01::","call_ids":["senseval3.d002.s017.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":13,"predicted_sense_key":"pass%2:30:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2037,"cached_input_tokens":0,"output_tokens":101,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.005084,"input_uncached_usd":0.0040739999999999995,"input_cached_usd":0.0,"output_usd":0.00101,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5861860829172656}
{"item_id":"senseval3.d002.s018.t000","gold_sense_keys":["time%1:11:00::"],"candidates":[{"index":1,"sense_key":"time%1:11:00::","synset_id":"time.n.01"},{"index":2,"sense_key":"time%1:28:05::","synset_id":"time.n.02"},{"index":3,"sense_key":"time%1:28:00::","synset_id":"time.n.03"},{"index":4,"sense_key":"time%1:28:06::","synset_id":"time.n.04"},{"index":5,"sense_key":"time%1:03:00::","synset_id":"time.n.05"},{"index":6,"sense_key":"time%1:11:01::","synset_id":"time.n.06"},{"index":7,"sense_key":"time%1:28:03::","synset_id":"clock_time.n.01"},{"index":8,"sense_key":"time%1:28:04::","synset_id":"fourth_dimension.n.01"},{"index":9,"sense_key":"time%1:07:01::","synset_id":"meter.n.04"},{"index":10,"sense_key":"time%1:28:01::","synset_id":"prison_term.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"time%1:11:00::","call_ids":["senseval3.d002.s018.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"time%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":950,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00202,"input_uncached_usd":0.0019,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1850422080606222}
{"item_id":"senseval3.d002.s018.t001","gold_sense_keys":["get%2:30:00::"],"candidates":[{"index":1,"sense_key":"get%2:40:00::","synset_id":"get.v.01"},{"index":2,"sense_key":"get%2:30:00::","synset_id":"become.v.01"},{"index":3,"sense_key":"get%2:30:02::","synset_id":"get.v.03"},{"index":4,"sense_key":"get%2:30:03::","synset_id":"receive.v.02"},{"index":5,"sense_key":"get%2:38:00::","synset_id":"arrive.v.01"},{"index":6,"sense_key":"get%2:35:03::","synset_id":"bring.v.04"},{"index":7,"sense_key":"get%2:39:00::","synset_id":"experience.v.03"},{"index":8,"sense_key":"get%2:33:00::","synset_id":"pay_back.v.02"},{"index":9,"sense_key":"get%2:40:01::","synset_id":"have.v.17"},{"index":10,"sense_key":"get%2:32:00::","synset_id":"induce.v.02"},{"index":11,"sense_key":"get%2:35:09::","synset_id":"get.v.11"},{"index":12,"sense_key":"get%2:29:01::","synset_id":"grow.v.08"},{"index":13,"sense_key":"get%2:29:00::","synset_id":"contract.v.04"},{"index":14,"sense_key":"get%2:32:03::","synset_id":"get.v.14"},{"index":15,"sense_key":"get%2:30:01::","synset_id":"make.v.02"},{"index":16,"sense_key":"get%2:32:08::","synset_id":"drive.v.11"},{"index":17,"sense_key":"get%2:31:09::","synset_id":"catch.v.18"},{"index":18,"sense_key":"get%2:35:00::","synset_id":"catch.v.07"},{"index":19,"sense_key":"get%2:35:10::","synset_id":"get.v.19"},{"index":20,"sense_key":"get%2:31:03::","synset_id":"get.v.20"},{"index":21,"sense_key":"get%2:40:14::","synset_id":"get.v.21"},{"index":22,"sense_key":"get%2:40:13::","synset_id":"get.v.22"},{"index":23,"sense_key":"get%2:39:13::","synset_id":"catch.v.21"},{"index":24,"sense_key":"get%2:39:09::","synset_id":"catch.v.22"},{"index":25,"sense_key":"get%2:39:14::","synset_id":"get.v.25"},{"index":26,"sense_key":"get%2:38:15::","synset_id":"scram.v.01"},{"index":27,"sense_key":"get%2:38:12::","synset_id":"get.v.27"},{"index":28,"sense_key":"get%2:37:12::","synset_id":"get.v.28"},{"index":29,"sense_key":"get%2:37:11::","synset_id":"get.v.29"},{"index":30,"sense_key":"get%2:36:14::","synset_id":"catch.v.24"},{"index":31,"sense_key":"get%2:36:00::","synset_id":"draw.v.15"},{"index":32,"sense_key":"get%2:35:12::","synset_id":"get.v.32"},{"index":33,"sense_key":"get%2:31:10::","synset_id":"perplex.v.01"},{"index":34,"sense_key":"get%2:30:12::","synset_id":"get_down.v.07"},{"index":35,"sense_key":"get%2:29:11::","synset_id":"suffer.v.02"},{"index":36,"sense_key":"get%2:29:08::","synset_id":"beget.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"get%2:30:00::","call_ids":["senseval3.d002.s018.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"get%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2722,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.006064,"input_uncached_usd":0.005444,"input_cached_usd":0.0,"output_usd":0.00062,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.103232124936767}
{"item_id":"senseval3.d002.s018.t002","gold_sense_keys":["building%1:06:00::"],"candidates":[{"index":1,"sense_key":"building%1:06:00::","synset_id":"building.n.01"},{"index":2,"sense_key":"building%1:04:00::","synset_id":"construction.n.01"},{"index":3,"sense_key":"building%1:04:01::","synset_id":"construction.n.07"},{"index":4,"sense_key":"building%1:14:00::","synset_id":"building.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"building%1:06:00::","call_ids":["senseval3.d002.s018.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"building%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":631,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001382,"input_uncached_usd":0.0012619999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8686176670016721}
{"item_id":"senseval3.d002.s018.t003","gold_sense_keys":["start%2:30:00::"],"candidates":[{"index":1,"sense_key":"start%2:30:00::","synset_id":"get_down.v.07"},{"index":2,"sense_key":"start%2:30:01::","synset_id":"begin.v.03"},{"index":3,"sense_key":"start%2:38:02::","synset_id":"depart.v.03"},{"index":4,"sense_key":"start%2:42:00::","synset_id":"begin.v.02"},{"index":5,"sense_key":"start%2:36:00::","synset_id":"originate.v.02"},{"index":6,"sense_key":"start%2:36:01::","synset_id":"start.v.06"},{"index":7,"sense_key":"start%2:38:04::","synset_id":"startle.v.02"},{"index":8,"sense_key":"start%2:38:01::","synset_id":"start.v.08"},{"index":9,"sense_key":"start%2:38:00::","synset_id":"start.v.09"},{"index":10,"sense_key":"start%2:41:00::","synset_id":"start.v.10"},{"index":11,"sense_key":"start%2:33:00::","synset_id":"start.v.11"},{"index":12,"sense_key":"start%2:42:03::","synset_id":"begin.v.07"},{"index":13,"sense_key":"start%2:41:04::","synset_id":"begin.v.08"},{"index":14,"sense_key":"start%2:38:03::","synset_id":"start.v.14"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"start%2:30:00::","call_ids":["senseval3.d002.s018.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"start%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1264,"cached_input_tokens":0,"output_tokens":131,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0038380000000000003,"input_uncached_usd":0.002528,"input_cached_usd":0.0,"output_usd":0.0013100000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0126830419758335}
{"item_id":"senseval3.d002.s018.t004","gold_sense_keys":["shake%2:29:00::","shake%2:38:00::","shake%2:38:01::","shake%2:38:02::"],"candidates":[{"index":1,"sense_key":"shake%2:38:00::","synset_id":"shake.v.01"},{"index":2,"sense_key":"shake%2:29:00::","synset_id":"shake.v.02"},{"index":3,"sense_key":"shake%2:38:01::","synset_id":"judder.v.01"},{"index":4,"sense_key":"shake%2:38:02::","synset_id":"rock.v.01"},{"index":5,"sense_key":"shake%2:30:00::","synset_id":"shake.v.05"},{"index":6,"sense_key":"shake%2:37:00::","synset_id":"stimulate.v.03"},{"index":7,"sense_key":"shake%2:38:04::","synset_id":"shake.v.07"},{"index":8,"sense_key":"shake%2:30:01::","synset_id":"shake.v.08"},{"index":9,"sense_key":"shake%2:32:00::","synset_id":"shake.v.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"shake%2:38:00::","call_ids":["senseval3.d002.s018.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"shake%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":942,"cached_input_tokens":0,"output_tokens":287,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0047539999999999995,"input_uncached_usd":0.0018839999999999998,"input_cached_usd":0.0,"output_usd":0.00287,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.785419457941316}
{"item_id":"senseval3.d002.s018.t005","gold_sense_keys":["child%1:18:00::"],"candidates":[{"index":1,"sense_key":"child%1:18:00::","synset_id":"child.n.01"},{"index":2,"sense_key":"child%1:18:01::","synset_id":"child.n.02"},{"index":3,"sense_key":"child%1:18:06::","synset_id":"child.n.03"},{"index":4,"sense_key":"child%1:18:07::","synset_id":"child.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"child%1:18:00::","call_ids":["senseval3.d002.s018.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"child%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":618,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001356,"input_uncached_usd":0.001236,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.398966999957338}
{"item_id":"senseval3.d002.s018.t006","gold_sense_keys":["toy%1:06:00::"],"candidates":[{"index":1,"sense_key":"toy%1:06:00::","synset_id":"plaything.n.01"},{"index":2,"sense_key":"toy%1:06:02::","synset_id":"toy.n.02"},{"index":3,"sense_key":"toy%1:06:03::","synset_id":"toy.n.03"},{"index":4,"sense_key":"toy%1:06:01::","synset_id":"miniature.n.02"},{"index":5,"sense_key":"toy%1:05:00::","synset_id":"toy_dog.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"toy%1:06:00::","call_ids":["senseval3.d002.s018.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"toy%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":629,"cached_input_tokens":0,"output_tokens":88,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002138,"input_uncached_usd":0.001258,"input_cached_usd":0.0,"output_usd":0.00088,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5315445000305772}
{"item_id":"senseval3.d002.s018.t007","gold_sense_keys":["block%1:06:00::"],"candidates":[{"index":1,"sense_key":"block%1:06:00::","synset_id":"block.n.01"},{"index":2,"sense_key":"block%1:15:00::","synset_id":"block.n.02"},{"index":3,"sense_key":"block%1:25:01::","synset_id":"block.n.03"},{"index":4,"sense_key":"block%1:14:00::","synset_id":"block.n.04"},{"index":5,"sense_key":"block%1:06:06::","synset_id":"block.n.05"},{"index":6,"sense_key":"block%1:23:00::","synset_id":"block.n.06"},{"index":7,"sense_key":"block%1:09:00::","synset_id":"block.n.07"},{"index":8,"sense_key":"block%1:06:02::","synset_id":"pulley.n.01"},{"index":9,"sense_key":"block%1:06:04::","synset_id":"engine_block.n.01"},{"index":10,"sense_key":"block%1:06:01::","synset_id":"blockage.n.02"},{"index":11,"sense_key":"block%1:06:05::","synset_id":"auction_block.n.01"},{"index":12,"sense_key":"block%1:04:00::","synset_id":"blocking.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"block%1:06:00::","call_ids":["senseval3.d002.s018.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"block%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1179,"cached_input_tokens":0,"output_tokens":91,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003268,"input_uncached_usd":0.002358,"input_cached_usd":0.0,"output_usd":0.0009100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6435091249877587}
{"item_id":"senseval3.d002.s018.t008","gold_sense_keys":["toss%2:35:05::"],"candidates":[{"index":1,"sense_key":"toss%2:35:04::","synset_id":"flip.v.06"},{"index":2,"sense_key":"toss%2:38:00::","synset_id":"flip.v.01"},{"index":3,"sense_key":"toss%2:35:05::","synset_id":"chuck.v.01"},{"index":4,"sense_key":"toss%2:38:01::","synset_id":"convulse.v.03"},{"index":5,"sense_key":"toss%2:40:00::","synset_id":"discard.v.01"},{"index":6,"sense_key":"toss%2:35:00::","synset_id":"toss.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"toss%2:38:01::","call_ids":["senseval3.d002.s018.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"toss%2:38:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":738,"cached_input_tokens":0,"output_tokens":227,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003746,"input_uncached_usd":0.0014759999999999999,"input_cached_usd":0.0,"output_usd":0.0022700000000000003,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.543079916969873}
{"item_id":"senseval3.d002.s021.t000","gold_sense_keys":["library%1:06:00::","library%1:06:01::","library%1:06:02::"],"candidates":[{"index":1,"sense_key":"library%1:06:01::","synset_id":"library.n.01"},{"index":2,"sense_key":"library%1:14:00::","synset_id":"library.n.02"},{"index":3,"sense_key":"library%1:06:00::","synset_id":"library.n.03"},{"index":4,"sense_key":"library%1:14:01::","synset_id":"library.n.04"},{"index":5,"sense_key":"library%1:06:02::","synset_id":"library.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"library%1:06:02::","call_ids":["senseval3.d002.s021.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"library%1:06:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":620,"cached_input_tokens":0,"output_tokens":132,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00256,"input_uncached_usd":0.00124,"input_cached_usd":0.0,"output_usd":0.0013200000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6351036250125617}
{"item_id":"senseval3.d002.s021.t001","gold_sense_keys":["hit%2:35:05::"],"candidates":[{"index":1,"sense_key":"hit%2:35:03::","synset_id":"hit.v.01"},{"index":2,"sense_key":"hit%2:35:01::","synset_id":"hit.v.02"},{"index":3,"sense_key":"hit%2:35:00::","synset_id":"hit.v.03"},{"index":4,"sense_key":"hit%2:38:00::","synset_id":"reach.v.01"},{"index":5,"sense_key":"hit%2:35:05::","synset_id":"hit.v.05"},{"index":6,"sense_key":"hit%2:33:01::","synset_id":"shoot.v.01"},{"index":7,"sense_key":"hit%2:40:05::","synset_id":"stumble.v.03"},{"index":8,"sense_key":"hit%2:33:02::","synset_id":"score.v.01"},{"index":9,"sense_key":"hit%2:39:00::","synset_id":"hit.v.09"},{"index":10,"sense_key":"hit%2:33:00::","synset_id":"strike.v.04"},{"index":11,"sense_key":"hit%2:41:00::","synset_id":"murder.v.01"},{"index":12,"sense_key":"hit%2:38:02::","synset_id":"hit.v.12"},{"index":13,"sense_key":"hit%2:38:07::","synset_id":"reach.v.02"},{"index":14,"sense_key":"hit%2:35:02::","synset_id":"strike.v.10"},{"index":15,"sense_key":"hit%2:34:00::","synset_id":"hit.v.15"},{"index":16,"sense_key":"hit%2:33:03::","synset_id":"hit.v.16"},{"index":17,"sense_key":"hit%2:32:12::","synset_id":"hit.v.17"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"hit%2:35:05::","call_ids":["senseval3.d002.s021.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"hit%2:35:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1476,"cached_input_tokens":0,"output_tokens":82,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0037719999999999997,"input_uncached_usd":0.0029519999999999998,"input_cached_usd":0.0,"output_usd":0.0008200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7400705839972943}
{"item_id":"senseval3.d002.s022.t000","gold_sense_keys":["endless%5:00:00:long:02"],"candidates":[{"index":1,"sense_key":"endless%5:00:00:long:02","synset_id":"endless.s.01"},{"index":2,"sense_key":"endless%5:00:02:infinite:00","synset_id":"endless.s.02"},{"index":3,"sense_key":"endless%5:00:00:infinite:00","synset_id":"dateless.s.01"},{"index":4,"sense_key":"endless%5:00:00:continuous:01","synset_id":"endless.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"endless%5:00:00:long:02","call_ids":["senseval3.d002.s022.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"endless%5:00:00:long:02","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":594,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0013080000000000001,"input_uncached_usd":0.001188,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1700412499485537}
{"item_id":"senseval3.d002.s022.t001","gold_sense_keys":["second%1:28:00::"],"candidates":[{"index":1,"sense_key":"second%1:28:00::","synset_id":"second.n.01"},{"index":2,"sense_key":"second%1:28:01::","synset_id":"moment.n.02"},{"index":3,"sense_key":"second%1:04:00::","synset_id":"second_base.n.02"},{"index":4,"sense_key":"second%1:28:02::","synset_id":"moment.n.01"},{"index":5,"sense_key":"second%1:24:00::","synset_id":"second.n.05"},{"index":6,"sense_key":"second%1:23:00::","synset_id":"second.n.06"},{"index":7,"sense_key":"second%1:18:00::","synset_id":"second.n.07"},{"index":8,"sense_key":"second%1:10:00::","synset_id":"second.n.08"},{"index":9,"sense_key":"second%1:06:00::","synset_id":"second_gear.n.01"},{"index":10,"sense_key":"second%1:06:01::","synset_id":"irregular.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"second%1:28:00::","call_ids":["senseval3.d002.s022.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"second%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":938,"cached_input_tokens":0,"output_tokens":40,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002276,"input_uncached_usd":0.0018759999999999998,"input_cached_usd":0.0,"output_usd":0.0004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7419911249307916}
{"item_id":"senseval3.d002.s022.t002","gold_sense_keys":["wonder%2:32:01::"],"candidates":[{"index":1,"sense_key":"wonder%2:31:00::","synset_id":"wonder.v.01"},{"index":2,"sense_key":"wonder%2:32:01::","synset_id":"wonder.v.02"},{"index":3,"sense_key":"wonder%2:32:12::","synset_id":"wonder.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"wonder%2:32:01::","call_ids":["senseval3.d002.s022.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"wonder%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":556,"cached_input_tokens":0,"output_tokens":136,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0024720000000000002,"input_uncached_usd":0.001112,"input_cached_usd":0.0,"output_usd":0.00136,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8164586250204593}
{"item_id":"senseval3.d002.s022.t004","gold_sense_keys":["window%1:06:00::"],"candidates":[{"index":1,"sense_key":"window%1:06:00::","synset_id":"window.n.01"},{"index":2,"sense_key":"window%1:06:01::","synset_id":"window.n.02"},{"index":3,"sense_key":"window%1:06:02::","synset_id":"window.n.03"},{"index":4,"sense_key":"window%1:17:00::","synset_id":"window.n.04"},{"index":5,"sense_key":"window%1:28:00::","synset_id":"window.n.05"},{"index":6,"sense_key":"window%1:06:04::","synset_id":"windowpane.n.01"},{"index":7,"sense_key":"window%1:06:05::","synset_id":"window.n.07"},{"index":8,"sense_key":"window%1:06:03::","synset_id":"window.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"window%1:06:00::","call_ids":["senseval3.d002.s022.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"window%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":807,"cached_input_tokens":0,"output_tokens":163,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0032440000000000004,"input_uncached_usd":0.001614,"input_cached_usd":0.0,"output_usd":0.0016300000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.7099084159126505}
{"item_id":"senseval3.d002.s022.t005","gold_sense_keys":["buckle%2:30:00::","buckle%2:35:01::"],"candidates":[{"index":1,"sense_key":"buckle%2:35:00::","synset_id":"buckle.v.01"},{"index":2,"sense_key":"buckle%2:35:01::","synset_id":"buckle.v.02"},{"index":3,"sense_key":"buckle%2:30:00::","synset_id":"heave.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"buckle%2:30:00::","call_ids":["senseval3.d002.s022.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"buckle%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":518,"cached_input_tokens":0,"output_tokens":123,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0022660000000000002,"input_uncached_usd":0.001036,"input_cached_usd":0.0,"output_usd":0.0012300000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7210752919781953}
{"item_id":"senseval3.d002.s022.t006","gold_sense_keys":["shower%2:35:00::"],"candidates":[{"index":1,"sense_key":"shower%2:40:00::","synset_id":"lavish.v.01"},{"index":2,"sense_key":"shower%2:35:00::","synset_id":"shower.v.02"},{"index":3,"sense_key":"shower%2:29:00::","synset_id":"shower.v.03"},{"index":4,"sense_key":"shower%2:43:00::","synset_id":"shower.v.04"},{"index":5,"sense_key":"shower%2:34:12::","synset_id":"shower.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"shower%2:35:00::","call_ids":["senseval3.d002.s022.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"shower%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":607,"cached_input_tokens":0,"output_tokens":124,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002454,"input_uncached_usd":0.001214,"input_cached_usd":0.0,"output_usd":0.00124,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8007679999573156}
{"item_id":"senseval3.d002.s022.t007","gold_sense_keys":["glass%1:27:00::"],"candidates":[{"index":1,"sense_key":"glass%1:27:00::","synset_id":"glass.n.01"},{"index":2,"sense_key":"glass%1:06:00::","synset_id":"glass.n.02"},{"index":3,"sense_key":"glass%1:23:00::","synset_id":"glass.n.03"},{"index":4,"sense_key":"glass%1:06:03::","synset_id":"field_glass.n.01"},{"index":5,"sense_key":"glass%1:06:01::","synset_id":"methamphetamine.n.01"},{"index":6,"sense_key":"glass%1:06:02::","synset_id":"looking_glass.n.01"},{"index":7,"sense_key":"glass%1:06:04::","synset_id":"glass.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"glass%1:27:00::","call_ids":["senseval3.d002.s022.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"glass%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":744,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001608,"input_uncached_usd":0.001488,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4691597499186173}
{"item_id":"senseval3.d002.s023.t001","gold_sense_keys":["book%1:06:00::"],"candidates":[{"index":1,"sense_key":"book%1:10:00::","synset_id":"book.n.01"},{"index":2,"sense_key":"book%1:06:00::","synset_id":"book.n.02"},{"index":3,"sense_key":"book%1:10:02::","synset_id":"record.n.05"},{"index":4,"sense_key":"book%1:10:01::","synset_id":"script.n.01"},{"index":5,"sense_key":"book%1:21:00::","synset_id":"ledger.n.01"},{"index":6,"sense_key":"book%1:14:01::","synset_id":"book.n.06"},{"index":7,"sense_key":"book%1:14:00::","synset_id":"book.n.07"},{"index":8,"sense_key":"book%1:10:05::","synset_id":"koran.n.01"},{"index":9,"sense_key":"book%1:10:04::","synset_id":"bible.n.01"},{"index":10,"sense_key":"book%1:10:03::","synset_id":"book.n.10"},{"index":11,"sense_key":"book%1:06:02::","synset_id":"book.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"book%1:06:00::","call_ids":["senseval3.d002.s023.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"book%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1066,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002672,"input_uncached_usd":0.0021319999999999998,"input_cached_usd":0.0,"output_usd":0.00054,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4605037089204416}
{"item_id":"senseval3.d002.s023.t002","gold_sense_keys":["fall%2:38:03::"],"candidates":[{"index":1,"sense_key":"fall%2:38:03::","synset_id":"fall.v.01"},{"index":2,"sense_key":"fall%2:38:00::","synset_id":"descend.v.01"},{"index":3,"sense_key":"fall%2:30:08::","synset_id":"fall.v.03"},{"index":4,"sense_key":"fall%2:42:03::","synset_id":"fall.v.04"},{"index":5,"sense_key":"fall%2:43:00::","synset_id":"precipitate.v.03"},{"index":6,"sense_key":"fall%2:41:02::","synset_id":"fall.v.06"},{"index":7,"sense_key":"fall%2:42:00::","synset_id":"fall.v.07"},{"index":8,"sense_key":"fall%2:42:04::","synset_id":"fall.v.08"},{"index":9,"sense_key":"fall%2:40:05::","synset_id":"fall.v.09"},{"index":10,"sense_key":"fall%2:30:00::","synset_id":"fall.v.10"},{"index":11,"sense_key":"fall%2:30:06::","synset_id":"decrease.v.01"},{"index":12,"sense_key":"fall%2:41:01::","synset_id":"fall.v.12"},{"index":13,"sense_key":"fall%2:41:00::","synset_id":"fall.v.13"},{"index":14,"sense_key":"fall%2:40:03::","synset_id":"fall.v.14"},{"index":15,"sense_key":"fall%2:38:01::","synset_id":"fall.v.15"},{"index":16,"sense_key":"fall%2:42:02::","synset_id":"fall.v.16"},{"index":17,"sense_key":"fall%2:41:15::","synset_id":"fall.v.17"},{"index":18,"sense_key":"fall%2:40:12::","synset_id":"fall.v.18"},{"index":19,"sense_key":"fall%2:40:09::","synset_id":"accrue.v.02"},{"index":20,"sense_key":"fall%2:40:08::","synset_id":"fall.v.20"},{"index":21,"sense_key":"fall%2:40:13::","synset_id":"fall.v.21"},{"index":22,"sense_key":"fall%2:38:11::","synset_id":"fall.v.22"},{"index":23,"sense_key":"fall%2:38:15::","synset_id":"fall.v.23"},{"index":24,"sense_key":"fall%2:38:14::","synset_id":"fall.v.24"},{"index":25,"sense_key":"fall%2:35:00::","synset_id":"hang.v.05"},{"index":26,"sense_key":"fall%2:30:11::","synset_id":"fall.v.26"},{"index":27,"sense_key":"fall%2:30:14::","synset_id":"fall.v.27"},{"index":28,"sense_key":"fall%2:30:10::","synset_id":"fall.v.28"},{"index":29,"sense_key":"fall%2:30:09::","synset_id":"fall.v.29"},{"index":30,"sense_key":"fall%2:30:12::","synset_id":"fall.v.30"},{"index":31,"sense_key":"fall%2:30:15::","synset_id":"fall.v.31"},{"index":32,"sense_key":"fall%2:30:13::","synset_id":"fall.v.32"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"fall%2:38:03::","call_ids":["senseval3.d002.s023.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"fall%2:38:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2160,"cached_input_tokens":0,"output_tokens":100,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00532,"input_uncached_usd":0.00432,"input_cached_usd":0.0,"output_usd":0.001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7124392909463495}
{"item_id":"senseval3.d002.s024.t001","gold_sense_keys":["paint%1:06:00::"],"candidates":[{"index":1,"sense_key":"paint%1:06:00::","synset_id":"paint.n.01"},{"index":2,"sense_key":"paint%1:15:00::","synset_id":"key.n.09"},{"index":3,"sense_key":"paint%1:06:01::","synset_id":"rouge.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"paint%1:06:00::","call_ids":["senseval3.d002.s024.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"paint%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":598,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0013160000000000001,"input_uncached_usd":0.001196,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1806275829439983}
{"item_id":"senseval3.d002.s024.t002","gold_sense_keys":["shop%1:06:01::"],"candidates":[{"index":1,"sense_key":"shop%1:06:00::","synset_id":"shop.n.01"},{"index":2,"sense_key":"shop%1:06:01::","synset_id":"workshop.n.01"},{"index":3,"sense_key":"shop%1:04:00::","synset_id":"shop_class.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"shop%1:06:01::","call_ids":["senseval3.d002.s024.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"shop%1:06:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":523,"cached_input_tokens":0,"output_tokens":82,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001866,"input_uncached_usd":0.001046,"input_cached_usd":0.0,"output_usd":0.0008200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2826199169503525}
{"item_id":"senseval3.d002.s024.t003","gold_sense_keys":["fire%1:11:00::"],"candidates":[{"index":1,"sense_key":"fire%1:11:00::","synset_id":"fire.n.01"},{"index":2,"sense_key":"fire%1:04:00::","synset_id":"fire.n.02"},{"index":3,"sense_key":"fire%1:22:00::","synset_id":"fire.n.03"},{"index":4,"sense_key":"fire%1:06:00::","synset_id":"fire.n.04"},{"index":5,"sense_key":"fire%1:27:01::","synset_id":"fire.n.05"},{"index":6,"sense_key":"fire%1:12:00::","synset_id":"ardor.n.03"},{"index":7,"sense_key":"fire%1:27:00::","synset_id":"fire.n.07"},{"index":8,"sense_key":"fire%1:11:01::","synset_id":"fire.n.08"},{"index":9,"sense_key":"fire%1:10:00::","synset_id":"fire.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"fire%1:11:00::","call_ids":["senseval3.d002.s024.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"fire%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":902,"cached_input_tokens":0,"output_tokens":51,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002314,"input_uncached_usd":0.0018039999999999998,"input_cached_usd":0.0,"output_usd":0.00051,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5978618329390883}
{"item_id":"senseval3.d002.s024.t004","gold_sense_keys":["send%2:38:01::"],"candidates":[{"index":1,"sense_key":"send%2:38:01::","synset_id":"send.v.01"},{"index":2,"sense_key":"send%2:35:00::","synset_id":"send.v.02"},{"index":3,"sense_key":"send%2:32:00::","synset_id":"mail.v.02"},{"index":4,"sense_key":"send%2:38:00::","synset_id":"transport.v.04"},{"index":5,"sense_key":"send%2:33:00::","synset_id":"station.v.01"},{"index":6,"sense_key":"send%2:32:02::","synset_id":"send.v.06"},{"index":7,"sense_key":"send%2:40:00::","synset_id":"commit.v.03"},{"index":8,"sense_key":"send%2:32:03::","synset_id":"air.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"send%2:38:01::","call_ids":["senseval3.d002.s024.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"send%2:38:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":828,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001776,"input_uncached_usd":0.001656,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.449920125072822}
{"item_id":"senseval3.d002.s024.t006","gold_sense_keys":["cloud%1:19:01::"],"candidates":[{"index":1,"sense_key":"cloud%1:19:01::","synset_id":"cloud.n.01"},{"index":2,"sense_key":"cloud%1:17:00::","synset_id":"cloud.n.02"},{"index":3,"sense_key":"cloud%1:26:01::","synset_id":"cloud.n.03"},{"index":4,"sense_key":"cloud%1:26:02::","synset_id":"cloud.n.04"},{"index":5,"sense_key":"cloud%1:26:00::","synset_id":"cloud.n.05"},{"index":6,"sense_key":"cloud%1:14:00::","synset_id":"swarm.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cloud%1:19:01::","call_ids":["senseval3.d002.s024.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cloud%1:19:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":661,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0014420000000000001,"input_uncached_usd":0.001322,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0148092079907656}
{"item_id":"senseval3.d002.s024.t007","gold_sense_keys":["black%3:00:01::"],"candidates":[{"index":1,"sense_key":"black%3:00:01::","synset_id":"black.a.01"},{"index":2,"sense_key":"black%3:00:02::","synset_id":"black.a.02"},{"index":3,"sense_key":"black%5:00:00:angry:00","synset_id":"black.s.01"},{"index":4,"sense_key":"black%5:00:00:hopeless:00","synset_id":"black.s.02"},{"index":5,"sense_key":"black%5:00:00:evil:00","synset_id":"black.s.03"},{"index":6,"sense_key":"black%5:00:00:unfortunate:00","synset_id":"black.s.04"},{"index":7,"sense_key":"black%5:00:00:colored:00","synset_id":"black.s.05"},{"index":8,"sense_key":"black%5:00:00:dark:01","synset_id":"black.s.06"},{"index":9,"sense_key":"black%5:00:00:sarcastic:00","synset_id":"black.s.07"},{"index":10,"sense_key":"black%5:00:00:covert:00","synset_id":"black.s.08"},{"index":11,"sense_key":"black%5:00:00:illegal:00","synset_id":"bootleg.s.01"},{"index":12,"sense_key":"black%5:00:00:dishonorable:00","synset_id":"black.s.10"},{"index":13,"sense_key":"black%5:00:00:undiluted:00","synset_id":"black.s.11"},{"index":14,"sense_key":"black%5:00:00:dirty:01","synset_id":"black.s.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"black%3:00:01::","call_ids":["senseval3.d002.s024.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"black%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1406,"cached_input_tokens":0,"output_tokens":85,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003662,"input_uncached_usd":0.002812,"input_cached_usd":0.0,"output_usd":0.0008500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9838227910222486}
{"item_id":"senseval3.d002.s024.t008","gold_sense_keys":["smoke%1:19:00::","smoke%1:22:00::"],"candidates":[{"index":1,"sense_key":"smoke%1:19:00::","synset_id":"smoke.n.01"},{"index":2,"sense_key":"smoke%1:22:00::","synset_id":"smoke.n.02"},{"index":3,"sense_key":"smoke%1:10:00::","synset_id":"smoke.n.03"},{"index":4,"sense_key":"smoke%1:07:00::","synset_id":"smoke.n.04"},{"index":5,"sense_key":"smoke%1:06:00::","synset_id":"roll_of_tobacco.n.01"},{"index":6,"sense_key":"smoke%1:06:01::","synset_id":"pot.n.09"},{"index":7,"sense_key":"smoke%1:04:00::","synset_id":"smoke.n.07"},{"index":8,"sense_key":"smoke%1:04:02::","synset_id":"fastball.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"smoke%1:22:00::","call_ids":["senseval3.d002.s024.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"smoke%1:22:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":842,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001804,"input_uncached_usd":0.001684,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2322133340639994}
{"item_id":"senseval3.d002.s024.t009","gold_sense_keys":["air%1:15:00::","air%1:27:00::"],"candidates":[{"index":1,"sense_key":"air%1:27:00::","synset_id":"air.n.01"},{"index":2,"sense_key":"air%1:15:00::","synset_id":"air.n.02"},{"index":3,"sense_key":"air%1:07:00::","synset_id":"air.n.03"},{"index":4,"sense_key":"air%1:19:00::","synset_id":"breeze.n.01"},{"index":5,"sense_key":"air%1:15:01::","synset_id":"atmosphere.n.03"},{"index":6,"sense_key":"air%1:27:01::","synset_id":"air.n.06"},{"index":7,"sense_key":"air%1:10:01::","synset_id":"tune.n.01"},{"index":8,"sense_key":"air%1:10:02::","synset_id":"air.n.08"},{"index":9,"sense_key":"air%1:04:00::","synset_id":"air_travel.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"air%1:15:00::","call_ids":["senseval3.d002.s024.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"air%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":965,"cached_input_tokens":0,"output_tokens":132,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0032500000000000003,"input_uncached_usd":0.0019299999999999999,"input_cached_usd":0.0,"output_usd":0.0013200000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.703192500048317}
{"item_id":"senseval3.d002.s027.t001","gold_sense_keys":["fine%5:00:00:satisfactory:00"],"candidates":[{"index":1,"sense_key":"fine%5:00:00:satisfactory:00","synset_id":"all_right.s.01"},{"index":2,"sense_key":"fine%5:00:00:precise:00","synset_id":"fine.s.02"},{"index":3,"sense_key":"fine%5:00:00:thin:01","synset_id":"fine.s.03"},{"index":4,"sense_key":"fine%5:00:00:elegant:00","synset_id":"fine.s.04"},{"index":5,"sense_key":"fine%3:00:00::","synset_id":"fine.a.05"},{"index":6,"sense_key":"fine%5:00:00:pure:02","synset_id":"fine.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"fine%5:00:00:satisfactory:00","call_ids":["senseval3.d002.s027.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"fine%5:00:00:satisfactory:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":727,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001574,"input_uncached_usd":0.001454,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.526071541942656}
{"item_id":"senseval3.d002.s028.t000","gold_sense_keys":["building%1:06:00::"],"candidates":[{"index":1,"sense_key":"building%1:06:00::","synset_id":"building.n.01"},{"index":2,"sense_key":"building%1:04:00::","synset_id":"construction.n.01"},{"index":3,"sense_key":"building%1:04:01::","synset_id":"construction.n.07"},{"index":4,"sense_key":"building%1:14:00::","synset_id":"building.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"building%1:06:00::","call_ids":["senseval3.d002.s028.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"building%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":615,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00135,"input_uncached_usd":0.00123,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8420327920466661}
{"item_id":"senseval3.d002.s028.t001","gold_sense_keys":["shake%2:29:00::","shake%2:38:00::","shake%2:38:01::","shake%2:38:02::"],"candidates":[{"index":1,"sense_key":"shake%2:38:00::","synset_id":"shake.v.01"},{"index":2,"sense_key":"shake%2:29:00::","synset_id":"shake.v.02"},{"index":3,"sense_key":"shake%2:38:01::","synset_id":"judder.v.01"},{"index":4,"sense_key":"shake%2:38:02::","synset_id":"rock.v.01"},{"index":5,"sense_key":"shake%2:30:00::","synset_id":"shake.v.05"},{"index":6,"sense_key":"shake%2:37:00::","synset_id":"stimulate.v.03"},{"index":7,"sense_key":"shake%2:38:04::","synset_id":"shake.v.07"},{"index":8,"sense_key":"shake%2:30:01::","synset_id":"shake.v.08"},{"index":9,"sense_key":"shake%2:32:00::","synset_id":"shake.v.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"shake%2:38:01::","call_ids":["senseval3.d002.s028.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"shake%2:38:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":925,"cached_input_tokens":0,"output_tokens":296,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00481,"input_uncached_usd":0.0018499999999999999,"input_cached_usd":0.0,"output_usd":0.0029600000000000004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.79362387501169}
{"item_id":"senseval3.d002.s028.t002","gold_sense_keys":["like_hell%4:02:00::"],"candidates":[{"index":1,"sense_key":"like_hell%4:02:00::","synset_id":"like_hell.r.01"},{"index":2,"sense_key":"like_hell%4:02:01::","synset_id":"like_hell.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"like_hell%4:02:00::","call_ids":["senseval3.d002.s028.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"like_hell%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":511,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001142,"input_uncached_usd":0.001022,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0113070420920849}
{"item_id":"senseval3.d002.s028.t003","gold_sense_keys":["keep%2:42:07::"],"candidates":[{"index":1,"sense_key":"keep%2:42:00::","synset_id":"keep.v.01"},{"index":2,"sense_key":"keep%2:42:07::","synset_id":"continue.v.01"},{"index":3,"sense_key":"keep%2:40:00::","synset_id":"keep.v.03"},{"index":4,"sense_key":"keep%2:41:01::","synset_id":"prevent.v.02"},{"index":5,"sense_key":"keep%2:41:03::","synset_id":"observe.v.09"},{"index":6,"sense_key":"keep%2:31:00::","synset_id":"observe.v.08"},{"index":7,"sense_key":"keep%2:40:01::","synset_id":"keep.v.07"},{"index":8,"sense_key":"keep%2:32:00::","synset_id":"keep.v.08"},{"index":9,"sense_key":"keep%2:42:01::","synset_id":"keep.v.09"},{"index":10,"sense_key":"keep%2:41:02::","synset_id":"retain.v.02"},{"index":11,"sense_key":"keep%2:34:00::","synset_id":"sustain.v.04"},{"index":12,"sense_key":"keep%2:42:03::","synset_id":"keep.v.12"},{"index":13,"sense_key":"keep%2:41:05::","synset_id":"observe.v.06"},{"index":14,"sense_key":"keep%2:41:00::","synset_id":"restrain.v.01"},{"index":15,"sense_key":"keep%2:42:02::","synset_id":"keep.v.15"},{"index":16,"sense_key":"keep%2:42:12::","synset_id":"keep.v.16"},{"index":17,"sense_key":"keep%2:40:09::","synset_id":"keep_open.v.01"},{"index":18,"sense_key":"keep%2:40:13::","synset_id":"keep.v.18"},{"index":19,"sense_key":"keep%2:40:12::","synset_id":"keep.v.19"},{"index":20,"sense_key":"keep%2:40:10::","synset_id":"keep.v.20"},{"index":21,"sense_key":"keep%2:35:10::","synset_id":"keep.v.21"},{"index":22,"sense_key":"keep%2:30:10::","synset_id":"preserve.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"keep%2:42:07::","call_ids":["senseval3.d002.s028.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"keep%2:42:07::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1710,"cached_input_tokens":0,"output_tokens":88,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0043,"input_uncached_usd":0.00342,"input_cached_usd":0.0,"output_usd":0.00088,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.8323211249662563}
{"item_id":"senseval3.d002.s028.t004","gold_sense_keys":["get%2:30:00::"],"candidates":[{"index":1,"sense_key":"get%2:40:00::","synset_id":"get.v.01"},{"index":2,"sense_key":"get%2:30:00::","synset_id":"become.v.01"},{"index":3,"sense_key":"get%2:30:02::","synset_id":"get.v.03"},{"index":4,"sense_key":"get%2:30:03::","synset_id":"receive.v.02"},{"index":5,"sense_key":"get%2:38:00::","synset_id":"arrive.v.01"},{"index":6,"sense_key":"get%2:35:03::","synset_id":"bring.v.04"},{"index":7,"sense_key":"get%2:39:00::","synset_id":"experience.v.03"},{"index":8,"sense_key":"get%2:33:00::","synset_id":"pay_back.v.02"},{"index":9,"sense_key":"get%2:40:01::","synset_id":"have.v.17"},{"index":10,"sense_key":"get%2:32:00::","synset_id":"induce.v.02"},{"index":11,"sense_key":"get%2:35:09::","synset_id":"get.v.11"},{"index":12,"sense_key":"get%2:29:01::","synset_id":"grow.v.08"},{"index":13,"sense_key":"get%2:29:00::","synset_id":"contract.v.04"},{"index":14,"sense_key":"get%2:32:03::","synset_id":"get.v.14"},{"index":15,"sense_key":"get%2:30:01::","synset_id":"make.v.02"},{"index":16,"sense_key":"get%2:32:08::","synset_id":"drive.v.11"},{"index":17,"sense_key":"get%2:31:09::","synset_id":"catch.v.18"},{"index":18,"sense_key":"get%2:35:00::","synset_id":"catch.v.07"},{"index":19,"sense_key":"get%2:35:10::","synset_id":"get.v.19"},{"index":20,"sense_key":"get%2:31:03::","synset_id":"get.v.20"},{"index":21,"sense_key":"get%2:40:14::","synset_id":"get.v.21"},{"index":22,"sense_key":"get%2:40:13::","synset_id":"get.v.22"},{"index":23,"sense_key":"get%2:39:13::","synset_id":"catch.v.21"},{"index":24,"sense_key":"get%2:39:09::","synset_id":"catch.v.22"},{"index":25,"sense_key":"get%2:39:14::","synset_id":"get.v.25"},{"index":26,"sense_key":"get%2:38:15::","synset_id":"scram.v.01"},{"index":27,"sense_key":"get%2:38:12::","synset_id":"get.v.27"},{"index":28,"sense_key":"get%2:37:12::","synset_id":"get.v.28"},{"index":29,"sense_key":"get%2:37:11::","synset_id":"get.v.29"},{"index":30,"sense_key":"get%2:36:14::","synset_id":"catch.v.24"},{"index":31,"sense_key":"get%2:36:00::","synset_id":"draw.v.15"},{"index":32,"sense_key":"get%2:35:12::","synset_id":"get.v.32"},{"index":33,"sense_key":"get%2:31:10::","synset_id":"perplex.v.01"},{"index":34,"sense_key":"get%2:30:12::","synset_id":"get_down.v.07"},{"index":35,"sense_key":"get%2:29:11::","synset_id":"suffer.v.02"},{"index":36,"sense_key":"get%2:29:08::","synset_id":"beget.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"get%2:30:00::","call_ids":["senseval3.d002.s028.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"get%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2707,"cached_input_tokens":0,"output_tokens":78,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.006194,"input_uncached_usd":0.0054139999999999995,"input_cached_usd":0.0,"output_usd":0.0007800000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6285268339561298}
{"item_id":"senseval3.d002.s029.t001","gold_sense_keys":["explode%2:30:04::"],"candidates":[{"index":1,"sense_key":"explode%2:30:01::","synset_id":"explode.v.01"},{"index":2,"sense_key":"explode%2:30:00::","synset_id":"explode.v.02"},{"index":3,"sense_key":"explode%2:31:00::","synset_id":"explode.v.03"},{"index":4,"sense_key":"explode%2:30:03::","synset_id":"explode.v.04"},{"index":5,"sense_key":"explode%2:35:00::","synset_id":"explode.v.05"},{"index":6,"sense_key":"explode%2:32:02::","synset_id":"explode.v.06"},{"index":7,"sense_key":"explode%2:32:00::","synset_id":"explode.v.07"},{"index":8,"sense_key":"explode%2:31:01::","synset_id":"explode.v.08"},{"index":9,"sense_key":"explode%2:30:04::","synset_id":"detonate.v.02"},{"index":10,"sense_key":"explode%2:30:02::","synset_id":"explode.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"explode%2:30:04::","call_ids":["senseval3.d002.s029.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"explode%2:30:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":938,"cached_input_tokens":0,"output_tokens":100,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002876,"input_uncached_usd":0.0018759999999999998,"input_cached_usd":0.0,"output_usd":0.001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6741532500600442}
{"item_id":"senseval3.d002.s029.t002","gold_sense_keys":["burn%2:30:00::"],"candidates":[{"index":1,"sense_key":"burn%2:30:02::","synset_id":"burn.v.01"},{"index":2,"sense_key":"burn%2:43:01::","synset_id":"burn.v.02"},{"index":3,"sense_key":"burn%2:30:00::","synset_id":"burn.v.03"},{"index":4,"sense_key":"burn%2:39:00::","synset_id":"bite.v.02"},{"index":5,"sense_key":"burn%2:43:02::","synset_id":"burn.v.05"},{"index":6,"sense_key":"burn%2:37:00::","synset_id":"burn.v.06"},{"index":7,"sense_key":"burn%2:30:01::","synset_id":"burn.v.07"},{"index":8,"sense_key":"burn%2:41:00::","synset_id":"burn.v.08"},{"index":9,"sense_key":"burn%2:40:01::","synset_id":"burn.v.09"},{"index":10,"sense_key":"burn%2:39:04::","synset_id":"burn.v.10"},{"index":11,"sense_key":"burn%2:30:08::","synset_id":"cauterize.v.01"},{"index":12,"sense_key":"burn%2:29:00::","synset_id":"sunburn.v.01"},{"index":13,"sense_key":"burn%2:36:05::","synset_id":"cut.v.21"},{"index":14,"sense_key":"burn%2:34:00::","synset_id":"burn_off.v.01"},{"index":15,"sense_key":"burn%2:30:03::","synset_id":"burn.v.15"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"burn%2:30:00::","call_ids":["senseval3.d002.s029.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"burn%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1195,"cached_input_tokens":0,"output_tokens":118,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00357,"input_uncached_usd":0.0023899999999999998,"input_cached_usd":0.0,"output_usd":0.00118,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.742184958071448}
{"item_id":"senseval3.d002.s029.t004","gold_sense_keys":["thing%1:09:02::"],"candidates":[{"index":1,"sense_key":"thing%1:26:00::","synset_id":"thing.n.01"},{"index":2,"sense_key":"thing%1:04:00::","synset_id":"thing.n.02"},{"index":3,"sense_key":"thing%1:09:00::","synset_id":"thing.n.03"},{"index":4,"sense_key":"thing%1:06:00::","synset_id":"thing.n.04"},{"index":5,"sense_key":"thing%1:11:00::","synset_id":"thing.n.05"},{"index":6,"sense_key":"thing%1:09:02::","synset_id":"matter.n.01"},{"index":7,"sense_key":"thing%1:10:00::","synset_id":"thing.n.07"},{"index":8,"sense_key":"thing%1:06:01::","synset_id":"thing.n.08"},{"index":9,"sense_key":"thing%1:07:00::","synset_id":"thing.n.09"},{"index":10,"sense_key":"thing%1:09:01::","synset_id":"thing.n.10"},{"index":11,"sense_key":"thing%1:12:00::","synset_id":"thing.n.11"},{"index":12,"sense_key":"thing%1:03:00::","synset_id":"thing.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"thing%1:26:00::","call_ids":["senseval3.d002.s029.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"thing%1:26:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":918,"cached_input_tokens":0,"output_tokens":137,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003206,"input_uncached_usd":0.001836,"input_cached_usd":0.0,"output_usd":0.0013700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8146087089553475}
{"item_id":"senseval3.d002.s029.t005","gold_sense_keys":["peaceful%3:00:00::"],"candidates":[{"index":1,"sense_key":"peaceful%3:00:00::","synset_id":"peaceful.a.01"},{"index":2,"sense_key":"peaceful%5:00:00:nonviolent:00","synset_id":"passive.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"peaceful%3:00:00::","call_ids":["senseval3.d002.s029.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"peaceful%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":494,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001108,"input_uncached_usd":0.000988,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0415374579606578}
{"item_id":"senseval3.d002.s030.t000","gold_sense_keys":["lot%1:23:00::"],"candidates":[{"index":1,"sense_key":"lot%1:23:00::","synset_id":"batch.n.02"},{"index":2,"sense_key":"lot%1:15:00::","synset_id":"lot.n.02"},{"index":3,"sense_key":"lot%1:14:00::","synset_id":"set.n.05"},{"index":4,"sense_key":"lot%1:26:00::","synset_id":"fortune.n.04"},{"index":5,"sense_key":"lot%1:06:00::","synset_id":"draw.n.04"},{"index":6,"sense_key":"lot%1:14:01::","synset_id":"bunch.n.03"},{"index":7,"sense_key":"lot%1:18:00::","synset_id":"lot.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"lot%1:23:00::","call_ids":["senseval3.d002.s030.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"lot%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":842,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001804,"input_uncached_usd":0.001684,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9854361671023071}
{"item_id":"senseval3.d002.s030.t001","gold_sense_keys":["car%1:06:00::"],"candidates":[{"index":1,"sense_key":"car%1:06:00::","synset_id":"car.n.01"},{"index":2,"sense_key":"car%1:06:01::","synset_id":"car.n.02"},{"index":3,"sense_key":"car%1:06:03::","synset_id":"car.n.03"},{"index":4,"sense_key":"car%1:06:02::","synset_id":"car.n.04"},{"index":5,"sense_key":"car%1:06:04::","synset_id":"cable_car.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"car%1:06:00::","call_ids":["senseval3.d002.s030.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"car%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":673,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001466,"input_uncached_usd":0.001346,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3781120829517022}
{"item_id":"senseval3.d002.s030.t002","gold_sense_keys":["alarm%1:06:00::"],"candidates":[{"index":1,"sense_key":"alarm%1:12:00::","synset_id":"alarm.n.01"},{"index":2,"sense_key":"alarm%1:06:00::","synset_id":"alarm.n.02"},{"index":3,"sense_key":"alarm%1:10:00::","synset_id":"alarm.n.03"},{"index":4,"sense_key":"alarm%1:06:01::","synset_id":"alarm_clock.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"alarm%1:06:00::","call_ids":["senseval3.d002.s030.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"alarm%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":557,"cached_input_tokens":0,"output_tokens":105,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002164,"input_uncached_usd":0.001114,"input_cached_usd":0.0,"output_usd":0.0010500000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8768487919587642}
{"item_id":"senseval3.d002.s030.t003","gold_sense_keys":["go_off%2:30:00::"],"candidates":[{"index":1,"sense_key":"go_off%2:38:00::","synset_id":"abscond.v.01"},{"index":2,"sense_key":"go_off%2:30:00::","synset_id":"go_off.v.02"},{"index":3,"sense_key":"go_off%2:33:00::","synset_id":"fire.v.05"},{"index":4,"sense_key":"go_off%2:35:00::","synset_id":"go_off.v.04"},{"index":5,"sense_key":"go_off%2:30:01::","synset_id":"go_off.v.05"},{"index":6,"sense_key":"go_off%2:30:02::","synset_id":"implode.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"go_off%2:30:00::","call_ids":["senseval3.d002.s030.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"go_off%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":681,"cached_input_tokens":0,"output_tokens":48,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0018419999999999999,"input_uncached_usd":0.001362,"input_cached_usd":0.0,"output_usd":0.00048000000000000007,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2679599170805886}
{"item_id":"senseval3.d002.s031.t000","gold_sense_keys":["cat%1:05:00::"],"candidates":[{"index":1,"sense_key":"cat%1:05:00::","synset_id":"cat.n.01"},{"index":2,"sense_key":"cat%1:18:01::","synset_id":"guy.n.01"},{"index":3,"sense_key":"cat%1:18:00::","synset_id":"cat.n.03"},{"index":4,"sense_key":"cat%1:06:02::","synset_id":"kat.n.01"},{"index":5,"sense_key":"cat%1:06:00::","synset_id":"cat-o'-nine-tails.n.01"},{"index":6,"sense_key":"cat%1:06:01::","synset_id":"caterpillar.n.02"},{"index":7,"sense_key":"cat%1:05:02::","synset_id":"big_cat.n.01"},{"index":8,"sense_key":"cat%1:04:00::","synset_id":"computerized_tomography.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cat%1:05:00::","call_ids":["senseval3.d002.s031.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cat%1:05:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":924,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0019679999999999997,"input_uncached_usd":0.0018479999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4874311669263989}
{"item_id":"senseval3.d002.s031.t001","gold_sense_keys":["fine%5:00:00:satisfactory:00"],"candidates":[{"index":1,"sense_key":"fine%5:00:00:satisfactory:00","synset_id":"all_right.s.01"},{"index":2,"sense_key":"fine%5:00:00:precise:00","synset_id":"fine.s.02"},{"index":3,"sense_key":"fine%5:00:00:thin:01","synset_id":"fine.s.03"},{"index":4,"sense_key":"fine%5:00:00:elegant:00","synset_id":"fine.s.04"},{"index":5,"sense_key":"fine%3:00:00::","synset_id":"fine.a.05"},{"index":6,"sense_key":"fine%5:00:00:pure:02","synset_id":"fine.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"fine%5:00:00:satisfactory:00","call_ids":["senseval3.d002.s031.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"fine%5:00:00:satisfactory:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":724,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001568,"input_uncached_usd":0.0014479999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.106745291966945}
{"item_id":"senseval3.d002.s031.t002","gold_sense_keys":["nervous%5:00:00:troubled:00"],"candidates":[{"index":1,"sense_key":"nervous%5:00:00:tense:03","synset_id":"nervous.s.01"},{"index":2,"sense_key":"nervous%5:00:00:troubled:00","synset_id":"anxious.s.02"},{"index":3,"sense_key":"nervous%3:01:00::","synset_id":"nervous.a.03"},{"index":4,"sense_key":"nervous%5:00:00:excited:00","synset_id":"aflutter.s.01"},{"index":5,"sense_key":"nervous%5:00:00:excitable:00","synset_id":"skittish.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"nervous%5:00:00:troubled:00","call_ids":["senseval3.d002.s031.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"nervous%5:00:00:troubled:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":653,"cached_input_tokens":0,"output_tokens":195,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003256,"input_uncached_usd":0.0013059999999999999,"input_cached_usd":0.0,"output_usd":0.0019500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.5960482920054346}
{"item_id":"senseval3.d002.s034.t001","gold_sense_keys":["fire%1:11:00::"],"candidates":[{"index":1,"sense_key":"fire%1:11:00::","synset_id":"fire.n.01"},{"index":2,"sense_key":"fire%1:04:00::","synset_id":"fire.n.02"},{"index":3,"sense_key":"fire%1:22:00::","synset_id":"fire.n.03"},{"index":4,"sense_key":"fire%1:06:00::","synset_id":"fire.n.04"},{"index":5,"sense_key":"fire%1:27:01::","synset_id":"fire.n.05"},{"index":6,"sense_key":"fire%1:12:00::","synset_id":"ardor.n.03"},{"index":7,"sense_key":"fire%1:27:00::","synset_id":"fire.n.07"},{"index":8,"sense_key":"fire%1:11:01::","synset_id":"fire.n.08"},{"index":9,"sense_key":"fire%1:10:00::","synset_id":"fire.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"fire%1:11:00::","call_ids":["senseval3.d002.s034.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"fire%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":913,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001946,"input_uncached_usd":0.001826,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0426600000355393}
{"item_id":"senseval3.d002.s035.t001","gold_sense_keys":["fill%1:27:00::"],"candidates":[{"index":1,"sense_key":"fill%1:23:00::","synset_id":"fill.n.01"},{"index":2,"sense_key":"fill%1:27:00::","synset_id":"filling.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"fill%1:27:00::","call_ids":["senseval3.d002.s035.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"fill%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":454,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001558,"input_uncached_usd":0.000908,"input_cached_usd":0.0,"output_usd":0.0006500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.904758584103547}
{"item_id":"senseval3.d002.s035.t002","gold_sense_keys":["liquefy%2:30:00::","liquefy%2:43:00::"],"candidates":[{"index":1,"sense_key":"liquefy%2:43:00::","synset_id":"liquefy.v.01"},{"index":2,"sense_key":"liquefy%2:30:01::","synset_id":"liquefy.v.02"},{"index":3,"sense_key":"liquefy%2:30:00::","synset_id":"liquefy.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"liquefy%2:43:00::","call_ids":["senseval3.d002.s035.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"liquefy%2:43:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":523,"cached_input_tokens":0,"output_tokens":136,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002406,"input_uncached_usd":0.001046,"input_cached_usd":0.0,"output_usd":0.00136,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9617751670302823}
{"item_id":"senseval3.d002.s036.t000","gold_sense_keys":["woman%1:18:00::"],"candidates":[{"index":1,"sense_key":"woman%1:18:00::","synset_id":"woman.n.01"},{"index":2,"sense_key":"woman%1:18:01::","synset_id":"woman.n.02"},{"index":3,"sense_key":"woman%1:18:02::","synset_id":"charwoman.n.01"},{"index":4,"sense_key":"woman%1:14:00::","synset_id":"womanhood.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"woman%1:18:00::","call_ids":["senseval3.d002.s036.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"woman%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":622,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001364,"input_uncached_usd":0.0012439999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.471316665993072}
{"item_id":"senseval3.d002.s036.t003","gold_sense_keys":["able%3:00:00::"],"candidates":[{"index":1,"sense_key":"able%3:00:00::","synset_id":"able.a.01"},{"index":2,"sense_key":"able%5:00:00:competent:00","synset_id":"able.s.01"},{"index":3,"sense_key":"able%5:00:00:capable:00","synset_id":"able.s.02"},{"index":4,"sense_key":"able%5:00:00:fit:01","synset_id":"able.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"able%3:00:00::","call_ids":["senseval3.d002.s036.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"able%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":592,"cached_input_tokens":0,"output_tokens":104,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0022240000000000003,"input_uncached_usd":0.001184,"input_cached_usd":0.0,"output_usd":0.0010400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1994300419464707}
{"item_id":"senseval3.d002.s036.t004","gold_sense_keys":["walk%2:38:00::"],"candidates":[{"index":1,"sense_key":"walk%2:38:00::","synset_id":"walk.v.01"},{"index":2,"sense_key":"walk%2:38:01::","synset_id":"walk.v.02"},{"index":3,"sense_key":"walk%2:33:00::","synset_id":"walk.v.03"},{"index":4,"sense_key":"walk%2:38:03::","synset_id":"walk.v.04"},{"index":5,"sense_key":"walk%2:33:01::","synset_id":"walk.v.05"},{"index":6,"sense_key":"walk%2:41:00::","synset_id":"walk.v.06"},{"index":7,"sense_key":"walk%2:41:01::","synset_id":"walk.v.07"},{"index":8,"sense_key":"walk%2:38:02::","synset_id":"walk.v.08"},{"index":9,"sense_key":"walk%2:38:05::","synset_id":"walk.v.09"},{"index":10,"sense_key":"walk%2:38:04::","synset_id":"walk.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"walk%2:38:00::","call_ids":["senseval3.d002.s036.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"walk%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":819,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001758,"input_uncached_usd":0.001638,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2867005419684574}
{"item_id":"senseval3.d002.s036.t005","gold_sense_keys":["window%1:06:00::"],"candidates":[{"index":1,"sense_key":"window%1:06:00::","synset_id":"window.n.01"},{"index":2,"sense_key":"window%1:06:01::","synset_id":"window.n.02"},{"index":3,"sense_key":"window%1:06:02::","synset_id":"window.n.03"},{"index":4,"sense_key":"window%1:17:00::","synset_id":"window.n.04"},{"index":5,"sense_key":"window%1:28:00::","synset_id":"window.n.05"},{"index":6,"sense_key":"window%1:06:04::","synset_id":"windowpane.n.01"},{"index":7,"sense_key":"window%1:06:05::","synset_id":"window.n.07"},{"index":8,"sense_key":"window%1:06:03::","synset_id":"window.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"window%1:06:00::","call_ids":["senseval3.d002.s036.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"window%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":788,"cached_input_tokens":0,"output_tokens":93,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002506,"input_uncached_usd":0.001576,"input_cached_usd":0.0,"output_usd":0.00093,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.750547625008039}
{"item_id":"senseval3.d002.s036.t007","gold_sense_keys":["floor%1:06:01::"],"candidates":[{"index":1,"sense_key":"floor%1:06:00::","synset_id":"floor.n.01"},{"index":2,"sense_key":"floor%1:06:01::","synset_id":"floor.n.02"},{"index":3,"sense_key":"floor%1:10:00::","synset_id":"floor.n.03"},{"index":4,"sense_key":"floor%1:17:01::","synset_id":"floor.n.04"},{"index":5,"sense_key":"floor%1:17:00::","synset_id":"floor.n.05"},{"index":6,"sense_key":"floor%1:17:02::","synset_id":"floor.n.06"},{"index":7,"sense_key":"floor%1:14:00::","synset_id":"floor.n.07"},{"index":8,"sense_key":"floor%1:07:00::","synset_id":"floor.n.08"},{"index":9,"sense_key":"floor%1:06:03::","synset_id":"floor.n.09"},{"index":10,"sense_key":"floor%1:06:02::","synset_id":"floor.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"floor%1:06:01::","call_ids":["senseval3.d002.s036.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"floor%1:06:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":905,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00193,"input_uncached_usd":0.00181,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.7822357499971986}
{"item_id":"senseval3.d002.s036.t008","gold_sense_keys":["street%1:06:00::"],"candidates":[{"index":1,"sense_key":"street%1:06:00::","synset_id":"street.n.01"},{"index":2,"sense_key":"street%1:06:01::","synset_id":"street.n.02"},{"index":3,"sense_key":"street%1:26:00::","synset_id":"street.n.03"},{"index":4,"sense_key":"street%1:26:01::","synset_id":"street.n.04"},{"index":5,"sense_key":"street%1:14:00::","synset_id":"street.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"street%1:06:00::","call_ids":["senseval3.d002.s036.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"street%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":650,"cached_input_tokens":0,"output_tokens":111,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00241,"input_uncached_usd":0.0013,"input_cached_usd":0.0,"output_usd":0.00111,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.042720124940388}
{"item_id":"senseval3.d002.s036.t009","gold_sense_keys":["level%1:07:01::"],"candidates":[{"index":1,"sense_key":"level%1:07:00::","synset_id":"degree.n.01"},{"index":2,"sense_key":"level%1:26:01::","synset_id":"grade.n.02"},{"index":3,"sense_key":"level%1:26:00::","synset_id":"degree.n.02"},{"index":4,"sense_key":"level%1:07:01::","synset_id":"level.n.04"},{"index":5,"sense_key":"level%1:06:00::","synset_id":"level.n.05"},{"index":6,"sense_key":"level%1:06:03::","synset_id":"horizontal_surface.n.01"},{"index":7,"sense_key":"level%1:09:00::","synset_id":"level.n.07"},{"index":8,"sense_key":"level%1:06:01::","synset_id":"floor.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"level%1:07:01::","call_ids":["senseval3.d002.s036.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"level%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":861,"cached_input_tokens":0,"output_tokens":89,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002612,"input_uncached_usd":0.001722,"input_cached_usd":0.0,"output_usd":0.0008900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8102031250018626}
{"item_id":"senseval3.d002.s037.t000","gold_sense_keys":["house%1:06:00::"],"candidates":[{"index":1,"sense_key":"house%1:06:00::","synset_id":"house.n.01"},{"index":2,"sense_key":"house%1:14:01::","synset_id":"firm.n.01"},{"index":3,"sense_key":"house%1:14:04::","synset_id":"house.n.03"},{"index":4,"sense_key":"house%1:14:03::","synset_id":"house.n.04"},{"index":5,"sense_key":"house%1:14:02::","synset_id":"house.n.05"},{"index":6,"sense_key":"house%1:14:00::","synset_id":"house.n.06"},{"index":7,"sense_key":"house%1:04:00::","synset_id":"house.n.07"},{"index":8,"sense_key":"house%1:15:00::","synset_id":"sign_of_the_zodiac.n.01"},{"index":9,"sense_key":"house%1:14:06::","synset_id":"house.n.09"},{"index":10,"sense_key":"house%1:14:05::","synset_id":"family.n.01"},{"index":11,"sense_key":"house%1:06:02::","synset_id":"theater.n.01"},{"index":12,"sense_key":"house%1:06:01::","synset_id":"house.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"house%1:06:00::","call_ids":["senseval3.d002.s037.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"house%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":982,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002084,"input_uncached_usd":0.001964,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.040270625031553}
{"item_id":"senseval3.d002.s037.t001","gold_sense_keys":["ground%1:17:00::"],"candidates":[{"index":1,"sense_key":"ground%1:17:00::","synset_id":"land.n.04"},{"index":2,"sense_key":"ground%1:16:00::","synset_id":"reason.n.01"},{"index":3,"sense_key":"ground%1:27:00::","synset_id":"earth.n.02"},{"index":4,"sense_key":"ground%1:24:00::","synset_id":"footing.n.02"},{"index":5,"sense_key":"ground%1:15:00::","synset_id":"ground.n.05"},{"index":6,"sense_key":"ground%1:09:00::","synset_id":"background.n.02"},{"index":7,"sense_key":"ground%1:17:01::","synset_id":"land.n.02"},{"index":8,"sense_key":"ground%1:09:01::","synset_id":"ground.n.08"},{"index":9,"sense_key":"ground%1:06:02::","synset_id":"ground.n.09"},{"index":10,"sense_key":"ground%1:06:01::","synset_id":"ground.n.10"},{"index":11,"sense_key":"ground%1:06:00::","synset_id":"flat_coat.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"ground%1:27:00::","call_ids":["senseval3.d002.s037.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"ground%1:27:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1074,"cached_input_tokens":0,"output_tokens":167,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0038179999999999998,"input_uncached_usd":0.0021479999999999997,"input_cached_usd":0.0,"output_usd":0.00167,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0874787910142913}
{"item_id":"senseval3.d002.s040.t000","gold_sense_keys":["drive%2:38:01::"],"candidates":[{"index":1,"sense_key":"drive%2:38:01::","synset_id":"drive.v.01"},{"index":2,"sense_key":"drive%2:38:00::","synset_id":"drive.v.02"},{"index":3,"sense_key":"drive%2:38:02::","synset_id":"drive.v.03"},{"index":4,"sense_key":"drive%2:35:00::","synset_id":"force.v.06"},{"index":5,"sense_key":"drive%2:41:02::","synset_id":"drive.v.05"},{"index":6,"sense_key":"drive%2:35:11::","synset_id":"repel.v.01"},{"index":7,"sense_key":"drive%2:36:00::","synset_id":"drive.v.07"},{"index":8,"sense_key":"drive%2:35:07::","synset_id":"drive.v.08"},{"index":9,"sense_key":"drive%2:35:01::","synset_id":"drive.v.09"},{"index":10,"sense_key":"drive%2:41:00::","synset_id":"tug.v.02"},{"index":11,"sense_key":"drive%2:32:00::","synset_id":"drive.v.11"},{"index":12,"sense_key":"drive%2:42:00::","synset_id":"drive.v.12"},{"index":13,"sense_key":"drive%2:41:13::","synset_id":"drive.v.13"},{"index":14,"sense_key":"drive%2:38:09::","synset_id":"drive.v.14"},{"index":15,"sense_key":"drive%2:38:10::","synset_id":"drive.v.15"},{"index":16,"sense_key":"drive%2:38:11::","synset_id":"drive.v.16"},{"index":17,"sense_key":"drive%2:35:03::","synset_id":"drive.v.17"},{"index":18,"sense_key":"drive%2:35:02::","synset_id":"drive.v.18"},{"index":19,"sense_key":"drive%2:35:06::","synset_id":"drive.v.19"},{"index":20,"sense_key":"drive%2:34:07::","synset_id":"drive.v.20"},{"index":21,"sense_key":"drive%2:33:01::","synset_id":"drive.v.21"},{"index":22,"sense_key":"drive%2:33:00::","synset_id":"drive.v.22"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"drive%2:38:01::","call_ids":["senseval3.d002.s040.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"drive%2:38:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1581,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003792,"input_uncached_usd":0.003162,"input_cached_usd":0.0,"output_usd":0.00063,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2452397090382874}
{"item_id":"senseval3.d002.s040.t001","gold_sense_keys":["truck%1:06:00::"],"candidates":[{"index":1,"sense_key":"truck%1:06:00::","synset_id":"truck.n.01"},{"index":2,"sense_key":"truck%1:06:01::","synset_id":"hand_truck.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"truck%1:06:00::","call_ids":["senseval3.d002.s040.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"truck%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":504,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001128,"input_uncached_usd":0.001008,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5997121670516208}
{"item_id":"senseval3.d002.s040.t002","gold_sense_keys":["stop%2:38:00::"],"candidates":[{"index":1,"sense_key":"stop%2:38:00::","synset_id":"stop.v.01"},{"index":2,"sense_key":"stop%2:42:00::","synset_id":"discontinue.v.01"},{"index":3,"sense_key":"stop%2:41:00::","synset_id":"stop.v.03"},{"index":4,"sense_key":"stop%2:38:02::","synset_id":"stop.v.04"},{"index":5,"sense_key":"stop%2:38:01::","synset_id":"stop.v.05"},{"index":6,"sense_key":"stop%2:30:05::","synset_id":"break.v.10"},{"index":7,"sense_key":"stop%2:33:00::","synset_id":"check.v.18"},{"index":8,"sense_key":"stop%2:35:00::","synset_id":"intercept.v.01"},{"index":9,"sense_key":"stop%2:42:13::","synset_id":"end.v.01"},{"index":10,"sense_key":"stop%2:35:12::","synset_id":"barricade.v.01"},{"index":11,"sense_key":"stop%2:30:12::","synset_id":"hold_on.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"stop%2:38:00::","call_ids":["senseval3.d002.s040.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"stop%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1143,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0024059999999999997,"input_uncached_usd":0.002286,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9237329170573503}
{"item_id":"senseval3.d002.s040.t003","gold_sense_keys":["red_light%1:10:00::"],"candidates":[{"index":1,"sense_key":"red_light%1:10:01::","synset_id":"red_light.n.01"},{"index":2,"sense_key":"red_light%1:10:00::","synset_id":"red_light.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"red_light%1:10:00::","call_ids":["senseval3.d002.s040.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"red_light%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":453,"cached_input_tokens":0,"output_tokens":70,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0016060000000000002,"input_uncached_usd":0.000906,"input_cached_usd":0.0,"output_usd":0.0007000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9103689580224454}
{"item_id":"senseval3.d002.s040.t004","gold_sense_keys":["corner%1:06:00::"],"candidates":[{"index":1,"sense_key":"corner%1:15:02::","synset_id":"corner.n.01"},{"index":2,"sense_key":"corner%1:15:00::","synset_id":"corner.n.02"},{"index":3,"sense_key":"corner%1:06:02::","synset_id":"corner.n.03"},{"index":4,"sense_key":"corner%1:06:00::","synset_id":"corner.n.04"},{"index":5,"sense_key":"corner%1:15:01::","synset_id":"corner.n.05"},{"index":6,"sense_key":"corner%1:25:00::","synset_id":"recess.n.02"},{"index":7,"sense_key":"corner%1:26:00::","synset_id":"corner.n.07"},{"index":8,"sense_key":"corner%1:26:01::","synset_id":"corner.n.08"},{"index":9,"sense_key":"corner%1:17:00::","synset_id":"corner.n.09"},{"index":10,"sense_key":"corner%1:15:03::","synset_id":"corner.n.10"},{"index":11,"sense_key":"corner%1:06:01::","synset_id":"corner.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"corner%1:06:00::","call_ids":["senseval3.d002.s040.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"corner%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":969,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002058,"input_uncached_usd":0.0019379999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8275437079137191}
{"item_id":"senseval3.d002.s040.t006","gold_sense_keys":["hit%2:35:05::"],"candidates":[{"index":1,"sense_key":"hit%2:35:03::","synset_id":"hit.v.01"},{"index":2,"sense_key":"hit%2:35:01::","synset_id":"hit.v.02"},{"index":3,"sense_key":"hit%2:35:00::","synset_id":"hit.v.03"},{"index":4,"sense_key":"hit%2:38:00::","synset_id":"reach.v.01"},{"index":5,"sense_key":"hit%2:35:05::","synset_id":"hit.v.05"},{"index":6,"sense_key":"hit%2:33:01::","synset_id":"shoot.v.01"},{"index":7,"sense_key":"hit%2:40:05::","synset_id":"stumble.v.03"},{"index":8,"sense_key":"hit%2:33:02::","synset_id":"score.v.01"},{"index":9,"sense_key":"hit%2:39:00::","synset_id":"hit.v.09"},{"index":10,"sense_key":"hit%2:33:00::","synset_id":"strike.v.04"},{"index":11,"sense_key":"hit%2:41:00::","synset_id":"murder.v.01"},{"index":12,"sense_key":"hit%2:38:02::","synset_id":"hit.v.12"},{"index":13,"sense_key":"hit%2:38:07::","synset_id":"reach.v.02"},{"index":14,"sense_key":"hit%2:35:02::","synset_id":"strike.v.10"},{"index":15,"sense_key":"hit%2:34:00::","synset_id":"hit.v.15"},{"index":16,"sense_key":"hit%2:33:03::","synset_id":"hit.v.16"},{"index":17,"sense_key":"hit%2:32:12::","synset_id":"hit.v.17"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"hit%2:35:05::","call_ids":["senseval3.d002.s040.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"hit%2:35:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1461,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003642,"input_uncached_usd":0.0029219999999999997,"input_cached_usd":0.0,"output_usd":0.00072,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5009090420790017}
{"item_id":"senseval3.d002.s041.t000","gold_sense_keys":["part%1:24:00::"],"candidates":[{"index":1,"sense_key":"part%1:24:00::","synset_id":"part.n.01"},{"index":2,"sense_key":"part%1:06:00::","synset_id":"part.n.02"},{"index":3,"sense_key":"part%1:17:00::","synset_id":"part.n.03"},{"index":4,"sense_key":"part%1:09:02::","synset_id":"part.n.04"},{"index":5,"sense_key":"part%1:15:00::","synset_id":"region.n.01"},{"index":6,"sense_key":"part%1:04:01::","synset_id":"function.n.03"},{"index":7,"sense_key":"part%1:09:01::","synset_id":"character.n.04"},{"index":8,"sense_key":"part%1:21:00::","synset_id":"share.n.01"},{"index":9,"sense_key":"part%1:09:00::","synset_id":"part.n.09"},{"index":10,"sense_key":"part%1:08:01::","synset_id":"part.n.10"},{"index":11,"sense_key":"part%1:10:00::","synset_id":"part.n.11"},{"index":12,"sense_key":"part%1:04:03::","synset_id":"contribution.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"part%1:24:00::","call_ids":["senseval3.d002.s041.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"part%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1182,"cached_input_tokens":0,"output_tokens":211,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0044740000000000005,"input_uncached_usd":0.0023639999999999998,"input_cached_usd":0.0,"output_usd":0.0021100000000000003,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2629422909813}
{"item_id":"senseval3.d002.s041.t001","gold_sense_keys":["watch%2:39:00::","watch%2:39:02::","watch%2:41:00::"],"candidates":[{"index":1,"sense_key":"watch%2:39:00::","synset_id":"watch.v.01"},{"index":2,"sense_key":"watch%2:41:00::","synset_id":"watch.v.02"},{"index":3,"sense_key":"watch%2:39:05::","synset_id":"watch.v.03"},{"index":4,"sense_key":"watch%2:39:02::","synset_id":"watch.v.04"},{"index":5,"sense_key":"watch%2:39:01::","synset_id":"watch.v.05"},{"index":6,"sense_key":"watch%2:39:03::","synset_id":"watch.v.06"},{"index":7,"sense_key":"watch%2:32:09::","synset_id":"determine.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"watch%2:39:00::","call_ids":["senseval3.d002.s041.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"watch%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":810,"cached_input_tokens":0,"output_tokens":159,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00321,"input_uncached_usd":0.00162,"input_cached_usd":0.0,"output_usd":0.00159,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.440379709005356}
{"item_id":"senseval3.d002.s041.t003","gold_sense_keys":["wave%2:38:00::"],"candidates":[{"index":1,"sense_key":"wave%2:32:00::","synset_id":"beckon.v.01"},{"index":2,"sense_key":"wave%2:35:00::","synset_id":"brandish.v.01"},{"index":3,"sense_key":"wave%2:38:00::","synset_id":"roll.v.11"},{"index":4,"sense_key":"wave%2:35:01::","synset_id":"curl.v.04"},{"index":5,"sense_key":"wave%2:29:00::","synset_id":"wave.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"wave%2:38:00::","call_ids":["senseval3.d002.s041.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"wave%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":666,"cached_input_tokens":0,"output_tokens":104,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002372,"input_uncached_usd":0.0013319999999999999,"input_cached_usd":0.0,"output_usd":0.0010400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6433747500414029}
{"item_id":"senseval3.d002.s041.t004","gold_sense_keys":["head%1:08:00::"],"candidates":[{"index":1,"sense_key":"head%1:08:00::","synset_id":"head.n.01"},{"index":2,"sense_key":"head%1:05:00::","synset_id":"head.n.02"},{"index":3,"sense_key":"head%1:09:00::","synset_id":"mind.n.01"},{"index":4,"sense_key":"head%1:18:00::","synset_id":"head.n.04"},{"index":5,"sense_key":"head%1:14:00::","synset_id":"head.n.05"},{"index":6,"sense_key":"head%1:19:00::","synset_id":"head.n.06"},{"index":7,"sense_key":"head%1:15:00::","synset_id":"head.n.07"},{"index":8,"sense_key":"head%1:15:01::","synset_id":"fountainhead.n.02"},{"index":9,"sense_key":"head%1:10:03::","synset_id":"head.n.09"},{"index":10,"sense_key":"head%1:26:00::","synset_id":"head.n.10"},{"index":11,"sense_key":"head%1:23:00::","synset_id":"head.n.11"},{"index":12,"sense_key":"head%1:20:00::","synset_id":"capitulum.n.01"},{"index":13,"sense_key":"head%1:18:01::","synset_id":"principal.n.02"},{"index":14,"sense_key":"head%1:18:03::","synset_id":"head.n.14"},{"index":15,"sense_key":"head%1:18:02::","synset_id":"head.n.15"},{"index":16,"sense_key":"head%1:17:02::","synset_id":"promontory.n.01"},{"index":17,"sense_key":"head%1:17:01::","synset_id":"head.n.17"},{"index":18,"sense_key":"head%1:17:00::","synset_id":"head.n.18"},{"index":19,"sense_key":"head%1:15:02::","synset_id":"forefront.n.01"},{"index":20,"sense_key":"head%1:11:01::","synset_id":"pass.n.09"},{"index":21,"sense_key":"head%1:11:00::","synset_id":"headway.n.02"},{"index":22,"sense_key":"head%1:10:01::","synset_id":"point.n.20"},{"index":23,"sense_key":"head%1:10:02::","synset_id":"question.n.02"},{"index":24,"sense_key":"head%1:10:00::","synset_id":"heading.n.01"},{"index":25,"sense_key":"head%1:08:01::","synset_id":"head.n.25"},{"index":26,"sense_key":"head%1:08:02::","synset_id":"head.n.26"},{"index":27,"sense_key":"head%1:06:05::","synset_id":"read/write_head.n.01"},{"index":28,"sense_key":"head%1:06:04::","synset_id":"head.n.28"},{"index":29,"sense_key":"head%1:06:03::","synset_id":"head.n.29"},{"index":30,"sense_key":"head%1:06:02::","synset_id":"head.n.30"},{"index":31,"sense_key":"head%1:06:00::","synset_id":"head.n.31"},{"index":32,"sense_key":"head%1:06:01::","synset_id":"drumhead.n.01"},{"index":33,"sense_key":"head%1:04:00::","synset_id":"oral_sex.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"head%1:08:00::","call_ids":["senseval3.d002.s041.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"head%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2193,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0045060000000000005,"input_uncached_usd":0.004386,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6380549580790102}
{"item_id":"senseval3.d002.s041.t005","gold_sense_keys":["way%1:04:01::","way%1:07:01::"],"candidates":[{"index":1,"sense_key":"way%1:07:01::","synset_id":"manner.n.01"},{"index":2,"sense_key":"way%1:04:01::","synset_id":"means.n.01"},{"index":3,"sense_key":"way%1:15:00::","synset_id":"direction.n.01"},{"index":4,"sense_key":"way%1:26:00::","synset_id":"way.n.04"},{"index":5,"sense_key":"way%1:04:00::","synset_id":"way.n.05"},{"index":6,"sense_key":"way%1:06:00::","synset_id":"way.n.06"},{"index":7,"sense_key":"way%1:04:02::","synset_id":"way.n.07"},{"index":8,"sense_key":"way%1:23:00::","synset_id":"room.n.02"},{"index":9,"sense_key":"way%1:07:02::","synset_id":"way.n.09"},{"index":10,"sense_key":"way%1:09:00::","synset_id":"way.n.10"},{"index":11,"sense_key":"way%1:09:01::","synset_id":"way.n.11"},{"index":12,"sense_key":"way%1:21:00::","synset_id":"way.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"way%1:04:01::","call_ids":["senseval3.d002.s041.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"way%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":990,"cached_input_tokens":0,"output_tokens":120,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00318,"input_uncached_usd":0.00198,"input_cached_usd":0.0,"output_usd":0.0012000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.852957124938257}
{"item_id":"senseval3.d002.s041.t006","gold_sense_keys":["drive%2:38:00::"],"candidates":[{"index":1,"sense_key":"drive%2:38:01::","synset_id":"drive.v.01"},{"index":2,"sense_key":"drive%2:38:00::","synset_id":"drive.v.02"},{"index":3,"sense_key":"drive%2:38:02::","synset_id":"drive.v.03"},{"index":4,"sense_key":"drive%2:35:00::","synset_id":"force.v.06"},{"index":5,"sense_key":"drive%2:41:02::","synset_id":"drive.v.05"},{"index":6,"sense_key":"drive%2:35:11::","synset_id":"repel.v.01"},{"index":7,"sense_key":"drive%2:36:00::","synset_id":"drive.v.07"},{"index":8,"sense_key":"drive%2:35:07::","synset_id":"drive.v.08"},{"index":9,"sense_key":"drive%2:35:01::","synset_id":"drive.v.09"},{"index":10,"sense_key":"drive%2:41:00::","synset_id":"tug.v.02"},{"index":11,"sense_key":"drive%2:32:00::","synset_id":"drive.v.11"},{"index":12,"sense_key":"drive%2:42:00::","synset_id":"drive.v.12"},{"index":13,"sense_key":"drive%2:41:13::","synset_id":"drive.v.13"},{"index":14,"sense_key":"drive%2:38:09::","synset_id":"drive.v.14"},{"index":15,"sense_key":"drive%2:38:10::","synset_id":"drive.v.15"},{"index":16,"sense_key":"drive%2:38:11::","synset_id":"drive.v.16"},{"index":17,"sense_key":"drive%2:35:03::","synset_id":"drive.v.17"},{"index":18,"sense_key":"drive%2:35:02::","synset_id":"drive.v.18"},{"index":19,"sense_key":"drive%2:35:06::","synset_id":"drive.v.19"},{"index":20,"sense_key":"drive%2:34:07::","synset_id":"drive.v.20"},{"index":21,"sense_key":"drive%2:33:01::","synset_id":"drive.v.21"},{"index":22,"sense_key":"drive%2:33:00::","synset_id":"drive.v.22"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"drive%2:38:00::","call_ids":["senseval3.d002.s041.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"drive%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1573,"cached_input_tokens":0,"output_tokens":224,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.005386,"input_uncached_usd":0.003146,"input_cached_usd":0.0,"output_usd":0.0022400000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2339370410190895}
{"item_id":"senseval3.d002.s044.t000","gold_sense_keys":["foot%1:23:00::"],"candidates":[{"index":1,"sense_key":"foot%1:08:01::","synset_id":"foot.n.01"},{"index":2,"sense_key":"foot%1:23:00::","synset_id":"foot.n.02"},{"index":3,"sense_key":"foot%1:15:00::","synset_id":"foot.n.03"},{"index":4,"sense_key":"foot%1:05:01::","synset_id":"animal_foot.n.01"},{"index":5,"sense_key":"foot%1:06:00::","synset_id":"foundation.n.03"},{"index":6,"sense_key":"foot%1:05:00::","synset_id":"foot.n.06"},{"index":7,"sense_key":"foot%1:04:00::","synset_id":"foot.n.07"},{"index":8,"sense_key":"foot%1:18:00::","synset_id":"foot.n.08"},{"index":9,"sense_key":"foot%1:14:00::","synset_id":"infantry.n.01"},{"index":10,"sense_key":"foot%1:10:00::","synset_id":"metrical_foot.n.01"},{"index":11,"sense_key":"foot%1:06:01::","synset_id":"foot.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"foot%1:23:00::","call_ids":["senseval3.d002.s044.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"foot%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1025,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0021699999999999996,"input_uncached_usd":0.0020499999999999997,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.17552883294411}
{"item_id":"senseval3.d002.s044.t001","gold_sense_keys":["pier%1:06:00::"],"candidates":[{"index":1,"sense_key":"pier%1:06:00::","synset_id":"pier.n.01"},{"index":2,"sense_key":"pier%1:06:02::","synset_id":"pier.n.02"},{"index":3,"sense_key":"pier%1:06:01::","synset_id":"pier.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"pier%1:06:00::","call_ids":["senseval3.d002.s044.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"pier%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":512,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001144,"input_uncached_usd":0.001024,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9429420829983428}
{"item_id":"senseval3.d002.s045.t001","gold_sense_keys":["dramatic%5:00:00:impressive:00"],"candidates":[{"index":1,"sense_key":"dramatic%3:00:00::","synset_id":"dramatic.a.01"},{"index":2,"sense_key":"dramatic%5:00:00:impressive:00","synset_id":"dramatic.s.01"},{"index":3,"sense_key":"dramatic%3:01:00::","synset_id":"dramatic.a.03"},{"index":4,"sense_key":"dramatic%3:00:02::","synset_id":"dramatic.a.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"dramatic%5:00:00:impressive:00","call_ids":["senseval3.d002.s045.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"dramatic%5:00:00:impressive:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":647,"cached_input_tokens":0,"output_tokens":75,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002044,"input_uncached_usd":0.001294,"input_cached_usd":0.0,"output_usd":0.00075,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7447424579877406}
{"item_id":"senseval3.d002.s046.t001","gold_sense_keys":["hairline%1:10:00::"],"candidates":[{"index":1,"sense_key":"hairline%1:10:00::","synset_id":"hairline.n.01"},{"index":2,"sense_key":"hairline%1:08:00::","synset_id":"hairline.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"hairline%1:10:00::","call_ids":["senseval3.d002.s046.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"hairline%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":400,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0009199999999999999,"input_uncached_usd":0.0007999999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1743196250172332}
{"item_id":"senseval3.d002.s046.t002","gold_sense_keys":["crack%1:07:00::"],"candidates":[{"index":1,"sense_key":"crack%1:17:00::","synset_id":"crack.n.01"},{"index":2,"sense_key":"crack%1:06:00::","synset_id":"gap.n.03"},{"index":3,"sense_key":"crack%1:25:00::","synset_id":"crevice.n.01"},{"index":4,"sense_key":"crack%1:11:00::","synset_id":"crack.n.04"},{"index":5,"sense_key":"crack%1:26:00::","synset_id":"shot.n.04"},{"index":6,"sense_key":"crack%1:10:00::","synset_id":"wisecrack.n.01"},{"index":7,"sense_key":"crack%1:07:00::","synset_id":"crack.n.07"},{"index":8,"sense_key":"crack%1:06:01::","synset_id":"crack.n.08"},{"index":9,"sense_key":"crack%1:04:01::","synset_id":"crack.n.09"},{"index":10,"sense_key":"crack%1:04:00::","synset_id":"fracture.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"crack%1:07:00::","call_ids":["senseval3.d002.s046.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"crack%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":880,"cached_input_tokens":0,"output_tokens":158,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00334,"input_uncached_usd":0.0017599999999999998,"input_cached_usd":0.0,"output_usd":0.00158,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9755992499412969}
{"item_id":"senseval3.d002.s047.t000","gold_sense_keys":["ruin%2:41:00::"],"candidates":[{"index":1,"sense_key":"ruin%2:35:00::","synset_id":"destroy.v.02"},{"index":2,"sense_key":"ruin%2:41:00::","synset_id":"ruin.v.02"},{"index":3,"sense_key":"ruin%2:40:00::","synset_id":"bankrupt.v.01"},{"index":4,"sense_key":"ruin%2:35:01::","synset_id":"ruin.v.04"},{"index":5,"sense_key":"ruin%2:35:02::","synset_id":"deflower.v.01"},{"index":6,"sense_key":"ruin%2:30:00::","synset_id":"ruin.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"ruin%2:41:00::","call_ids":["senseval3.d002.s047.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"ruin%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":669,"cached_input_tokens":0,"output_tokens":119,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002528,"input_uncached_usd":0.001338,"input_cached_usd":0.0,"output_usd":0.00119,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1063758339732885}
{"item_id":"senseval3.d002.s047.t001","gold_sense_keys":["damn%5:00:02:cursed:00"],"candidates":[{"index":1,"sense_key":"damn%5:00:00:cursed:00","synset_id":"damn.s.01"},{"index":2,"sense_key":"damn%5:00:02:cursed:00","synset_id":"blasted.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"damn%5:00:02:cursed:00","call_ids":["senseval3.d002.s047.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"damn%5:00:02:cursed:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":465,"cached_input_tokens":0,"output_tokens":100,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0019299999999999999,"input_uncached_usd":0.0009299999999999999,"input_cached_usd":0.0,"output_usd":0.001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9096837079850957}
{"item_id":"senseval3.d002.s050.t001","gold_sense_keys":["second%5:00:00:ordinal:00"],"candidates":[{"index":1,"sense_key":"second%5:00:00:ordinal:00","synset_id":"second.s.01"},{"index":2,"sense_key":"second%3:00:00::","synset_id":"second.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"second%5:00:00:ordinal:00","call_ids":["senseval3.d002.s050.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"second%5:00:00:ordinal:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":459,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001038,"input_uncached_usd":0.000918,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4933934580767527}
{"item_id":"senseval3.d002.s050.t002","gold_sense_keys":["floor%1:06:01::"],"candidates":[{"index":1,"sense_key":"floor%1:06:00::","synset_id":"floor.n.01"},{"index":2,"sense_key":"floor%1:06:01::","synset_id":"floor.n.02"},{"index":3,"sense_key":"floor%1:10:00::","synset_id":"floor.n.03"},{"index":4,"sense_key":"floor%1:17:01::","synset_id":"floor.n.04"},{"index":5,"sense_key":"floor%1:17:00::","synset_id":"floor.n.05"},{"index":6,"sense_key":"floor%1:17:02::","synset_id":"floor.n.06"},{"index":7,"sense_key":"floor%1:14:00::","synset_id":"floor.n.07"},{"index":8,"sense_key":"floor%1:07:00::","synset_id":"floor.n.08"},{"index":9,"sense_key":"floor%1:06:03::","synset_id":"floor.n.09"},{"index":10,"sense_key":"floor%1:06:02::","synset_id":"floor.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"floor%1:06:01::","call_ids":["senseval3.d002.s050.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"floor%1:06:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":903,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001926,"input_uncached_usd":0.0018059999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0198343750089407}
{"item_id":"senseval3.d002.s051.t000","gold_sense_keys":["hear%2:39:00::"],"candidates":[{"index":1,"sense_key":"hear%2:39:00::","synset_id":"hear.v.01"},{"index":2,"sense_key":"hear%2:31:00::","synset_id":"learn.v.02"},{"index":3,"sense_key":"hear%2:41:00::","synset_id":"hear.v.03"},{"index":4,"sense_key":"hear%2:39:01::","synset_id":"hear.v.04"},{"index":5,"sense_key":"hear%2:39:05::","synset_id":"listen.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"hear%2:39:00::","call_ids":["senseval3.d002.s051.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"hear%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":642,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001404,"input_uncached_usd":0.001284,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.65525541594252}
{"item_id":"senseval3.d002.s051.t001","gold_sense_keys":["part%1:06:00::","part%1:15:00::","part%1:24:00::"],"candidates":[{"index":1,"sense_key":"part%1:24:00::","synset_id":"part.n.01"},{"index":2,"sense_key":"part%1:06:00::","synset_id":"part.n.02"},{"index":3,"sense_key":"part%1:17:00::","synset_id":"part.n.03"},{"index":4,"sense_key":"part%1:09:02::","synset_id":"part.n.04"},{"index":5,"sense_key":"part%1:15:00::","synset_id":"region.n.01"},{"index":6,"sense_key":"part%1:04:01::","synset_id":"function.n.03"},{"index":7,"sense_key":"part%1:09:01::","synset_id":"character.n.04"},{"index":8,"sense_key":"part%1:21:00::","synset_id":"share.n.01"},{"index":9,"sense_key":"part%1:09:00::","synset_id":"part.n.09"},{"index":10,"sense_key":"part%1:08:01::","synset_id":"part.n.10"},{"index":11,"sense_key":"part%1:10:00::","synset_id":"part.n.11"},{"index":12,"sense_key":"part%1:04:03::","synset_id":"contribution.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"part%1:06:00::","call_ids":["senseval3.d002.s051.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"part%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1145,"cached_input_tokens":0,"output_tokens":69,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00298,"input_uncached_usd":0.00229,"input_cached_usd":0.0,"output_usd":0.0006900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.045012917020358}
{"item_id":"senseval3.d002.s051.t002","gold_sense_keys":["building%1:06:00::"],"candidates":[{"index":1,"sense_key":"building%1:06:00::","synset_id":"building.n.01"},{"index":2,"sense_key":"building%1:04:00::","synset_id":"construction.n.01"},{"index":3,"sense_key":"building%1:04:01::","synset_id":"construction.n.07"},{"index":4,"sense_key":"building%1:14:00::","synset_id":"building.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"building%1:06:00::","call_ids":["senseval3.d002.s051.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"building%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":585,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0012900000000000001,"input_uncached_usd":0.00117,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0946820829994977}
{"item_id":"senseval3.d002.s051.t003","gold_sense_keys":["head%1:08:00::"],"candidates":[{"index":1,"sense_key":"head%1:08:00::","synset_id":"head.n.01"},{"index":2,"sense_key":"head%1:05:00::","synset_id":"head.n.02"},{"index":3,"sense_key":"head%1:09:00::","synset_id":"mind.n.01"},{"index":4,"sense_key":"head%1:18:00::","synset_id":"head.n.04"},{"index":5,"sense_key":"head%1:14:00::","synset_id":"head.n.05"},{"index":6,"sense_key":"head%1:19:00::","synset_id":"head.n.06"},{"index":7,"sense_key":"head%1:15:00::","synset_id":"head.n.07"},{"index":8,"sense_key":"head%1:15:01::","synset_id":"fountainhead.n.02"},{"index":9,"sense_key":"head%1:10:03::","synset_id":"head.n.09"},{"index":10,"sense_key":"head%1:26:00::","synset_id":"head.n.10"},{"index":11,"sense_key":"head%1:23:00::","synset_id":"head.n.11"},{"index":12,"sense_key":"head%1:20:00::","synset_id":"capitulum.n.01"},{"index":13,"sense_key":"head%1:18:01::","synset_id":"principal.n.02"},{"index":14,"sense_key":"head%1:18:03::","synset_id":"head.n.14"},{"index":15,"sense_key":"head%1:18:02::","synset_id":"head.n.15"},{"index":16,"sense_key":"head%1:17:02::","synset_id":"promontory.n.01"},{"index":17,"sense_key":"head%1:17:01::","synset_id":"head.n.17"},{"index":18,"sense_key":"head%1:17:00::","synset_id":"head.n.18"},{"index":19,"sense_key":"head%1:15:02::","synset_id":"forefront.n.01"},{"index":20,"sense_key":"head%1:11:01::","synset_id":"pass.n.09"},{"index":21,"sense_key":"head%1:11:00::","synset_id":"headway.n.02"},{"index":22,"sense_key":"head%1:10:01::","synset_id":"point.n.20"},{"index":23,"sense_key":"head%1:10:02::","synset_id":"question.n.02"},{"index":24,"sense_key":"head%1:10:00::","synset_id":"heading.n.01"},{"index":25,"sense_key":"head%1:08:01::","synset_id":"head.n.25"},{"index":26,"sense_key":"head%1:08:02::","synset_id":"head.n.26"},{"index":27,"sense_key":"head%1:06:05::","synset_id":"read/write_head.n.01"},{"index":28,"sense_key":"head%1:06:04::","synset_id":"head.n.28"},{"index":29,"sense_key":"head%1:06:03::","synset_id":"head.n.29"},{"index":30,"sense_key":"head%1:06:02::","synset_id":"head.n.30"},{"index":31,"sense_key":"head%1:06:00::","synset_id":"head.n.31"},{"index":32,"sense_key":"head%1:06:01::","synset_id":"drumhead.n.01"},{"index":33,"sense_key":"head%1:04:00::","synset_id":"oral_sex.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"head%1:08:00::","call_ids":["senseval3.d002.s051.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"head%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2155,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00443,"input_uncached_usd":0.00431,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5678371249232441}
{"item_id":"senseval3.d002.s051.t004","gold_sense_keys":["crack%2:30:00::","crack%2:30:01::","crack%2:39:00::","crack%2:39:01::"],"candidates":[{"index":1,"sense_key":"crack%2:30:01::","synset_id":"crack.v.01"},{"index":2,"sense_key":"crack%2:39:01::","synset_id":"crack.v.02"},{"index":3,"sense_key":"crack%2:39:00::","synset_id":"snap.v.06"},{"index":4,"sense_key":"crack%2:35:00::","synset_id":"crack.v.04"},{"index":5,"sense_key":"crack%2:30:09::","synset_id":"break_through.v.01"},{"index":6,"sense_key":"crack%2:30:02::","synset_id":"crack.v.06"},{"index":7,"sense_key":"crack%2:30:00::","synset_id":"snap.v.03"},{"index":8,"sense_key":"crack%2:41:00::","synset_id":"crack.v.08"},{"index":9,"sense_key":"crack%2:37:00::","synset_id":"crack_up.v.01"},{"index":10,"sense_key":"crack%2:32:00::","synset_id":"crack.v.10"},{"index":11,"sense_key":"crack%2:30:05::","synset_id":"crack.v.11"},{"index":12,"sense_key":"crack%2:30:07::","synset_id":"crack.v.12"},{"index":13,"sense_key":"crack%2:30:08::","synset_id":"crack.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"crack%2:39:00::","call_ids":["senseval3.d002.s051.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"crack%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":994,"cached_input_tokens":0,"output_tokens":325,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.005238,"input_uncached_usd":0.0019879999999999997,"input_cached_usd":0.0,"output_usd":0.0032500000000000003,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.030359333031811}
{"item_id":"senseval3.d002.s052.t000","gold_sense_keys":["think%2:31:03::"],"candidates":[{"index":1,"sense_key":"think%2:31:01::","synset_id":"think.v.01"},{"index":2,"sense_key":"think%2:31:03::","synset_id":"think.v.02"},{"index":3,"sense_key":"think%2:31:00::","synset_id":"think.v.03"},{"index":4,"sense_key":"think%2:31:02::","synset_id":"remember.v.01"},{"index":5,"sense_key":"think%2:36:00::","synset_id":"think.v.05"},{"index":6,"sense_key":"think%2:31:09::","synset_id":"think.v.06"},{"index":7,"sense_key":"think%2:31:06::","synset_id":"intend.v.01"},{"index":8,"sense_key":"think%2:31:12::","synset_id":"think.v.08"},{"index":9,"sense_key":"think%2:31:13::","synset_id":"think.v.09"},{"index":10,"sense_key":"think%2:31:15::","synset_id":"think.v.10"},{"index":11,"sense_key":"think%2:31:14::","synset_id":"think.v.11"},{"index":12,"sense_key":"think%2:31:08::","synset_id":"think.v.12"},{"index":13,"sense_key":"think%2:30:00::","synset_id":"think.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"think%2:31:03::","call_ids":["senseval3.d002.s052.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"think%2:31:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1129,"cached_input_tokens":0,"output_tokens":126,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0035180000000000003,"input_uncached_usd":0.002258,"input_cached_usd":0.0,"output_usd":0.00126,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.782715959008783}
{"item_id":"senseval3.d002.s052.t001","gold_sense_keys":["die%2:30:00::"],"candidates":[{"index":1,"sense_key":"die%2:30:00::","synset_id":"die.v.01"},{"index":2,"sense_key":"die%2:39:00::","synset_id":"die.v.02"},{"index":3,"sense_key":"die%2:37:00::","synset_id":"die.v.03"},{"index":4,"sense_key":"die%2:30:04::","synset_id":"fail.v.04"},{"index":5,"sense_key":"die%2:37:03::","synset_id":"die.v.05"},{"index":6,"sense_key":"die%2:37:01::","synset_id":"die.v.06"},{"index":7,"sense_key":"die%2:35:00::","synset_id":"die.v.07"},{"index":8,"sense_key":"die%2:33:00::","synset_id":"die.v.08"},{"index":9,"sense_key":"die%2:30:05::","synset_id":"die.v.09"},{"index":10,"sense_key":"die%2:30:01::","synset_id":"die.v.10"},{"index":11,"sense_key":"die%2:30:02::","synset_id":"die.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"die%2:30:00::","call_ids":["senseval3.d002.s052.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"die%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":982,"cached_input_tokens":0,"output_tokens":45,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002414,"input_uncached_usd":0.001964,"input_cached_usd":0.0,"output_usd":0.00045000000000000004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7631735000759363}
{"item_id":"senseval3.d002.s053.t000","gold_sense_keys":["decide%2:31:00::"],"candidates":[{"index":1,"sense_key":"decide%2:31:00::","synset_id":"decide.v.01"},{"index":2,"sense_key":"decide%2:31:01::","synset_id":"decide.v.02"},{"index":3,"sense_key":"decide%2:31:02::","synset_id":"decide.v.03"},{"index":4,"sense_key":"decide%2:31:06::","synset_id":"decide.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"decide%2:31:00::","call_ids":["senseval3.d002.s053.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"decide%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":580,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00128,"input_uncached_usd":0.00116,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.435325541999191}
{"item_id":"senseval3.d002.s053.t001","gold_sense_keys":["come%2:38:00::"],"candidates":[{"index":1,"sense_key":"come%2:38:00::","synset_id":"come.v.01"},{"index":2,"sense_key":"come%2:38:04::","synset_id":"arrive.v.01"},{"index":3,"sense_key":"come%2:30:01::","synset_id":"come.v.03"},{"index":4,"sense_key":"come%2:30:03::","synset_id":"come.v.04"},{"index":5,"sense_key":"come%2:42:04::","synset_id":"come.v.05"},{"index":6,"sense_key":"come%2:30:02::","synset_id":"come.v.06"},{"index":7,"sense_key":"come%2:42:13::","synset_id":"issue_forth.v.01"},{"index":8,"sense_key":"come%2:42:09::","synset_id":"hail.v.02"},{"index":9,"sense_key":"come%2:42:03::","synset_id":"come.v.09"},{"index":10,"sense_key":"come%2:42:05::","synset_id":"come.v.10"},{"index":11,"sense_key":"come%2:38:03::","synset_id":"come.v.11"},{"index":12,"sense_key":"come%2:42:00::","synset_id":"fall.v.04"},{"index":13,"sense_key":"come%2:30:13::","synset_id":"come.v.13"},{"index":14,"sense_key":"come%2:42:12::","synset_id":"total.v.01"},{"index":15,"sense_key":"come%2:42:11::","synset_id":"come.v.15"},{"index":16,"sense_key":"come%2:38:02::","synset_id":"come.v.16"},{"index":17,"sense_key":"come%2:31:14::","synset_id":"occur.v.02"},{"index":18,"sense_key":"come%2:42:15::","synset_id":"derive.v.05"},{"index":19,"sense_key":"come%2:42:02::","synset_id":"do.v.04"},{"index":20,"sense_key":"come%2:39:00::","synset_id":"come.v.20"},{"index":21,"sense_key":"come%2:31:13::","synset_id":"come.v.21"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"come%2:38:00::","call_ids":["senseval3.d002.s053.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"come%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1511,"cached_input_tokens":0,"output_tokens":143,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004452,"input_uncached_usd":0.003022,"input_cached_usd":0.0,"output_usd":0.00143,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1700957919238135}
{"item_id":"senseval3.d002.s053.t002","gold_sense_keys":["home%1:15:04::"],"candidates":[{"index":1,"sense_key":"home%1:15:04::","synset_id":"home.n.01"},{"index":2,"sense_key":"home%1:06:00::","synset_id":"dwelling.n.01"},{"index":3,"sense_key":"home%1:15:02::","synset_id":"home.n.03"},{"index":4,"sense_key":"home%1:06:02::","synset_id":"home_plate.n.01"},{"index":5,"sense_key":"home%1:15:05::","synset_id":"base.n.14"},{"index":6,"sense_key":"home%1:15:03::","synset_id":"home.n.06"},{"index":7,"sense_key":"home%1:26:00::","synset_id":"home.n.07"},{"index":8,"sense_key":"home%1:14:00::","synset_id":"family.n.01"},{"index":9,"sense_key":"home%1:06:01::","synset_id":"home.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"home%1:15:04::","call_ids":["senseval3.d002.s053.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"home%1:15:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":902,"cached_input_tokens":0,"output_tokens":131,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003114,"input_uncached_usd":0.0018039999999999998,"input_cached_usd":0.0,"output_usd":0.0013100000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.231778540997766}
{"item_id":"senseval3.d002.s053.t003","gold_sense_keys":["house%1:06:00::"],"candidates":[{"index":1,"sense_key":"house%1:06:00::","synset_id":"house.n.01"},{"index":2,"sense_key":"house%1:14:01::","synset_id":"firm.n.01"},{"index":3,"sense_key":"house%1:14:04::","synset_id":"house.n.03"},{"index":4,"sense_key":"house%1:14:03::","synset_id":"house.n.04"},{"index":5,"sense_key":"house%1:14:02::","synset_id":"house.n.05"},{"index":6,"sense_key":"house%1:14:00::","synset_id":"house.n.06"},{"index":7,"sense_key":"house%1:04:00::","synset_id":"house.n.07"},{"index":8,"sense_key":"house%1:15:00::","synset_id":"sign_of_the_zodiac.n.01"},{"index":9,"sense_key":"house%1:14:06::","synset_id":"house.n.09"},{"index":10,"sense_key":"house%1:14:05::","synset_id":"family.n.01"},{"index":11,"sense_key":"house%1:06:02::","synset_id":"theater.n.01"},{"index":12,"sense_key":"house%1:06:01::","synset_id":"house.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"house%1:06:00::","call_ids":["senseval3.d002.s053.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"house%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":984,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0020879999999999996,"input_uncached_usd":0.0019679999999999997,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0507226660847664}
{"item_id":"senseval3.d002.s053.t004","gold_sense_keys":["stilt%1:06:01::"],"candidates":[{"index":1,"sense_key":"stilt%1:06:01::","synset_id":"pile.n.06"},{"index":2,"sense_key":"stilt%1:06:00::","synset_id":"stilt.n.02"},{"index":3,"sense_key":"stilt%1:05:02::","synset_id":"stilt.n.03"},{"index":4,"sense_key":"stilt%1:05:01::","synset_id":"stilt.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"stilt%1:06:01::","call_ids":["senseval3.d002.s053.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"stilt%1:06:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":619,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001878,"input_uncached_usd":0.001238,"input_cached_usd":0.0,"output_usd":0.00064,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6345121660269797}
{"item_id":"senseval3.d002.s054.t000","gold_sense_keys":["decide%2:31:00::"],"candidates":[{"index":1,"sense_key":"decide%2:31:00::","synset_id":"decide.v.01"},{"index":2,"sense_key":"decide%2:31:01::","synset_id":"decide.v.02"},{"index":3,"sense_key":"decide%2:31:02::","synset_id":"decide.v.03"},{"index":4,"sense_key":"decide%2:31:06::","synset_id":"decide.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"decide%2:31:00::","call_ids":["senseval3.d002.s054.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"decide%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":615,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00135,"input_uncached_usd":0.00123,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4981362080434337}
{"item_id":"senseval3.d002.s055.t001","gold_sense_keys":["smell%1:07:00::","smell%1:09:02::"],"candidates":[{"index":1,"sense_key":"smell%1:09:02::","synset_id":"smell.n.01"},{"index":2,"sense_key":"smell%1:07:00::","synset_id":"olfactory_property.n.01"},{"index":3,"sense_key":"smell%1:26:00::","synset_id":"spirit.n.02"},{"index":4,"sense_key":"smell%1:09:01::","synset_id":"smell.n.04"},{"index":5,"sense_key":"smell%1:04:00::","synset_id":"smell.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"smell%1:07:00::","call_ids":["senseval3.d002.s055.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"smell%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":755,"cached_input_tokens":0,"output_tokens":168,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00319,"input_uncached_usd":0.0015099999999999998,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1963315409375355}
{"item_id":"senseval3.d002.s055.t002","gold_sense_keys":["gas%1:27:02::"],"candidates":[{"index":1,"sense_key":"gas%1:26:00::","synset_id":"gas.n.01"},{"index":2,"sense_key":"gas%1:27:00::","synset_id":"gas.n.02"},{"index":3,"sense_key":"gas%1:27:02::","synset_id":"gasoline.n.01"},{"index":4,"sense_key":"gas%1:26:01::","synset_id":"flatulence.n.01"},{"index":5,"sense_key":"gas%1:06:00::","synset_id":"accelerator.n.01"},{"index":6,"sense_key":"gas%1:27:03::","synset_id":"natural_gas.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"gas%1:27:00::","call_ids":["senseval3.d002.s055.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"gas%1:27:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":863,"cached_input_tokens":0,"output_tokens":264,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004366,"input_uncached_usd":0.0017259999999999999,"input_cached_usd":0.0,"output_usd":0.0026400000000000004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.86620229203254}
{"item_id":"senseval3.d002.s055.t003","gold_sense_keys":["pass%2:38:05::"],"candidates":[{"index":1,"sense_key":"pass%2:38:00::","synset_id":"pass.v.01"},{"index":2,"sense_key":"pass%2:38:05::","synset_id":"travel_by.v.01"},{"index":3,"sense_key":"pass%2:41:02::","synset_id":"legislate.v.01"},{"index":4,"sense_key":"pass%2:38:03::","synset_id":"elapse.v.01"},{"index":5,"sense_key":"pass%2:40:00::","synset_id":"pass.v.05"},{"index":6,"sense_key":"pass%2:42:08::","synset_id":"run.v.03"},{"index":7,"sense_key":"pass%2:38:01::","synset_id":"pass.v.07"},{"index":8,"sense_key":"pass%2:30:00::","synset_id":"happen.v.01"},{"index":9,"sense_key":"pass%2:41:04::","synset_id":"pass.v.09"},{"index":10,"sense_key":"pass%2:42:00::","synset_id":"spend.v.01"},{"index":11,"sense_key":"pass%2:35:00::","synset_id":"guide.v.05"},{"index":12,"sense_key":"pass%2:32:01::","synset_id":"communicate.v.01"},{"index":13,"sense_key":"pass%2:30:01::","synset_id":"evanesce.v.01"},{"index":14,"sense_key":"pass%2:41:08::","synset_id":"pass.v.14"},{"index":15,"sense_key":"pass%2:42:04::","synset_id":"exceed.v.02"},{"index":16,"sense_key":"pass%2:41:07::","synset_id":"pass.v.16"},{"index":17,"sense_key":"pass%2:41:11::","synset_id":"pass.v.17"},{"index":18,"sense_key":"pass%2:40:02::","synset_id":"pass.v.18"},{"index":19,"sense_key":"pass%2:38:02::","synset_id":"sink.v.03"},{"index":20,"sense_key":"pass%2:40:01::","synset_id":"pass.v.20"},{"index":21,"sense_key":"pass%2:40:13::","synset_id":"fall.v.21"},{"index":22,"sense_key":"pass%2:38:04::","synset_id":"pass.v.22"},{"index":23,"sense_key":"pass%2:32:00::","synset_id":"authorize.v.01"},{"index":24,"sense_key":"pass%2:30:11::","synset_id":"die.v.01"},{"index":25,"sense_key":"pass%2:29:00::","synset_id":"excrete.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"pass%2:38:05::","call_ids":["senseval3.d002.s055.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"pass%2:38:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2014,"cached_input_tokens":0,"output_tokens":120,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.005228,"input_uncached_usd":0.0040279999999999995,"input_cached_usd":0.0,"output_usd":0.0012000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6172617500415072}
{"item_id":"senseval3.d002.s055.t005","gold_sense_keys":["cross%2:38:00::"],"candidates":[{"index":1,"sense_key":"cross%2:38:00::","synset_id":"traverse.v.01"},{"index":2,"sense_key":"cross%2:38:03::","synset_id":"intersect.v.01"},{"index":3,"sense_key":"cross%2:41:00::","synset_id":"thwart.v.01"},{"index":4,"sense_key":"cross%2:38:04::","synset_id":"cross.v.04"},{"index":5,"sense_key":"cross%2:42:00::","synset_id":"cross.v.05"},{"index":6,"sense_key":"cross%2:38:01::","synset_id":"cross.v.06"},{"index":7,"sense_key":"cross%2:36:09::","synset_id":"cross.v.07"},{"index":8,"sense_key":"cross%2:35:00::","synset_id":"crossbreed.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cross%2:38:00::","call_ids":["senseval3.d002.s055.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cross%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":906,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0019320000000000001,"input_uncached_usd":0.001812,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4494013329967856}
{"item_id":"senseval3.d002.s056.t000","gold_sense_keys":["see%2:39:00::"],"candidates":[{"index":1,"sense_key":"see%2:39:00::","synset_id":"see.v.01"},{"index":2,"sense_key":"see%2:31:01::","synset_id":"understand.v.02"},{"index":3,"sense_key":"see%2:39:02::","synset_id":"witness.v.02"},{"index":4,"sense_key":"see%2:36:00::","synset_id":"visualize.v.01"},{"index":5,"sense_key":"see%2:31:00::","synset_id":"see.v.05"},{"index":6,"sense_key":"see%2:31:02::","synset_id":"learn.v.02"},{"index":7,"sense_key":"see%2:39:03::","synset_id":"watch.v.03"},{"index":8,"sense_key":"see%2:38:00::","synset_id":"meet.v.01"},{"index":9,"sense_key":"see%2:32:00::","synset_id":"determine.v.08"},{"index":10,"sense_key":"see%2:31:03::","synset_id":"see.v.10"},{"index":11,"sense_key":"see%2:41:11::","synset_id":"see.v.11"},{"index":12,"sense_key":"see%2:41:12::","synset_id":"see.v.12"},{"index":13,"sense_key":"see%2:41:10::","synset_id":"visit.v.01"},{"index":14,"sense_key":"see%2:41:14::","synset_id":"attend.v.02"},{"index":15,"sense_key":"see%2:35:09::","synset_id":"see.v.15"},{"index":16,"sense_key":"see%2:41:00::","synset_id":"go_steady.v.01"},{"index":17,"sense_key":"see%2:39:05::","synset_id":"see.v.17"},{"index":18,"sense_key":"see%2:31:08::","synset_id":"see.v.18"},{"index":19,"sense_key":"see%2:39:11::","synset_id":"see.v.19"},{"index":20,"sense_key":"see%2:39:13::","synset_id":"examine.v.02"},{"index":21,"sense_key":"see%2:39:12::","synset_id":"experience.v.01"},{"index":22,"sense_key":"see%2:38:03::","synset_id":"see.v.22"},{"index":23,"sense_key":"see%2:33:00::","synset_id":"see.v.23"},{"index":24,"sense_key":"see%2:31:13::","synset_id":"interpret.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"see%2:39:00::","call_ids":["senseval3.d002.s056.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"see%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1931,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003982,"input_uncached_usd":0.003862,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.913384290994145}
{"item_id":"senseval3.d002.s056.t001","gold_sense_keys":["cloud%1:19:01::"],"candidates":[{"index":1,"sense_key":"cloud%1:19:01::","synset_id":"cloud.n.01"},{"index":2,"sense_key":"cloud%1:17:00::","synset_id":"cloud.n.02"},{"index":3,"sense_key":"cloud%1:26:01::","synset_id":"cloud.n.03"},{"index":4,"sense_key":"cloud%1:26:02::","synset_id":"cloud.n.04"},{"index":5,"sense_key":"cloud%1:26:00::","synset_id":"cloud.n.05"},{"index":6,"sense_key":"cloud%1:14:00::","synset_id":"swarm.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cloud%1:19:01::","call_ids":["senseval3.d002.s056.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cloud%1:19:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":699,"cached_input_tokens":0,"output_tokens":94,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002338,"input_uncached_usd":0.001398,"input_cached_usd":0.0,"output_usd":0.0009400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6738747090566903}
{"item_id":"senseval3.d002.s056.t002","gold_sense_keys":["bay%1:17:00::"],"candidates":[{"index":1,"sense_key":"bay%1:17:00::","synset_id":"bay.n.01"},{"index":2,"sense_key":"bay%1:11:00::","synset_id":"bay.n.02"},{"index":3,"sense_key":"bay%1:20:00::","synset_id":"true_laurel.n.01"},{"index":4,"sense_key":"bay%1:06:02::","synset_id":"bay.n.04"},{"index":5,"sense_key":"bay%1:06:01::","synset_id":"bay.n.05"},{"index":6,"sense_key":"bay%1:06:00::","synset_id":"alcove.n.01"},{"index":7,"sense_key":"bay%1:05:00::","synset_id":"bay.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bay%1:17:00::","call_ids":["senseval3.d002.s056.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bay%1:17:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":762,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001644,"input_uncached_usd":0.001524,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0566124999895692}
{"item_id":"senseval3.d002.s056.t004","gold_sense_keys":["fire%1:11:00::"],"candidates":[{"index":1,"sense_key":"fire%1:11:00::","synset_id":"fire.n.01"},{"index":2,"sense_key":"fire%1:04:00::","synset_id":"fire.n.02"},{"index":3,"sense_key":"fire%1:22:00::","synset_id":"fire.n.03"},{"index":4,"sense_key":"fire%1:06:00::","synset_id":"fire.n.04"},{"index":5,"sense_key":"fire%1:27:01::","synset_id":"fire.n.05"},{"index":6,"sense_key":"fire%1:12:00::","synset_id":"ardor.n.03"},{"index":7,"sense_key":"fire%1:27:00::","synset_id":"fire.n.07"},{"index":8,"sense_key":"fire%1:11:01::","synset_id":"fire.n.08"},{"index":9,"sense_key":"fire%1:10:00::","synset_id":"fire.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"fire%1:11:00::","call_ids":["senseval3.d002.s056.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"fire%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":940,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0025,"input_uncached_usd":0.00188,"input_cached_usd":0.0,"output_usd":0.00062,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.762200125027448}
{"item_id":"senseval3.d002.s057.t000","gold_sense_keys":["feel%2:39:00::"],"candidates":[{"index":1,"sense_key":"feel%2:37:00::","synset_id":"feel.v.01"},{"index":2,"sense_key":"feel%2:31:00::","synset_id":"find.v.05"},{"index":3,"sense_key":"feel%2:39:00::","synset_id":"feel.v.03"},{"index":4,"sense_key":"feel%2:29:00::","synset_id":"feel.v.04"},{"index":5,"sense_key":"feel%2:31:01::","synset_id":"feel.v.05"},{"index":6,"sense_key":"feel%2:39:01::","synset_id":"feel.v.06"},{"index":7,"sense_key":"feel%2:42:00::","synset_id":"feel.v.07"},{"index":8,"sense_key":"feel%2:39:10::","synset_id":"feel.v.08"},{"index":9,"sense_key":"feel%2:35:00::","synset_id":"feel.v.09"},{"index":10,"sense_key":"feel%2:35:01::","synset_id":"palpate.v.01"},{"index":11,"sense_key":"feel%2:40:00::","synset_id":"feel.v.11"},{"index":12,"sense_key":"feel%2:39:09::","synset_id":"feel.v.12"},{"index":13,"sense_key":"feel%2:35:11::","synset_id":"feel.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"feel%2:39:00::","call_ids":["senseval3.d002.s057.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"feel%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1170,"cached_input_tokens":0,"output_tokens":141,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00375,"input_uncached_usd":0.00234,"input_cached_usd":0.0,"output_usd":0.00141,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7154067500960082}
{"item_id":"senseval3.d002.s058.t000","gold_sense_keys":["back%1:08:00::"],"candidates":[{"index":1,"sense_key":"back%1:08:00::","synset_id":"back.n.01"},{"index":2,"sense_key":"back%1:06:00::","synset_id":"rear.n.05"},{"index":3,"sense_key":"back%1:15:02::","synset_id":"back.n.03"},{"index":4,"sense_key":"back%1:18:00::","synset_id":"back.n.04"},{"index":5,"sense_key":"back%1:08:01::","synset_id":"spinal_column.n.01"},{"index":6,"sense_key":"back%1:06:03::","synset_id":"binding.n.05"},{"index":7,"sense_key":"back%1:06:02::","synset_id":"back.n.07"},{"index":8,"sense_key":"back%1:06:01::","synset_id":"back.n.08"},{"index":9,"sense_key":"back%1:04:00::","synset_id":"back.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"back%1:08:00::","call_ids":["senseval3.d002.s058.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"back%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":938,"cached_input_tokens":0,"output_tokens":113,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003006,"input_uncached_usd":0.0018759999999999998,"input_cached_usd":0.0,"output_usd":0.0011300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.760545999975875}
{"item_id":"senseval3.d002.s058.t002","gold_sense_keys":["hand%1:08:00::"],"candidates":[{"index":1,"sense_key":"hand%1:08:00::","synset_id":"hand.n.01"},{"index":2,"sense_key":"hand%1:18:00::","synset_id":"hired_hand.n.01"},{"index":3,"sense_key":"hand%1:10:00::","synset_id":"handwriting.n.01"},{"index":4,"sense_key":"hand%1:09:00::","synset_id":"hand.n.04"},{"index":5,"sense_key":"hand%1:15:00::","synset_id":"hand.n.05"},{"index":6,"sense_key":"hand%1:14:00::","synset_id":"hand.n.06"},{"index":7,"sense_key":"hand%1:09:01::","synset_id":"hand.n.07"},{"index":8,"sense_key":"hand%1:06:00::","synset_id":"hand.n.08"},{"index":9,"sense_key":"hand%1:23:00::","synset_id":"hand.n.09"},{"index":10,"sense_key":"hand%1:18:02::","synset_id":"hand.n.10"},{"index":11,"sense_key":"hand%1:18:01::","synset_id":"bridge_player.n.01"},{"index":12,"sense_key":"hand%1:10:01::","synset_id":"hand.n.12"},{"index":13,"sense_key":"hand%1:05:01::","synset_id":"hand.n.13"},{"index":14,"sense_key":"hand%1:04:00::","synset_id":"hand.n.14"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"hand%1:08:00::","call_ids":["senseval3.d002.s058.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"hand%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1226,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0025719999999999996,"input_uncached_usd":0.0024519999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4099889589706436}
{"item_id":"senseval3.d002.s058.t003","gold_sense_keys":["shake%2:29:00::","shake%2:38:00::","shake%2:38:01::","shake%2:38:02::"],"candidates":[{"index":1,"sense_key":"shake%2:38:00::","synset_id":"shake.v.01"},{"index":2,"sense_key":"shake%2:29:00::","synset_id":"shake.v.02"},{"index":3,"sense_key":"shake%2:38:01::","synset_id":"judder.v.01"},{"index":4,"sense_key":"shake%2:38:02::","synset_id":"rock.v.01"},{"index":5,"sense_key":"shake%2:30:00::","synset_id":"shake.v.05"},{"index":6,"sense_key":"shake%2:37:00::","synset_id":"stimulate.v.03"},{"index":7,"sense_key":"shake%2:38:04::","synset_id":"shake.v.07"},{"index":8,"sense_key":"shake%2:30:01::","synset_id":"shake.v.08"},{"index":9,"sense_key":"shake%2:32:00::","synset_id":"shake.v.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"shake%2:29:00::","call_ids":["senseval3.d002.s058.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"shake%2:29:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":951,"cached_input_tokens":0,"output_tokens":231,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004212,"input_uncached_usd":0.0019019999999999998,"input_cached_usd":0.0,"output_usd":0.00231,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.420340040931478}
{"item_id":"senseval3.d002.s059.t000","gold_sense_keys":["think%2:31:03::"],"candidates":[{"index":1,"sense_key":"think%2:31:01::","synset_id":"think.v.01"},{"index":2,"sense_key":"think%2:31:03::","synset_id":"think.v.02"},{"index":3,"sense_key":"think%2:31:00::","synset_id":"think.v.03"},{"index":4,"sense_key":"think%2:31:02::","synset_id":"remember.v.01"},{"index":5,"sense_key":"think%2:36:00::","synset_id":"think.v.05"},{"index":6,"sense_key":"think%2:31:09::","synset_id":"think.v.06"},{"index":7,"sense_key":"think%2:31:06::","synset_id":"intend.v.01"},{"index":8,"sense_key":"think%2:31:12::","synset_id":"think.v.08"},{"index":9,"sense_key":"think%2:31:13::","synset_id":"think.v.09"},{"index":10,"sense_key":"think%2:31:15::","synset_id":"think.v.10"},{"index":11,"sense_key":"think%2:31:14::","synset_id":"think.v.11"},{"index":12,"sense_key":"think%2:31:08::","synset_id":"think.v.12"},{"index":13,"sense_key":"think%2:30:00::","synset_id":"think.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"think%2:31:03::","call_ids":["senseval3.d002.s059.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"think%2:31:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1161,"cached_input_tokens":0,"output_tokens":134,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003662,"input_uncached_usd":0.002322,"input_cached_usd":0.0,"output_usd":0.00134,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1772465830435976}
{"item_id":"senseval3.d002.s059.t003","gold_sense_keys":["body%1:08:00::"],"candidates":[{"index":1,"sense_key":"body%1:08:00::","synset_id":"body.n.01"},{"index":2,"sense_key":"body%1:14:00::","synset_id":"body.n.02"},{"index":3,"sense_key":"body%1:08:02::","synset_id":"body.n.03"},{"index":4,"sense_key":"body%1:17:00::","synset_id":"body.n.04"},{"index":5,"sense_key":"body%1:08:01::","synset_id":"torso.n.01"},{"index":6,"sense_key":"body%1:14:02::","synset_id":"body.n.06"},{"index":7,"sense_key":"body%1:07:00::","synset_id":"consistency.n.01"},{"index":8,"sense_key":"body%1:10:00::","synset_id":"body.n.08"},{"index":9,"sense_key":"body%1:07:01::","synset_id":"body.n.09"},{"index":10,"sense_key":"body%1:06:01::","synset_id":"soundbox.n.01"},{"index":11,"sense_key":"body%1:06:00::","synset_id":"body.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"body%1:08:00::","call_ids":["senseval3.d002.s059.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"body%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":989,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0020979999999999996,"input_uncached_usd":0.0019779999999999997,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9931356669403613}
{"item_id":"senseval3.d002.s059.t004","gold_sense_keys":["shake%2:29:00::","shake%2:38:00::","shake%2:38:01::","shake%2:38:02::"],"candidates":[{"index":1,"sense_key":"shake%2:38:00::","synset_id":"shake.v.01"},{"index":2,"sense_key":"shake%2:29:00::","synset_id":"shake.v.02"},{"index":3,"sense_key":"shake%2:38:01::","synset_id":"judder.v.01"},{"index":4,"sense_key":"shake%2:38:02::","synset_id":"rock.v.01"},{"index":5,"sense_key":"shake%2:30:00::","synset_id":"shake.v.05"},{"index":6,"sense_key":"shake%2:37:00::","synset_id":"stimulate.v.03"},{"index":7,"sense_key":"shake%2:38:04::","synset_id":"shake.v.07"},{"index":8,"sense_key":"shake%2:30:01::","synset_id":"shake.v.08"},{"index":9,"sense_key":"shake%2:32:00::","synset_id":"shake.v.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"shake%2:29:00::","call_ids":["senseval3.d002.s059.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"shake%2:29:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":934,"cached_input_tokens":0,"output_tokens":125,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003118,"input_uncached_usd":0.0018679999999999999,"input_cached_usd":0.0,"output_usd":0.00125,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8700774580938742}
{"item_id":"senseval3.d002.s062.t000","gold_sense_keys":["see%2:39:00::"],"candidates":[{"index":1,"sense_key":"see%2:39:00::","synset_id":"see.v.01"},{"index":2,"sense_key":"see%2:31:01::","synset_id":"understand.v.02"},{"index":3,"sense_key":"see%2:39:02::","synset_id":"witness.v.02"},{"index":4,"sense_key":"see%2:36:00::","synset_id":"visualize.v.01"},{"index":5,"sense_key":"see%2:31:00::","synset_id":"see.v.05"},{"index":6,"sense_key":"see%2:31:02::","synset_id":"learn.v.02"},{"index":7,"sense_key":"see%2:39:03::","synset_id":"watch.v.03"},{"index":8,"sense_key":"see%2:38:00::","synset_id":"meet.v.01"},{"index":9,"sense_key":"see%2:32:00::","synset_id":"determine.v.08"},{"index":10,"sense_key":"see%2:31:03::","synset_id":"see.v.10"},{"index":11,"sense_key":"see%2:41:11::","synset_id":"see.v.11"},{"index":12,"sense_key":"see%2:41:12::","synset_id":"see.v.12"},{"index":13,"sense_key":"see%2:41:10::","synset_id":"visit.v.01"},{"index":14,"sense_key":"see%2:41:14::","synset_id":"attend.v.02"},{"index":15,"sense_key":"see%2:35:09::","synset_id":"see.v.15"},{"index":16,"sense_key":"see%2:41:00::","synset_id":"go_steady.v.01"},{"index":17,"sense_key":"see%2:39:05::","synset_id":"see.v.17"},{"index":18,"sense_key":"see%2:31:08::","synset_id":"see.v.18"},{"index":19,"sense_key":"see%2:39:11::","synset_id":"see.v.19"},{"index":20,"sense_key":"see%2:39:13::","synset_id":"examine.v.02"},{"index":21,"sense_key":"see%2:39:12::","synset_id":"experience.v.01"},{"index":22,"sense_key":"see%2:38:03::","synset_id":"see.v.22"},{"index":23,"sense_key":"see%2:33:00::","synset_id":"see.v.23"},{"index":24,"sense_key":"see%2:31:13::","synset_id":"interpret.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"see%2:39:00::","call_ids":["senseval3.d002.s062.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"see%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1882,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003884,"input_uncached_usd":0.003764,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1408905840944499}
{"item_id":"senseval3.d002.s062.t002","gold_sense_keys":["house%1:06:00::"],"candidates":[{"index":1,"sense_key":"house%1:06:00::","synset_id":"house.n.01"},{"index":2,"sense_key":"house%1:14:01::","synset_id":"firm.n.01"},{"index":3,"sense_key":"house%1:14:04::","synset_id":"house.n.03"},{"index":4,"sense_key":"house%1:14:03::","synset_id":"house.n.04"},{"index":5,"sense_key":"house%1:14:02::","synset_id":"house.n.05"},{"index":6,"sense_key":"house%1:14:00::","synset_id":"house.n.06"},{"index":7,"sense_key":"house%1:04:00::","synset_id":"house.n.07"},{"index":8,"sense_key":"house%1:15:00::","synset_id":"sign_of_the_zodiac.n.01"},{"index":9,"sense_key":"house%1:14:06::","synset_id":"house.n.09"},{"index":10,"sense_key":"house%1:14:05::","synset_id":"family.n.01"},{"index":11,"sense_key":"house%1:06:02::","synset_id":"theater.n.01"},{"index":12,"sense_key":"house%1:06:01::","synset_id":"house.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"house%1:06:00::","call_ids":["senseval3.d002.s062.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"house%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":975,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00207,"input_uncached_usd":0.00195,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8834625409217551}
{"item_id":"senseval3.d002.s062.t003","gold_sense_keys":["bay%1:17:00::"],"candidates":[{"index":1,"sense_key":"bay%1:17:00::","synset_id":"bay.n.01"},{"index":2,"sense_key":"bay%1:11:00::","synset_id":"bay.n.02"},{"index":3,"sense_key":"bay%1:20:00::","synset_id":"true_laurel.n.01"},{"index":4,"sense_key":"bay%1:06:02::","synset_id":"bay.n.04"},{"index":5,"sense_key":"bay%1:06:01::","synset_id":"bay.n.05"},{"index":6,"sense_key":"bay%1:06:00::","synset_id":"alcove.n.01"},{"index":7,"sense_key":"bay%1:05:00::","synset_id":"bay.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bay%1:17:00::","call_ids":["senseval3.d002.s062.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bay%1:17:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":713,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001546,"input_uncached_usd":0.001426,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.229236499988474}
{"item_id":"senseval3.d002.s063.t000","gold_sense_keys":["hard%3:00:06::"],"candidates":[{"index":1,"sense_key":"hard%3:00:06::","synset_id":"difficult.a.01"},{"index":2,"sense_key":"hard%3:00:02::","synset_id":"hard.a.02"},{"index":3,"sense_key":"hard%3:00:01::","synset_id":"hard.a.03"},{"index":4,"sense_key":"hard%5:00:00:strong:00","synset_id":"hard.s.01"},{"index":5,"sense_key":"hard%5:00:01:effortful:00","synset_id":"arduous.s.01"},{"index":6,"sense_key":"hard%3:00:00::","synset_id":"unvoiced.a.01"},{"index":7,"sense_key":"hard%3:00:05::","synset_id":"hard.a.07"},{"index":8,"sense_key":"hard%3:00:03::","synset_id":"hard.a.08"},{"index":9,"sense_key":"hard%5:00:00:indulgent:00","synset_id":"intemperate.s.01"},{"index":10,"sense_key":"hard%5:00:00:alcoholic:00","synset_id":"hard.s.04"},{"index":11,"sense_key":"hard%5:00:00:bad:00","synset_id":"hard.s.05"},{"index":12,"sense_key":"hard%5:00:00:stale:00","synset_id":"hard.s.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"hard%3:00:06::","call_ids":["senseval3.d002.s063.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"hard%3:00:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1112,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0023439999999999997,"input_uncached_usd":0.002224,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.323996250052005}
{"item_id":"senseval3.d002.s063.t001","gold_sense_keys":["believe%2:31:00::"],"candidates":[{"index":1,"sense_key":"believe%2:31:00::","synset_id":"believe.v.01"},{"index":2,"sense_key":"believe%2:31:04::","synset_id":"think.v.01"},{"index":3,"sense_key":"believe%2:31:03::","synset_id":"believe.v.03"},{"index":4,"sense_key":"believe%2:31:01::","synset_id":"believe.v.04"},{"index":5,"sense_key":"believe%2:31:02::","synset_id":"believe.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"believe%2:31:00::","call_ids":["senseval3.d002.s063.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"believe%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":616,"cached_input_tokens":0,"output_tokens":45,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0016820000000000001,"input_uncached_usd":0.001232,"input_cached_usd":0.0,"output_usd":0.00045000000000000004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2823522919788957}
{"item_id":"senseval3.d002.s063.t002","gold_sense_keys":["happen%2:30:00::"],"candidates":[{"index":1,"sense_key":"happen%2:30:00::","synset_id":"happen.v.01"},{"index":2,"sense_key":"happen%2:30:02::","synset_id":"happen.v.02"},{"index":3,"sense_key":"happen%2:41:00::","synset_id":"happen.v.03"},{"index":4,"sense_key":"happen%2:30:01::","synset_id":"happen.v.04"},{"index":5,"sense_key":"happen%2:40:12::","synset_id":"find.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"happen%2:30:00::","call_ids":["senseval3.d002.s063.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"happen%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":671,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001462,"input_uncached_usd":0.0013419999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1486008339561522}
{"item_id":"senseval3.d002.s066.t000","gold_sense_keys":["building%1:06:00::"],"candidates":[{"index":1,"sense_key":"building%1:06:00::","synset_id":"building.n.01"},{"index":2,"sense_key":"building%1:04:00::","synset_id":"construction.n.01"},{"index":3,"sense_key":"building%1:04:01::","synset_id":"construction.n.07"},{"index":4,"sense_key":"building%1:14:00::","synset_id":"building.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"building%1:06:00::","call_ids":["senseval3.d002.s066.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"building%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":574,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001268,"input_uncached_usd":0.001148,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1180417080176994}
{"item_id":"senseval3.d002.s066.t001","gold_sense_keys":["corner%1:06:00::"],"candidates":[{"index":1,"sense_key":"corner%1:15:02::","synset_id":"corner.n.01"},{"index":2,"sense_key":"corner%1:15:00::","synset_id":"corner.n.02"},{"index":3,"sense_key":"corner%1:06:02::","synset_id":"corner.n.03"},{"index":4,"sense_key":"corner%1:06:00::","synset_id":"corner.n.04"},{"index":5,"sense_key":"corner%1:15:01::","synset_id":"corner.n.05"},{"index":6,"sense_key":"corner%1:25:00::","synset_id":"recess.n.02"},{"index":7,"sense_key":"corner%1:26:00::","synset_id":"corner.n.07"},{"index":8,"sense_key":"corner%1:26:01::","synset_id":"corner.n.08"},{"index":9,"sense_key":"corner%1:17:00::","synset_id":"corner.n.09"},{"index":10,"sense_key":"corner%1:15:03::","synset_id":"corner.n.10"},{"index":11,"sense_key":"corner%1:06:01::","synset_id":"corner.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"corner%1:06:00::","call_ids":["senseval3.d002.s066.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"corner%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":911,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002362,"input_uncached_usd":0.0018219999999999998,"input_cached_usd":0.0,"output_usd":0.00054,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1906062080524862}
{"item_id":"senseval3.d002.s066.t002","gold_sense_keys":["damage%2:30:00::"],"candidates":[{"index":1,"sense_key":"damage%2:30:00::","synset_id":"damage.v.01"},{"index":2,"sense_key":"damage%2:30:01::","synset_id":"damage.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"damage%2:30:00::","call_ids":["senseval3.d002.s066.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"damage%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":428,"cached_input_tokens":0,"output_tokens":97,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001826,"input_uncached_usd":0.000856,"input_cached_usd":0.0,"output_usd":0.00097,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5167168750194833}
{"item_id":"senseval3.d002.s066.t003","gold_sense_keys":["old%3:00:02::"],"candidates":[{"index":1,"sense_key":"old%3:00:02::","synset_id":"old.a.01"},{"index":2,"sense_key":"old%3:00:01::","synset_id":"old.a.02"},{"index":3,"sense_key":"old%5:00:00:familiar:00","synset_id":"old.s.01"},{"index":4,"sense_key":"old%5:00:00:experienced:00","synset_id":"old.s.02"},{"index":5,"sense_key":"old%5:00:03:past:00","synset_id":"erstwhile.s.01"},{"index":6,"sense_key":"old%5:00:00:genuine:00","synset_id":"honest-to-god.s.01"},{"index":7,"sense_key":"old%5:00:00:early:01","synset_id":"old.s.05"},{"index":8,"sense_key":"old%5:00:00:preceding:00","synset_id":"previous.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"old%3:00:02::","call_ids":["senseval3.d002.s066.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"old%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":826,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001772,"input_uncached_usd":0.0016519999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1068064159480855}
{"item_id":"senseval3.d002.s066.t004","gold_sense_keys":["lady%1:18:02::"],"candidates":[{"index":1,"sense_key":"lady%1:18:02::","synset_id":"lady.n.01"},{"index":2,"sense_key":"lady%1:18:01::","synset_id":"dame.n.02"},{"index":3,"sense_key":"lady%1:18:00::","synset_id":"lady.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"lady%1:18:02::","call_ids":["senseval3.d002.s066.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"lady%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":494,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001108,"input_uncached_usd":0.000988,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3460138749796897}
{"item_id":"senseval3.d002.s066.t005","gold_sense_keys":["old%3:00:02::"],"candidates":[{"index":1,"sense_key":"old%3:00:02::","synset_id":"old.a.01"},{"index":2,"sense_key":"old%3:00:01::","synset_id":"old.a.02"},{"index":3,"sense_key":"old%5:00:00:familiar:00","synset_id":"old.s.01"},{"index":4,"sense_key":"old%5:00:00:experienced:00","synset_id":"old.s.02"},{"index":5,"sense_key":"old%5:00:03:past:00","synset_id":"erstwhile.s.01"},{"index":6,"sense_key":"old%5:00:00:genuine:00","synset_id":"honest-to-god.s.01"},{"index":7,"sense_key":"old%5:00:00:early:01","synset_id":"old.s.05"},{"index":8,"sense_key":"old%5:00:00:preceding:00","synset_id":"previous.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"old%3:00:02::","call_ids":["senseval3.d002.s066.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"old%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":826,"cached_input_tokens":0,"output_tokens":82,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002472,"input_uncached_usd":0.0016519999999999998,"input_cached_usd":0.0,"output_usd":0.0008200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2198247499763966}
{"item_id":"senseval3.d002.s066.t006","gold_sense_keys":["mother%1:18:00::"],"candidates":[{"index":1,"sense_key":"mother%1:18:00::","synset_id":"mother.n.01"},{"index":2,"sense_key":"mother%1:27:00::","synset_id":"mother.n.02"},{"index":3,"sense_key":"mother%1:18:02::","synset_id":"mother.n.03"},{"index":4,"sense_key":"mother%1:18:01::","synset_id":"mother.n.04"},{"index":5,"sense_key":"mother%1:09:00::","synset_id":"mother.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"mother%1:18:00::","call_ids":["senseval3.d002.s066.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"mother%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":575,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00168,"input_uncached_usd":0.00115,"input_cached_usd":0.0,"output_usd":0.0005300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4035148749826476}
{"item_id":"senseval3.d002.s067.t000","gold_sense_keys":["book%1:06:00::"],"candidates":[{"index":1,"sense_key":"book%1:10:00::","synset_id":"book.n.01"},{"index":2,"sense_key":"book%1:06:00::","synset_id":"book.n.02"},{"index":3,"sense_key":"book%1:10:02::","synset_id":"record.n.05"},{"index":4,"sense_key":"book%1:10:01::","synset_id":"script.n.01"},{"index":5,"sense_key":"book%1:21:00::","synset_id":"ledger.n.01"},{"index":6,"sense_key":"book%1:14:01::","synset_id":"book.n.06"},{"index":7,"sense_key":"book%1:14:00::","synset_id":"book.n.07"},{"index":8,"sense_key":"book%1:10:05::","synset_id":"koran.n.01"},{"index":9,"sense_key":"book%1:10:04::","synset_id":"bible.n.01"},{"index":10,"sense_key":"book%1:10:03::","synset_id":"book.n.10"},{"index":11,"sense_key":"book%1:06:02::","synset_id":"book.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"book%1:06:00::","call_ids":["senseval3.d002.s067.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"book%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1021,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002692,"input_uncached_usd":0.002042,"input_cached_usd":0.0,"output_usd":0.0006500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3887422499246895}
{"item_id":"senseval3.d002.s068.t000","gold_sense_keys":["type%2:32:00::"],"candidates":[{"index":1,"sense_key":"type%2:32:00::","synset_id":"type.v.01"},{"index":2,"sense_key":"type%2:31:00::","synset_id":"type.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"type%2:32:00::","call_ids":["senseval3.d002.s068.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"type%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":423,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.000966,"input_uncached_usd":0.000846,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7824845829745755}
{"item_id":"senseval3.d002.s068.t001","gold_sense_keys":["standing%3:00:00::"],"candidates":[{"index":1,"sense_key":"standing%5:00:00:erect:00","synset_id":"standing.s.01"},{"index":2,"sense_key":"standing%5:00:00:permanent:00","synset_id":"standing.s.02"},{"index":3,"sense_key":"standing%3:00:02::","synset_id":"standing.a.03"},{"index":4,"sense_key":"standing%3:00:01::","synset_id":"standing.a.04"},{"index":5,"sense_key":"standing%3:00:00::","synset_id":"standing.a.05"},{"index":6,"sense_key":"standing%5:00:00:regular:02","synset_id":"standing.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"standing%3:00:00::","call_ids":["senseval3.d002.s068.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"standing%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":595,"cached_input_tokens":0,"output_tokens":134,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00253,"input_uncached_usd":0.0011899999999999999,"input_cached_usd":0.0,"output_usd":0.00134,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1664560839999467}
{"item_id":"senseval3.d002.s068.t002","gold_sense_keys":["position%1:07:01::"],"candidates":[{"index":1,"sense_key":"position%1:15:00::","synset_id":"position.n.01"},{"index":2,"sense_key":"position%1:15:02::","synset_id":"military_position.n.01"},{"index":3,"sense_key":"position%1:09:00::","synset_id":"position.n.03"},{"index":4,"sense_key":"position%1:07:01::","synset_id":"position.n.04"},{"index":5,"sense_key":"position%1:26:00::","synset_id":"status.n.01"},{"index":6,"sense_key":"position%1:04:01::","synset_id":"position.n.06"},{"index":7,"sense_key":"position%1:07:00::","synset_id":"position.n.07"},{"index":8,"sense_key":"position%1:15:03::","synset_id":"position.n.08"},{"index":9,"sense_key":"position%1:04:02::","synset_id":"position.n.09"},{"index":10,"sense_key":"position%1:04:00::","synset_id":"placement.n.03"},{"index":11,"sense_key":"position%1:26:02::","synset_id":"situation.n.02"},{"index":12,"sense_key":"position%1:09:01::","synset_id":"position.n.12"},{"index":13,"sense_key":"position%1:10:00::","synset_id":"side.n.10"},{"index":14,"sense_key":"position%1:10:01::","synset_id":"place.n.15"},{"index":15,"sense_key":"position%1:04:03::","synset_id":"stead.n.01"},{"index":16,"sense_key":"position%1:04:04::","synset_id":"position.n.16"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"position%1:07:01::","call_ids":["senseval3.d002.s068.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"position%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1239,"cached_input_tokens":0,"output_tokens":49,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0029679999999999997,"input_uncached_usd":0.0024779999999999997,"input_cached_usd":0.0,"output_usd":0.0004900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6303718329872936}
{"item_id":"senseval3.d002.s072.t000","gold_sense_keys":["feel%2:39:00::"],"candidates":[{"index":1,"sense_key":"feel%2:37:00::","synset_id":"feel.v.01"},{"index":2,"sense_key":"feel%2:31:00::","synset_id":"find.v.05"},{"index":3,"sense_key":"feel%2:39:00::","synset_id":"feel.v.03"},{"index":4,"sense_key":"feel%2:29:00::","synset_id":"feel.v.04"},{"index":5,"sense_key":"feel%2:31:01::","synset_id":"feel.v.05"},{"index":6,"sense_key":"feel%2:39:01::","synset_id":"feel.v.06"},{"index":7,"sense_key":"feel%2:42:00::","synset_id":"feel.v.07"},{"index":8,"sense_key":"feel%2:39:10::","synset_id":"feel.v.08"},{"index":9,"sense_key":"feel%2:35:00::","synset_id":"feel.v.09"},{"index":10,"sense_key":"feel%2:35:01::","synset_id":"palpate.v.01"},{"index":11,"sense_key":"feel%2:40:00::","synset_id":"feel.v.11"},{"index":12,"sense_key":"feel%2:39:09::","synset_id":"feel.v.12"},{"index":13,"sense_key":"feel%2:35:11::","synset_id":"feel.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"feel%2:39:00::","call_ids":["senseval3.d002.s072.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"feel%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1104,"cached_input_tokens":0,"output_tokens":87,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003078,"input_uncached_usd":0.002208,"input_cached_usd":0.0,"output_usd":0.0008700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3879147919360548}
{"item_id":"senseval3.d002.s072.t001","gold_sense_keys":["thing%1:06:01::"],"candidates":[{"index":1,"sense_key":"thing%1:26:00::","synset_id":"thing.n.01"},{"index":2,"sense_key":"thing%1:04:00::","synset_id":"thing.n.02"},{"index":3,"sense_key":"thing%1:09:00::","synset_id":"thing.n.03"},{"index":4,"sense_key":"thing%1:06:00::","synset_id":"thing.n.04"},{"index":5,"sense_key":"thing%1:11:00::","synset_id":"thing.n.05"},{"index":6,"sense_key":"thing%1:09:02::","synset_id":"matter.n.01"},{"index":7,"sense_key":"thing%1:10:00::","synset_id":"thing.n.07"},{"index":8,"sense_key":"thing%1:06:01::","synset_id":"thing.n.08"},{"index":9,"sense_key":"thing%1:07:00::","synset_id":"thing.n.09"},{"index":10,"sense_key":"thing%1:09:01::","synset_id":"thing.n.10"},{"index":11,"sense_key":"thing%1:12:00::","synset_id":"thing.n.11"},{"index":12,"sense_key":"thing%1:03:00::","synset_id":"thing.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"thing%1:06:01::","call_ids":["senseval3.d002.s072.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"thing%1:06:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":870,"cached_input_tokens":0,"output_tokens":166,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0034000000000000002,"input_uncached_usd":0.00174,"input_cached_usd":0.0,"output_usd":0.0016600000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0824359169928357}
{"item_id":"senseval3.d002.s072.t002","gold_sense_keys":["notice%2:39:00::"],"candidates":[{"index":1,"sense_key":"notice%2:39:04::","synset_id":"detect.v.01"},{"index":2,"sense_key":"notice%2:39:00::","synset_id":"notice.v.02"},{"index":3,"sense_key":"notice%2:32:00::","synset_id":"comment.v.01"},{"index":4,"sense_key":"notice%2:32:01::","synset_id":"notice.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"notice%2:39:00::","call_ids":["senseval3.d002.s072.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"notice%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":607,"cached_input_tokens":0,"output_tokens":92,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002134,"input_uncached_usd":0.001214,"input_cached_usd":0.0,"output_usd":0.00092,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8937477500876412}
{"item_id":"senseval3.d002.s072.t003","gold_sense_keys":["strange%3:00:00::"],"candidates":[{"index":1,"sense_key":"strange%3:00:00::","synset_id":"strange.a.01"},{"index":2,"sense_key":"strange%5:00:01:unfamiliar:00","synset_id":"strange.s.01"},{"index":3,"sense_key":"strange%3:00:01::","synset_id":"foreign.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"strange%3:00:00::","call_ids":["senseval3.d002.s072.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"strange%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":532,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0011840000000000002,"input_uncached_usd":0.001064,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4005500000203028}
{"item_id":"senseval3.d002.s072.t004","gold_sense_keys":["bird%1:05:00::"],"candidates":[{"index":1,"sense_key":"bird%1:05:00::","synset_id":"bird.n.01"},{"index":2,"sense_key":"bird%1:13:00::","synset_id":"bird.n.02"},{"index":3,"sense_key":"bird%1:18:00::","synset_id":"dame.n.01"},{"index":4,"sense_key":"bird%1:10:00::","synset_id":"boo.n.01"},{"index":5,"sense_key":"bird%1:06:01::","synset_id":"shuttlecock.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bird%1:05:00::","call_ids":["senseval3.d002.s072.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bird%1:05:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":621,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0013620000000000001,"input_uncached_usd":0.001242,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9996838748920709}
{"item_id":"senseval3.d002.s072.t005","gold_sense_keys":["behavior%1:04:01::"],"candidates":[{"index":1,"sense_key":"behavior%1:04:00::","synset_id":"behavior.n.01"},{"index":2,"sense_key":"behavior%1:26:00::","synset_id":"behavior.n.02"},{"index":3,"sense_key":"behavior%1:07:00::","synset_id":"demeanor.n.01"},{"index":4,"sense_key":"behavior%1:04:01::","synset_id":"behavior.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"behavior%1:04:01::","call_ids":["senseval3.d002.s072.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"behavior%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":579,"cached_input_tokens":0,"output_tokens":141,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002568,"input_uncached_usd":0.001158,"input_cached_usd":0.0,"output_usd":0.00141,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.4892131249653175}
{"item_id":"senseval3.d002.s073.t000","gold_sense_keys":["duck%1:05:00::"],"candidates":[{"index":1,"sense_key":"duck%1:05:00::","synset_id":"duck.n.01"},{"index":2,"sense_key":"duck%1:23:00::","synset_id":"duck.n.02"},{"index":3,"sense_key":"duck%1:13:00::","synset_id":"duck.n.03"},{"index":4,"sense_key":"duck%1:06:00::","synset_id":"duck.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"duck%1:05:00::","call_ids":["senseval3.d002.s073.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"duck%1:05:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":503,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001126,"input_uncached_usd":0.001006,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.536644084029831}
{"item_id":"senseval3.d002.s073.t001","gold_sense_keys":["swarm%1:14:00::"],"candidates":[{"index":1,"sense_key":"swarm%1:14:01::","synset_id":"drove.n.02"},{"index":2,"sense_key":"swarm%1:14:00::","synset_id":"swarm.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"swarm%1:14:01::","call_ids":["senseval3.d002.s073.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"swarm%1:14:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":426,"cached_input_tokens":0,"output_tokens":251,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003362,"input_uncached_usd":0.000852,"input_cached_usd":0.0,"output_usd":0.00251,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.7221794580109417}
{"item_id":"senseval3.d002.s076.t000","gold_sense_keys":["feel%2:39:00::"],"candidates":[{"index":1,"sense_key":"feel%2:37:00::","synset_id":"feel.v.01"},{"index":2,"sense_key":"feel%2:31:00::","synset_id":"find.v.05"},{"index":3,"sense_key":"feel%2:39:00::","synset_id":"feel.v.03"},{"index":4,"sense_key":"feel%2:29:00::","synset_id":"feel.v.04"},{"index":5,"sense_key":"feel%2:31:01::","synset_id":"feel.v.05"},{"index":6,"sense_key":"feel%2:39:01::","synset_id":"feel.v.06"},{"index":7,"sense_key":"feel%2:42:00::","synset_id":"feel.v.07"},{"index":8,"sense_key":"feel%2:39:10::","synset_id":"feel.v.08"},{"index":9,"sense_key":"feel%2:35:00::","synset_id":"feel.v.09"},{"index":10,"sense_key":"feel%2:35:01::","synset_id":"palpate.v.01"},{"index":11,"sense_key":"feel%2:40:00::","synset_id":"feel.v.11"},{"index":12,"sense_key":"feel%2:39:09::","synset_id":"feel.v.12"},{"index":13,"sense_key":"feel%2:35:11::","synset_id":"feel.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"feel%2:39:00::","call_ids":["senseval3.d002.s076.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"feel%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1110,"cached_input_tokens":0,"output_tokens":89,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00311,"input_uncached_usd":0.0022199999999999998,"input_cached_usd":0.0,"output_usd":0.0008900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.7960091669810936}
{"item_id":"senseval3.d002.s076.t003","gold_sense_keys":["second%1:28:00::"],"candidates":[{"index":1,"sense_key":"second%1:28:00::","synset_id":"second.n.01"},{"index":2,"sense_key":"second%1:28:01::","synset_id":"moment.n.02"},{"index":3,"sense_key":"second%1:04:00::","synset_id":"second_base.n.02"},{"index":4,"sense_key":"second%1:28:02::","synset_id":"moment.n.01"},{"index":5,"sense_key":"second%1:24:00::","synset_id":"second.n.05"},{"index":6,"sense_key":"second%1:23:00::","synset_id":"second.n.06"},{"index":7,"sense_key":"second%1:18:00::","synset_id":"second.n.07"},{"index":8,"sense_key":"second%1:10:00::","synset_id":"second.n.08"},{"index":9,"sense_key":"second%1:06:00::","synset_id":"second_gear.n.01"},{"index":10,"sense_key":"second%1:06:01::","synset_id":"irregular.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"second%1:28:00::","call_ids":["senseval3.d002.s076.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"second%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":900,"cached_input_tokens":0,"output_tokens":94,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0027400000000000002,"input_uncached_usd":0.0018,"input_cached_usd":0.0,"output_usd":0.0009400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.715320791932754}
{"item_id":"senseval3.d002.s077.t000","gold_sense_keys":["numb%5:00:00:afraid:00"],"candidates":[{"index":1,"sense_key":"numb%5:00:00:insensible:00","synset_id":"asleep.s.01"},{"index":2,"sense_key":"numb%5:00:00:insensitive:02","synset_id":"dead.s.04"},{"index":3,"sense_key":"numb%5:00:00:afraid:00","synset_id":"numb.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"numb%5:00:00:insensitive:02","call_ids":["senseval3.d002.s077.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"numb%5:00:00:insensitive:02","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":530,"cached_input_tokens":0,"output_tokens":153,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0025900000000000003,"input_uncached_usd":0.00106,"input_cached_usd":0.0,"output_usd":0.0015300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8329337919130921}
{"item_id":"senseval3.d002.s080.t001","gold_sense_keys":["seem%2:39:00::"],"candidates":[{"index":1,"sense_key":"seem%2:39:00::","synset_id":"look.v.02"},{"index":2,"sense_key":"seem%2:39:01::","synset_id":"appear.v.04"},{"index":3,"sense_key":"seem%2:42:00::","synset_id":"seem.v.03"},{"index":4,"sense_key":"seem%2:39:02::","synset_id":"seem.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"seem%2:39:00::","call_ids":["senseval3.d002.s080.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"seem%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":607,"cached_input_tokens":0,"output_tokens":129,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002504,"input_uncached_usd":0.001214,"input_cached_usd":0.0,"output_usd":0.0012900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7539746250258759}
{"item_id":"senseval3.d002.s080.t002","gold_sense_keys":["part%1:09:00::","part%1:15:00::","part%1:24:00::"],"candidates":[{"index":1,"sense_key":"part%1:24:00::","synset_id":"part.n.01"},{"index":2,"sense_key":"part%1:06:00::","synset_id":"part.n.02"},{"index":3,"sense_key":"part%1:17:00::","synset_id":"part.n.03"},{"index":4,"sense_key":"part%1:09:02::","synset_id":"part.n.04"},{"index":5,"sense_key":"part%1:15:00::","synset_id":"region.n.01"},{"index":6,"sense_key":"part%1:04:01::","synset_id":"function.n.03"},{"index":7,"sense_key":"part%1:09:01::","synset_id":"character.n.04"},{"index":8,"sense_key":"part%1:21:00::","synset_id":"share.n.01"},{"index":9,"sense_key":"part%1:09:00::","synset_id":"part.n.09"},{"index":10,"sense_key":"part%1:08:01::","synset_id":"part.n.10"},{"index":11,"sense_key":"part%1:10:00::","synset_id":"part.n.11"},{"index":12,"sense_key":"part%1:04:03::","synset_id":"contribution.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"part%1:15:00::","call_ids":["senseval3.d002.s080.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"part%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1157,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002994,"input_uncached_usd":0.002314,"input_cached_usd":0.0,"output_usd":0.00068,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6454302080674097}
{"item_id":"senseval3.d002.s080.t003","gold_sense_keys":["town%1:15:00::"],"candidates":[{"index":1,"sense_key":"town%1:15:00::","synset_id":"town.n.01"},{"index":2,"sense_key":"town%1:14:00::","synset_id":"town.n.02"},{"index":3,"sense_key":"town%1:15:01::","synset_id":"township.n.01"},{"index":4,"sense_key":"town%1:18:00::","synset_id":"town.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"town%1:15:00::","call_ids":["senseval3.d002.s080.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"town%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":577,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001274,"input_uncached_usd":0.0011539999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.008030250086449}
{"item_id":"senseval3.d002.s081.t001","gold_sense_keys":["phone%1:06:00::"],"candidates":[{"index":1,"sense_key":"phone%1:06:00::","synset_id":"telephone.n.01"},{"index":2,"sense_key":"phone%1:10:00::","synset_id":"phone.n.02"},{"index":3,"sense_key":"phone%1:06:01::","synset_id":"earphone.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"phone%1:06:00::","call_ids":["senseval3.d002.s081.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"phone%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":643,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0014060000000000001,"input_uncached_usd":0.001286,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0315401669358835}
{"item_id":"senseval3.d002.s081.t002","gold_sense_keys":["mess%1:23:00::","mess%1:26:00::"],"candidates":[{"index":1,"sense_key":"mess%1:26:00::","synset_id":"mess.n.01"},{"index":2,"sense_key":"mess%1:26:02::","synset_id":"fix.n.01"},{"index":3,"sense_key":"mess%1:13:01::","synset_id":"mess.n.03"},{"index":4,"sense_key":"mess%1:13:00::","synset_id":"mess.n.04"},{"index":5,"sense_key":"mess%1:06:00::","synset_id":"mess.n.05"},{"index":6,"sense_key":"mess%1:23:00::","synset_id":"batch.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"mess%1:26:00::","call_ids":["senseval3.d002.s081.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"mess%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":766,"cached_input_tokens":0,"output_tokens":130,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0028320000000000003,"input_uncached_usd":0.001532,"input_cached_usd":0.0,"output_usd":0.0013000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6040131250629202}
{"item_id":"senseval3.d002.s081.t004","gold_sense_keys":["wine%1:13:00::"],"candidates":[{"index":1,"sense_key":"wine%1:13:00::","synset_id":"wine.n.01"},{"index":2,"sense_key":"wine%1:07:00::","synset_id":"wine.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"wine%1:13:00::","call_ids":["senseval3.d002.s081.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"wine%1:13:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":482,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001084,"input_uncached_usd":0.0009639999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.958790042088367}
{"item_id":"senseval3.d002.s081.t005","gold_sense_keys":["floor%1:06:00::"],"candidates":[{"index":1,"sense_key":"floor%1:06:00::","synset_id":"floor.n.01"},{"index":2,"sense_key":"floor%1:06:01::","synset_id":"floor.n.02"},{"index":3,"sense_key":"floor%1:10:00::","synset_id":"floor.n.03"},{"index":4,"sense_key":"floor%1:17:01::","synset_id":"floor.n.04"},{"index":5,"sense_key":"floor%1:17:00::","synset_id":"floor.n.05"},{"index":6,"sense_key":"floor%1:17:02::","synset_id":"floor.n.06"},{"index":7,"sense_key":"floor%1:14:00::","synset_id":"floor.n.07"},{"index":8,"sense_key":"floor%1:07:00::","synset_id":"floor.n.08"},{"index":9,"sense_key":"floor%1:06:03::","synset_id":"floor.n.09"},{"index":10,"sense_key":"floor%1:06:02::","synset_id":"floor.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"floor%1:06:00::","call_ids":["senseval3.d002.s081.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"floor%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":955,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0020299999999999997,"input_uncached_usd":0.0019099999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1160761249484494}
{"item_id":"senseval3.d002.s081.t006","gold_sense_keys":["big%3:00:01::"],"candidates":[{"index":1,"sense_key":"big%3:00:01::","synset_id":"large.a.01"},{"index":2,"sense_key":"big%5:00:00:important:00","synset_id":"big.s.01"},{"index":3,"sense_key":"big%5:00:00:intense:00","synset_id":"bad.s.01"},{"index":4,"sense_key":"big%5:00:00:loud:00","synset_id":"big.s.03"},{"index":5,"sense_key":"big%5:00:00:conspicuous:00","synset_id":"big.s.04"},{"index":6,"sense_key":"big%5:00:00:intemperate:00","synset_id":"big.s.05"},{"index":7,"sense_key":"big%5:00:02:proud:00","synset_id":"boastful.s.01"},{"index":8,"sense_key":"big%5:00:00:proud:00","synset_id":"big.s.07"},{"index":9,"sense_key":"big%5:00:00:mature:01","synset_id":"adult.s.01"},{"index":10,"sense_key":"big%5:00:00:heavy:08","synset_id":"big.s.09"},{"index":11,"sense_key":"big%5:00:00:generous:02","synset_id":"big.s.10"},{"index":12,"sense_key":"big%5:00:00:generous:01","synset_id":"big.s.11"},{"index":13,"sense_key":"big%5:00:00:pregnant:00","synset_id":"big.s.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"big%3:00:01::","call_ids":["senseval3.d002.s081.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"big%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1202,"cached_input_tokens":0,"output_tokens":48,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002884,"input_uncached_usd":0.002404,"input_cached_usd":0.0,"output_usd":0.00048000000000000007,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.565145374974236}
{"item_id":"senseval3.d002.s081.t007","gold_sense_keys":["old%3:00:01::"],"candidates":[{"index":1,"sense_key":"old%3:00:02::","synset_id":"old.a.01"},{"index":2,"sense_key":"old%3:00:01::","synset_id":"old.a.02"},{"index":3,"sense_key":"old%5:00:00:familiar:00","synset_id":"old.s.01"},{"index":4,"sense_key":"old%5:00:00:experienced:00","synset_id":"old.s.02"},{"index":5,"sense_key":"old%5:00:03:past:00","synset_id":"erstwhile.s.01"},{"index":6,"sense_key":"old%5:00:00:genuine:00","synset_id":"honest-to-god.s.01"},{"index":7,"sense_key":"old%5:00:00:early:01","synset_id":"old.s.05"},{"index":8,"sense_key":"old%5:00:00:preceding:00","synset_id":"previous.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"old%3:00:01::","call_ids":["senseval3.d002.s081.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"old%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":893,"cached_input_tokens":0,"output_tokens":166,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0034460000000000003,"input_uncached_usd":0.0017859999999999998,"input_cached_usd":0.0,"output_usd":0.0016600000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.135626250063069}
{"item_id":"senseval3.d002.s082.t000","gold_sense_keys":["quiver%1:26:00::"],"candidates":[{"index":1,"sense_key":"quiver%1:12:00::","synset_id":"frisson.n.01"},{"index":2,"sense_key":"quiver%1:26:00::","synset_id":"shaking.n.02"},{"index":3,"sense_key":"quiver%1:06:00::","synset_id":"quiver.n.03"},{"index":4,"sense_key":"quiver%1:04:00::","synset_id":"vibration.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"quiver%1:26:00::","call_ids":["senseval3.d002.s082.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"quiver%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":633,"cached_input_tokens":0,"output_tokens":161,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002876,"input_uncached_usd":0.001266,"input_cached_usd":0.0,"output_usd":0.00161,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.4821333329891786}
{"item_id":"senseval3.d002.s082.t002","gold_sense_keys":["house%1:06:00::"],"candidates":[{"index":1,"sense_key":"house%1:06:00::","synset_id":"house.n.01"},{"index":2,"sense_key":"house%1:14:01::","synset_id":"firm.n.01"},{"index":3,"sense_key":"house%1:14:04::","synset_id":"house.n.03"},{"index":4,"sense_key":"house%1:14:03::","synset_id":"house.n.04"},{"index":5,"sense_key":"house%1:14:02::","synset_id":"house.n.05"},{"index":6,"sense_key":"house%1:14:00::","synset_id":"house.n.06"},{"index":7,"sense_key":"house%1:04:00::","synset_id":"house.n.07"},{"index":8,"sense_key":"house%1:15:00::","synset_id":"sign_of_the_zodiac.n.01"},{"index":9,"sense_key":"house%1:14:06::","synset_id":"house.n.09"},{"index":10,"sense_key":"house%1:14:05::","synset_id":"family.n.01"},{"index":11,"sense_key":"house%1:06:02::","synset_id":"theater.n.01"},{"index":12,"sense_key":"house%1:06:01::","synset_id":"house.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"house%1:06:00::","call_ids":["senseval3.d002.s082.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"house%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1042,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002204,"input_uncached_usd":0.002084,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1836161670507863}
{"item_id":"senseval3.d002.s082.t004","gold_sense_keys":["minute%1:28:00::"],"candidates":[{"index":1,"sense_key":"minute%1:28:00::","synset_id":"minute.n.01"},{"index":2,"sense_key":"minute%1:28:01::","synset_id":"moment.n.02"},{"index":3,"sense_key":"minute%1:28:02::","synset_id":"moment.n.01"},{"index":4,"sense_key":"minute%1:23:00::","synset_id":"minute.n.04"},{"index":5,"sense_key":"minute%1:10:00::","synset_id":"minute.n.05"},{"index":6,"sense_key":"minute%1:07:00::","synset_id":"hour.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"minute%1:28:00::","call_ids":["senseval3.d002.s082.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"minute%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":749,"cached_input_tokens":0,"output_tokens":51,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0020080000000000002,"input_uncached_usd":0.001498,"input_cached_usd":0.0,"output_usd":0.00051,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.402618500054814}
{"item_id":"senseval3.d002.s082.t005","gold_sense_keys":["unpredictable%5:00:00:sporadic:00"],"candidates":[{"index":1,"sense_key":"unpredictable%3:00:00::","synset_id":"unpredictable.a.01"},{"index":2,"sense_key":"unpredictable%5:00:00:indeterminable:00","synset_id":"unpredictable.s.01"},{"index":3,"sense_key":"unpredictable%5:00:00:sporadic:00","synset_id":"irregular.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"unpredictable%5:00:00:sporadic:00","call_ids":["senseval3.d002.s082.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"unpredictable%5:00:00:sporadic:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":540,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00175,"input_uncached_usd":0.00108,"input_cached_usd":0.0,"output_usd":0.00067,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3020114579703659}
{"item_id":"senseval3.d002.s082.t006","gold_sense_keys":["interval%1:28:00::"],"candidates":[{"index":1,"sense_key":"interval%1:28:00::","synset_id":"time_interval.n.01"},{"index":2,"sense_key":"interval%1:09:00::","synset_id":"interval.n.02"},{"index":3,"sense_key":"interval%1:07:00::","synset_id":"interval.n.03"},{"index":4,"sense_key":"interval%1:10:00::","synset_id":"interval.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"interval%1:28:00::","call_ids":["senseval3.d002.s082.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"interval%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":578,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001276,"input_uncached_usd":0.001156,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.7745747920125723}
{"item_id":"senseval3.d002.s082.t007","gold_sense_keys":["mouse%1:05:00::"],"candidates":[{"index":1,"sense_key":"mouse%1:05:00::","synset_id":"mouse.n.01"},{"index":2,"sense_key":"mouse%1:26:00::","synset_id":"shiner.n.01"},{"index":3,"sense_key":"mouse%1:18:00::","synset_id":"mouse.n.03"},{"index":4,"sense_key":"mouse%1:06:00::","synset_id":"mouse.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"mouse%1:05:00::","call_ids":["senseval3.d002.s082.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"mouse%1:05:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":669,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001458,"input_uncached_usd":0.001338,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5825800420716405}
{"item_id":"senseval3.d002.s082.t008","gold_sense_keys":["live%2:42:08::"],"candidates":[{"index":1,"sense_key":"live%2:42:08::","synset_id":"populate.v.01"},{"index":2,"sense_key":"live%2:42:06::","synset_id":"live.v.02"},{"index":3,"sense_key":"live%2:42:04::","synset_id":"survive.v.01"},{"index":4,"sense_key":"live%2:42:01::","synset_id":"exist.v.02"},{"index":5,"sense_key":"live%2:42:00::","synset_id":"be.v.11"},{"index":6,"sense_key":"live%2:31:00::","synset_id":"know.v.05"},{"index":7,"sense_key":"live%2:42:07::","synset_id":"live.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"live%2:42:08::","call_ids":["senseval3.d002.s082.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"live%2:42:08::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":878,"cached_input_tokens":0,"output_tokens":47,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002226,"input_uncached_usd":0.001756,"input_cached_usd":0.0,"output_usd":0.00047000000000000004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2696982499910519}
{"item_id":"senseval3.d002.s082.t010","gold_sense_keys":["take%2:40:08::"],"candidates":[{"index":1,"sense_key":"take%2:41:04::","synset_id":"take.v.01"},{"index":2,"sense_key":"take%2:40:06::","synset_id":"take.v.02"},{"index":3,"sense_key":"take%2:38:09::","synset_id":"lead.v.01"},{"index":4,"sense_key":"take%2:35:00::","synset_id":"take.v.04"},{"index":5,"sense_key":"take%2:30:01::","synset_id":"assume.v.03"},{"index":6,"sense_key":"take%2:31:07::","synset_id":"take.v.06"},{"index":7,"sense_key":"take%2:38:10::","synset_id":"bring.v.01"},{"index":8,"sense_key":"take%2:40:00::","synset_id":"take.v.08"},{"index":9,"sense_key":"take%2:38:02::","synset_id":"take.v.09"},{"index":10,"sense_key":"take%2:31:01::","synset_id":"choose.v.01"},{"index":11,"sense_key":"take%2:40:05::","synset_id":"accept.v.02"},{"index":12,"sense_key":"take%2:41:00::","synset_id":"fill.v.04"},{"index":13,"sense_key":"take%2:31:03::","synset_id":"consider.v.03"},{"index":14,"sense_key":"take%2:42:00::","synset_id":"necessitate.v.01"},{"index":15,"sense_key":"take%2:39:00::","synset_id":"take.v.15"},{"index":16,"sense_key":"take%2:32:02::","synset_id":"film.v.01"},{"index":17,"sense_key":"take%2:30:00::","synset_id":"remove.v.01"},{"index":18,"sense_key":"take%2:34:00::","synset_id":"consume.v.02"},{"index":19,"sense_key":"take%2:31:09::","synset_id":"take.v.19"},{"index":20,"sense_key":"take%2:40:15::","synset_id":"take.v.20"},{"index":21,"sense_key":"take%2:40:01::","synset_id":"take.v.21"},{"index":22,"sense_key":"take%2:38:00::","synset_id":"assume.v.05"},{"index":23,"sense_key":"take%2:40:07::","synset_id":"accept.v.05"},{"index":24,"sense_key":"take%2:30:08::","synset_id":"take.v.24"},{"index":25,"sense_key":"take%2:31:02::","synset_id":"learn.v.04"},{"index":26,"sense_key":"take%2:32:00::","synset_id":"claim.v.05"},{"index":27,"sense_key":"take%2:38:05::","synset_id":"take.v.27"},{"index":28,"sense_key":"take%2:33:09::","synset_id":"aim.v.01"},{"index":29,"sense_key":"take%2:30:09::","synset_id":"take.v.29"},{"index":30,"sense_key":"take%2:42:10::","synset_id":"carry.v.02"},{"index":31,"sense_key":"take%2:40:03::","synset_id":"lease.v.04"},{"index":32,"sense_key":"take%2:40:04::","synset_id":"subscribe.v.05"},{"index":33,"sense_key":"take%2:40:02::","synset_id":"take.v.33"},{"index":34,"sense_key":"take%2:40:08::","synset_id":"take.v.34"},{"index":35,"sense_key":"take%2:35:14::","synset_id":"take.v.35"},{"index":36,"sense_key":"take%2:32:09::","synset_id":"claim.v.04"},{"index":37,"sense_key":"take%2:42:03::","synset_id":"accept.v.08"},{"index":38,"sense_key":"take%2:42:15::","synset_id":"contain.v.05"},{"index":39,"sense_key":"take%2:41:13::","synset_id":"take.v.39"},{"index":40,"sense_key":"take%2:38:11::","synset_id":"drive.v.16"},{"index":41,"sense_key":"take%2:33:08::","synset_id":"take.v.41"},{"index":42,"sense_key":"take%2:29:08::","synset_id":"contract.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":34,"chosen_sense_key":"take%2:40:08::","call_ids":["senseval3.d002.s082.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":34,"predicted_sense_key":"take%2:40:08::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2997,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.006633999999999999,"input_uncached_usd":0.005993999999999999,"input_cached_usd":0.0,"output_usd":0.00064,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9354194160550833}
{"item_id":"senseval3.d002.s082.t011","gold_sense_keys":["refuge%1:06:00::"],"candidates":[{"index":1,"sense_key":"refuge%1:15:00::","synset_id":"safety.n.02"},{"index":2,"sense_key":"refuge%1:07:00::","synset_id":"recourse.n.02"},{"index":3,"sense_key":"refuge%1:06:00::","synset_id":"refuge.n.03"},{"index":4,"sense_key":"refuge%1:04:00::","synset_id":"recourse.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"refuge%1:06:00::","call_ids":["senseval3.d002.s082.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"refuge%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":631,"cached_input_tokens":0,"output_tokens":179,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003052,"input_uncached_usd":0.0012619999999999999,"input_cached_usd":0.0,"output_usd":0.0017900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2386530420044437}
{"item_id":"senseval3.d002.s083.t001","gold_sense_keys":["distressed%5:00:01:troubled:00","distressed%5:00:02:troubled:00"],"candidates":[{"index":1,"sense_key":"distressed%5:00:00:troubled:00","synset_id":"distressed.s.01"},{"index":2,"sense_key":"distressed%3:00:04::","synset_id":"dysphoric.a.01"},{"index":3,"sense_key":"distressed%5:00:02:troubled:00","synset_id":"stressed.s.01"},{"index":4,"sense_key":"distressed%5:00:01:troubled:00","synset_id":"disquieted.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"distressed%5:00:01:troubled:00","call_ids":["senseval3.d002.s083.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"distressed%5:00:01:troubled:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":736,"cached_input_tokens":0,"output_tokens":81,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002282,"input_uncached_usd":0.001472,"input_cached_usd":0.0,"output_usd":0.0008100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4931998749962077}
{"item_id":"senseval3.d002.s084.t001","gold_sense_keys":["roll%2:38:01::","roll%2:38:10::"],"candidates":[{"index":1,"sense_key":"roll%2:38:00::","synset_id":"roll.v.01"},{"index":2,"sense_key":"roll%2:38:01::","synset_id":"wheel.v.03"},{"index":3,"sense_key":"roll%2:42:00::","synset_id":"roll.v.03"},{"index":4,"sense_key":"roll%2:35:05::","synset_id":"roll_out.v.01"},{"index":5,"sense_key":"roll%2:39:00::","synset_id":"roll.v.05"},{"index":6,"sense_key":"roll%2:35:00::","synset_id":"wind.v.03"},{"index":7,"sense_key":"roll%2:35:01::","synset_id":"roll.v.07"},{"index":8,"sense_key":"roll%2:30:10::","synset_id":"roll.v.08"},{"index":9,"sense_key":"roll%2:29:06::","synset_id":"roll.v.09"},{"index":10,"sense_key":"roll%2:40:00::","synset_id":"hustle.v.03"},{"index":11,"sense_key":"roll%2:38:10::","synset_id":"roll.v.11"},{"index":12,"sense_key":"roll%2:38:09::","synset_id":"roll.v.12"},{"index":13,"sense_key":"roll%2:38:12::","synset_id":"roll.v.13"},{"index":14,"sense_key":"roll%2:38:11::","synset_id":"roll.v.14"},{"index":15,"sense_key":"roll%2:32:00::","synset_id":"roll.v.15"},{"index":16,"sense_key":"roll%2:30:08::","synset_id":"seethe.v.04"},{"index":17,"sense_key":"roll%2:30:06::","synset_id":"roll.v.17"},{"index":18,"sense_key":"roll%2:30:07::","synset_id":"roll.v.18"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"roll%2:38:10::","call_ids":["senseval3.d002.s084.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"roll%2:38:10::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1500,"cached_input_tokens":0,"output_tokens":135,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00435,"input_uncached_usd":0.003,"input_cached_usd":0.0,"output_usd":0.00135,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8863567089429125}
{"item_id":"senseval3.d002.s084.t002","gold_sense_keys":["town%1:15:00::"],"candidates":[{"index":1,"sense_key":"town%1:15:00::","synset_id":"town.n.01"},{"index":2,"sense_key":"town%1:14:00::","synset_id":"town.n.02"},{"index":3,"sense_key":"town%1:15:01::","synset_id":"township.n.01"},{"index":4,"sense_key":"town%1:18:00::","synset_id":"town.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"town%1:15:00::","call_ids":["senseval3.d002.s084.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"town%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":639,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001928,"input_uncached_usd":0.0012779999999999998,"input_cached_usd":0.0,"output_usd":0.0006500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4065589999081567}
{"item_id":"senseval3.d002.s086.t001","gold_sense_keys":["get%2:30:00::"],"candidates":[{"index":1,"sense_key":"get%2:40:00::","synset_id":"get.v.01"},{"index":2,"sense_key":"get%2:30:00::","synset_id":"become.v.01"},{"index":3,"sense_key":"get%2:30:02::","synset_id":"get.v.03"},{"index":4,"sense_key":"get%2:30:03::","synset_id":"receive.v.02"},{"index":5,"sense_key":"get%2:38:00::","synset_id":"arrive.v.01"},{"index":6,"sense_key":"get%2:35:03::","synset_id":"bring.v.04"},{"index":7,"sense_key":"get%2:39:00::","synset_id":"experience.v.03"},{"index":8,"sense_key":"get%2:33:00::","synset_id":"pay_back.v.02"},{"index":9,"sense_key":"get%2:40:01::","synset_id":"have.v.17"},{"index":10,"sense_key":"get%2:32:00::","synset_id":"induce.v.02"},{"index":11,"sense_key":"get%2:35:09::","synset_id":"get.v.11"},{"index":12,"sense_key":"get%2:29:01::","synset_id":"grow.v.08"},{"index":13,"sense_key":"get%2:29:00::","synset_id":"contract.v.04"},{"index":14,"sense_key":"get%2:32:03::","synset_id":"get.v.14"},{"index":15,"sense_key":"get%2:30:01::","synset_id":"make.v.02"},{"index":16,"sense_key":"get%2:32:08::","synset_id":"drive.v.11"},{"index":17,"sense_key":"get%2:31:09::","synset_id":"catch.v.18"},{"index":18,"sense_key":"get%2:35:00::","synset_id":"catch.v.07"},{"index":19,"sense_key":"get%2:35:10::","synset_id":"get.v.19"},{"index":20,"sense_key":"get%2:31:03::","synset_id":"get.v.20"},{"index":21,"sense_key":"get%2:40:14::","synset_id":"get.v.21"},{"index":22,"sense_key":"get%2:40:13::","synset_id":"get.v.22"},{"index":23,"sense_key":"get%2:39:13::","synset_id":"catch.v.21"},{"index":24,"sense_key":"get%2:39:09::","synset_id":"catch.v.22"},{"index":25,"sense_key":"get%2:39:14::","synset_id":"get.v.25"},{"index":26,"sense_key":"get%2:38:15::","synset_id":"scram.v.01"},{"index":27,"sense_key":"get%2:38:12::","synset_id":"get.v.27"},{"index":28,"sense_key":"get%2:37:12::","synset_id":"get.v.28"},{"index":29,"sense_key":"get%2:37:11::","synset_id":"get.v.29"},{"index":30,"sense_key":"get%2:36:14::","synset_id":"catch.v.24"},{"index":31,"sense_key":"get%2:36:00::","synset_id":"draw.v.15"},{"index":32,"sense_key":"get%2:35:12::","synset_id":"get.v.32"},{"index":33,"sense_key":"get%2:31:10::","synset_id":"perplex.v.01"},{"index":34,"sense_key":"get%2:30:12::","synset_id":"get_down.v.07"},{"index":35,"sense_key":"get%2:29:11::","synset_id":"suffer.v.02"},{"index":36,"sense_key":"get%2:29:08::","synset_id":"beget.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"get%2:30:00::","call_ids":["senseval3.d002.s086.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"get%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2809,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.006298,"input_uncached_usd":0.005618,"input_cached_usd":0.0,"output_usd":0.00068,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.81753241608385}
{"item_id":"senseval3.d002.s086.t002","gold_sense_keys":["run%2:38:00::"],"candidates":[{"index":1,"sense_key":"run%2:38:00::","synset_id":"run.v.01"},{"index":2,"sense_key":"run%2:38:04::","synset_id":"scat.v.01"},{"index":3,"sense_key":"run%2:42:00::","synset_id":"run.v.03"},{"index":4,"sense_key":"run%2:41:02::","synset_id":"operate.v.01"},{"index":5,"sense_key":"run%2:42:04::","synset_id":"run.v.05"},{"index":6,"sense_key":"run%2:38:01::","synset_id":"run.v.06"},{"index":7,"sense_key":"run%2:35:00::","synset_id":"function.v.01"},{"index":8,"sense_key":"run%2:42:07::","synset_id":"range.v.01"},{"index":9,"sense_key":"run%2:33:06::","synset_id":"campaign.v.01"},{"index":10,"sense_key":"run%2:36:01::","synset_id":"play.v.18"},{"index":11,"sense_key":"run%2:38:11::","synset_id":"run.v.11"},{"index":12,"sense_key":"run%2:42:01::","synset_id":"tend.v.01"},{"index":13,"sense_key":"run%2:35:04::","synset_id":"run.v.13"},{"index":14,"sense_key":"run%2:30:03::","synset_id":"run.v.14"},{"index":15,"sense_key":"run%2:30:07::","synset_id":"run.v.15"},{"index":16,"sense_key":"run%2:42:02::","synset_id":"run.v.16"},{"index":17,"sense_key":"run%2:42:03::","synset_id":"prevail.v.03"},{"index":18,"sense_key":"run%2:42:06::","synset_id":"run.v.18"},{"index":19,"sense_key":"run%2:41:00::","synset_id":"run.v.19"},{"index":20,"sense_key":"run%2:36:02::","synset_id":"carry.v.15"},{"index":21,"sense_key":"run%2:36:00::","synset_id":"run.v.21"},{"index":22,"sense_key":"run%2:35:01::","synset_id":"guide.v.05"},{"index":23,"sense_key":"run%2:42:08::","synset_id":"run.v.23"},{"index":24,"sense_key":"run%2:41:01::","synset_id":"run.v.24"},{"index":25,"sense_key":"run%2:40:06::","synset_id":"run.v.25"},{"index":26,"sense_key":"run%2:38:10::","synset_id":"run.v.26"},{"index":27,"sense_key":"run%2:38:08::","synset_id":"run.v.27"},{"index":28,"sense_key":"run%2:38:03::","synset_id":"run.v.28"},{"index":29,"sense_key":"run%2:38:05::","synset_id":"run.v.29"},{"index":30,"sense_key":"run%2:42:05::","synset_id":"run.v.30"},{"index":31,"sense_key":"run%2:41:04::","synset_id":"run.v.31"},{"index":32,"sense_key":"run%2:38:02::","synset_id":"run.v.32"},{"index":33,"sense_key":"run%2:38:06::","synset_id":"run.v.33"},{"index":34,"sense_key":"run%2:38:13::","synset_id":"run.v.34"},{"index":35,"sense_key":"run%2:38:09::","synset_id":"ply.v.03"},{"index":36,"sense_key":"run%2:33:02::","synset_id":"hunt.v.01"},{"index":37,"sense_key":"run%2:33:01::","synset_id":"race.v.02"},{"index":38,"sense_key":"run%2:30:02::","synset_id":"move.v.13"},{"index":39,"sense_key":"run%2:30:00::","synset_id":"melt.v.01"},{"index":40,"sense_key":"run%2:30:01::","synset_id":"ladder.v.01"},{"index":41,"sense_key":"run%2:30:04::","synset_id":"run.v.41"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"run%2:38:00::","call_ids":["senseval3.d002.s086.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"run%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2811,"cached_input_tokens":0,"output_tokens":132,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.006941999999999999,"input_uncached_usd":0.005621999999999999,"input_cached_usd":0.0,"output_usd":0.0013200000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0906617080327123}
{"item_id":"senseval3.d002.s086.t003","gold_sense_keys":["door%1:06:00::","door%1:06:01::"],"candidates":[{"index":1,"sense_key":"door%1:06:00::","synset_id":"door.n.01"},{"index":2,"sense_key":"door%1:06:01::","synset_id":"doorway.n.01"},{"index":3,"sense_key":"door%1:07:00::","synset_id":"door.n.03"},{"index":4,"sense_key":"door%1:06:04::","synset_id":"door.n.04"},{"index":5,"sense_key":"door%1:06:03::","synset_id":"door.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"door%1:06:01::","call_ids":["senseval3.d002.s086.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"door%1:06:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":786,"cached_input_tokens":0,"output_tokens":142,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002992,"input_uncached_usd":0.0015719999999999998,"input_cached_usd":0.0,"output_usd":0.00142,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.4390882910229266}
{"item_id":"senseval3.d002.s086.t004","gold_sense_keys":["spend%2:42:00::"],"candidates":[{"index":1,"sense_key":"spend%2:42:00::","synset_id":"spend.v.01"},{"index":2,"sense_key":"spend%2:40:00::","synset_id":"spend.v.02"},{"index":3,"sense_key":"spend%2:40:01::","synset_id":"spend.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"spend%2:42:00::","call_ids":["senseval3.d002.s086.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"spend%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":606,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001332,"input_uncached_usd":0.001212,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8995742499828339}
{"item_id":"senseval3.d002.s086.t005","gold_sense_keys":["next%5:00:00:succeeding:00"],"candidates":[{"index":1,"sense_key":"next%5:00:00:succeeding:00","synset_id":"following.s.02"},{"index":2,"sense_key":"next%5:00:00:close:01","synset_id":"adjacent.s.01"},{"index":3,"sense_key":"next%5:00:00:incoming:00","synset_id":"future.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"next%5:00:00:succeeding:00","call_ids":["senseval3.d002.s086.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"next%5:00:00:succeeding:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":654,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001428,"input_uncached_usd":0.001308,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.0376417910447344}
{"item_id":"senseval3.d002.s086.t007","gold_sense_keys":["minute%1:28:00::"],"candidates":[{"index":1,"sense_key":"minute%1:28:00::","synset_id":"minute.n.01"},{"index":2,"sense_key":"minute%1:28:01::","synset_id":"moment.n.02"},{"index":3,"sense_key":"minute%1:28:02::","synset_id":"moment.n.01"},{"index":4,"sense_key":"minute%1:23:00::","synset_id":"minute.n.04"},{"index":5,"sense_key":"minute%1:10:00::","synset_id":"minute.n.05"},{"index":6,"sense_key":"minute%1:07:00::","synset_id":"hour.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"minute%1:28:00::","call_ids":["senseval3.d002.s086.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"minute%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":821,"cached_input_tokens":0,"output_tokens":46,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002102,"input_uncached_usd":0.001642,"input_cached_usd":0.0,"output_usd":0.00046,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3080848749959841}
{"item_id":"senseval3.d002.s086.t008","gold_sense_keys":["watch%2:39:00::","watch%2:39:02::","watch%2:41:00::"],"candidates":[{"index":1,"sense_key":"watch%2:39:00::","synset_id":"watch.v.01"},{"index":2,"sense_key":"watch%2:41:00::","synset_id":"watch.v.02"},{"index":3,"sense_key":"watch%2:39:05::","synset_id":"watch.v.03"},{"index":4,"sense_key":"watch%2:39:02::","synset_id":"watch.v.04"},{"index":5,"sense_key":"watch%2:39:01::","synset_id":"watch.v.05"},{"index":6,"sense_key":"watch%2:39:03::","synset_id":"watch.v.06"},{"index":7,"sense_key":"watch%2:32:09::","synset_id":"determine.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"watch%2:39:02::","call_ids":["senseval3.d002.s086.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"watch%2:39:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":905,"cached_input_tokens":0,"output_tokens":153,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00334,"input_uncached_usd":0.00181,"input_cached_usd":0.0,"output_usd":0.0015300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.894546957919374}
{"item_id":"senseval3.d002.s086.t009","gold_sense_keys":["brick%1:06:00::"],"candidates":[{"index":1,"sense_key":"brick%1:06:00::","synset_id":"brick.n.01"},{"index":2,"sense_key":"brick%1:18:00::","synset_id":"brick.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"brick%1:06:00::","call_ids":["senseval3.d002.s086.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"brick%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":556,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001232,"input_uncached_usd":0.001112,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4619812079472467}
{"item_id":"senseval3.d002.s086.t011","gold_sense_keys":["foot%1:08:01::"],"candidates":[{"index":1,"sense_key":"foot%1:08:01::","synset_id":"foot.n.01"},{"index":2,"sense_key":"foot%1:23:00::","synset_id":"foot.n.02"},{"index":3,"sense_key":"foot%1:15:00::","synset_id":"foot.n.03"},{"index":4,"sense_key":"foot%1:05:01::","synset_id":"animal_foot.n.01"},{"index":5,"sense_key":"foot%1:06:00::","synset_id":"foundation.n.03"},{"index":6,"sense_key":"foot%1:05:00::","synset_id":"foot.n.06"},{"index":7,"sense_key":"foot%1:04:00::","synset_id":"foot.n.07"},{"index":8,"sense_key":"foot%1:18:00::","synset_id":"foot.n.08"},{"index":9,"sense_key":"foot%1:14:00::","synset_id":"infantry.n.01"},{"index":10,"sense_key":"foot%1:10:00::","synset_id":"metrical_foot.n.01"},{"index":11,"sense_key":"foot%1:06:01::","synset_id":"foot.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"foot%1:08:01::","call_ids":["senseval3.d002.s086.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"foot%1:08:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1134,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002388,"input_uncached_usd":0.002268,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9476362499408424}
{"item_id":"senseval3.d002.s086.t013","gold_sense_keys":["flower%1:20:00::"],"candidates":[{"index":1,"sense_key":"flower%1:20:00::","synset_id":"flower.n.01"},{"index":2,"sense_key":"flower%1:20:02::","synset_id":"flower.n.02"},{"index":3,"sense_key":"flower%1:28:00::","synset_id":"flower.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"flower%1:20:00::","call_ids":["senseval3.d002.s086.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"flower%1:20:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":634,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001998,"input_uncached_usd":0.001268,"input_cached_usd":0.0,"output_usd":0.0007300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4411818330409005}
{"item_id":"senseval3.d002.s086.t014","gold_sense_keys":["wave%2:38:00::"],"candidates":[{"index":1,"sense_key":"wave%2:32:00::","synset_id":"beckon.v.01"},{"index":2,"sense_key":"wave%2:35:00::","synset_id":"brandish.v.01"},{"index":3,"sense_key":"wave%2:38:00::","synset_id":"roll.v.11"},{"index":4,"sense_key":"wave%2:35:01::","synset_id":"curl.v.04"},{"index":5,"sense_key":"wave%2:29:00::","synset_id":"wave.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"wave%2:38:00::","call_ids":["senseval3.d002.s086.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"wave%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":761,"cached_input_tokens":0,"output_tokens":120,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002722,"input_uncached_usd":0.001522,"input_cached_usd":0.0,"output_usd":0.0012000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.8867888749809936}
{"item_id":"senseval3.d002.s087.t000","gold_sense_keys":["amazing%5:00:00:surprising:00"],"candidates":[{"index":1,"sense_key":"amazing%5:00:00:surprising:00","synset_id":"amazing.s.01"},{"index":2,"sense_key":"amazing%5:00:00:impressive:00","synset_id":"amazing.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"amazing%5:00:00:surprising:00","call_ids":["senseval3.d002.s087.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"amazing%5:00:00:surprising:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":636,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001952,"input_uncached_usd":0.001272,"input_cached_usd":0.0,"output_usd":0.00068,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8330938750877976}
{"item_id":"senseval3.d002.s087.t001","gold_sense_keys":["do%2:36:02::"],"candidates":[{"index":1,"sense_key":"do%2:41:01::","synset_id":"make.v.01"},{"index":2,"sense_key":"do%2:36:01::","synset_id":"perform.v.01"},{"index":3,"sense_key":"do%2:41:04::","synset_id":"do.v.03"},{"index":4,"sense_key":"do%2:42:00::","synset_id":"do.v.04"},{"index":5,"sense_key":"do%2:36:02::","synset_id":"cause.v.01"},{"index":6,"sense_key":"do%2:41:02::","synset_id":"practice.v.01"},{"index":7,"sense_key":"do%2:42:01::","synset_id":"suffice.v.01"},{"index":8,"sense_key":"do%2:36:00::","synset_id":"do.v.08"},{"index":9,"sense_key":"do%2:29:09::","synset_id":"act.v.02"},{"index":10,"sense_key":"do%2:42:02::","synset_id":"serve.v.09"},{"index":11,"sense_key":"do%2:41:03::","synset_id":"do.v.11"},{"index":12,"sense_key":"do%2:29:04::","synset_id":"dress.v.16"},{"index":13,"sense_key":"do%2:38:00::","synset_id":"do.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"do%2:36:02::","call_ids":["senseval3.d002.s087.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"do%2:36:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1261,"cached_input_tokens":0,"output_tokens":160,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004122,"input_uncached_usd":0.002522,"input_cached_usd":0.0,"output_usd":0.0016,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9739685830427334}
{"item_id":"senseval3.d002.s087.t002","gold_sense_keys":["one%1:09:00::"],"candidates":[{"index":1,"sense_key":"one%1:23:00::","synset_id":"one.n.01"},{"index":2,"sense_key":"one%1:09:00::","synset_id":"one.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"one%1:09:00::","call_ids":["senseval3.d002.s087.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"one%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":577,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0018839999999999998,"input_uncached_usd":0.0011539999999999999,"input_cached_usd":0.0,"output_usd":0.0007300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9385210409527645}
{"item_id":"senseval3.d002.s087.t004","gold_sense_keys":["one%1:09:00::"],"candidates":[{"index":1,"sense_key":"one%1:23:00::","synset_id":"one.n.01"},{"index":2,"sense_key":"one%1:09:00::","synset_id":"one.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"one%1:09:00::","call_ids":["senseval3.d002.s087.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"one%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":577,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0018739999999999998,"input_uncached_usd":0.0011539999999999999,"input_cached_usd":0.0,"output_usd":0.00072,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.067121500032954}
{"item_id":"senseval3.d002.s088.t000","gold_sense_keys":["look%2:29:00::","look%2:39:01::"],"candidates":[{"index":1,"sense_key":"look%2:39:00::","synset_id":"look.v.01"},{"index":2,"sense_key":"look%2:39:01::","synset_id":"look.v.02"},{"index":3,"sense_key":"look%2:29:00::","synset_id":"look.v.03"},{"index":4,"sense_key":"look%2:39:02::","synset_id":"search.v.02"},{"index":5,"sense_key":"look%2:42:00::","synset_id":"front.v.01"},{"index":6,"sense_key":"look%2:41:13::","synset_id":"attend.v.02"},{"index":7,"sense_key":"look%2:32:00::","synset_id":"look.v.07"},{"index":8,"sense_key":"look%2:31:00::","synset_id":"expect.v.03"},{"index":9,"sense_key":"look%2:42:02::","synset_id":"look.v.09"},{"index":10,"sense_key":"look%2:31:02::","synset_id":"count.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"look%2:39:01::","call_ids":["senseval3.d002.s088.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"look%2:39:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1026,"cached_input_tokens":0,"output_tokens":144,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0034920000000000003,"input_uncached_usd":0.002052,"input_cached_usd":0.0,"output_usd":0.00144,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0027990830130875}
{"item_id":"senseval3.d002.s088.t001","gold_sense_keys":["calm%5:00:00:composed:00"],"candidates":[{"index":1,"sense_key":"calm%5:00:00:composed:00","synset_id":"calm.s.01"},{"index":2,"sense_key":"calm%3:00:00::","synset_id":"calm.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"calm%5:00:00:composed:00","call_ids":["senseval3.d002.s088.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"calm%5:00:00:composed:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":535,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00119,"input_uncached_usd":0.00107,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3172840420156717}
{"item_id":"senseval3.d002.s088.t002","gold_sense_keys":["surreal%5:00:00:unreal:02"],"candidates":[{"index":1,"sense_key":"surreal%5:00:00:unrealistic:00","synset_id":"phantasmagoric.s.01"},{"index":2,"sense_key":"surreal%5:00:00:unreal:02","synset_id":"dreamlike.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"surreal%5:00:00:unreal:02","call_ids":["senseval3.d002.s088.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"surreal%5:00:00:unreal:02","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":616,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001872,"input_uncached_usd":0.001232,"input_cached_usd":0.0,"output_usd":0.00064,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3938390420516953}
{"item_id":"senseval3.d002.s088.t003","gold_sense_keys":["low%3:00:02::"],"candidates":[{"index":1,"sense_key":"low%3:00:02::","synset_id":"low.a.01"},{"index":2,"sense_key":"low%3:00:01::","synset_id":"low.a.02"},{"index":3,"sense_key":"low%5:00:00:soft:04","synset_id":"low.s.01"},{"index":4,"sense_key":"low%5:00:00:unrefined:01","synset_id":"low.s.02"},{"index":5,"sense_key":"low%3:00:03::","synset_id":"low.a.05"},{"index":6,"sense_key":"low%5:00:00:contemptible:00","synset_id":"abject.s.01"},{"index":7,"sense_key":"low%5:00:00:inferior:01","synset_id":"humble.s.01"},{"index":8,"sense_key":"low%5:00:00:insufficient:00","synset_id":"depleted.s.01"},{"index":9,"sense_key":"low%5:00:00:humble:00","synset_id":"broken.s.01"},{"index":10,"sense_key":"low%5:00:00:dejected:00","synset_id":"gloomy.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"low%3:00:02::","call_ids":["senseval3.d002.s088.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"low%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1043,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0026759999999999996,"input_uncached_usd":0.0020859999999999997,"input_cached_usd":0.0,"output_usd":0.00059,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2744959169067442}
{"item_id":"senseval3.d002.s088.t004","gold_sense_keys":["level%1:07:00::"],"candidates":[{"index":1,"sense_key":"level%1:07:00::","synset_id":"degree.n.01"},{"index":2,"sense_key":"level%1:26:01::","synset_id":"grade.n.02"},{"index":3,"sense_key":"level%1:26:00::","synset_id":"degree.n.02"},{"index":4,"sense_key":"level%1:07:01::","synset_id":"level.n.04"},{"index":5,"sense_key":"level%1:06:00::","synset_id":"level.n.05"},{"index":6,"sense_key":"level%1:06:03::","synset_id":"horizontal_surface.n.01"},{"index":7,"sense_key":"level%1:09:00::","synset_id":"level.n.07"},{"index":8,"sense_key":"level%1:06:01::","synset_id":"floor.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"level%1:07:00::","call_ids":["senseval3.d002.s088.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"level%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":933,"cached_input_tokens":0,"output_tokens":47,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002336,"input_uncached_usd":0.0018659999999999998,"input_cached_usd":0.0,"output_usd":0.00047000000000000004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2901562500046566}
{"item_id":"senseval3.d002.s088.t005","gold_sense_keys":["confusion%1:09:00::"],"candidates":[{"index":1,"sense_key":"confusion%1:26:01::","synset_id":"confusion.n.01"},{"index":2,"sense_key":"confusion%1:09:00::","synset_id":"confusion.n.02"},{"index":3,"sense_key":"confusion%1:12:00::","synset_id":"confusion.n.03"},{"index":4,"sense_key":"confusion%1:04:01::","synset_id":"confusion.n.04"},{"index":5,"sense_key":"confusion%1:04:00::","synset_id":"confusion.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"confusion%1:09:00::","call_ids":["senseval3.d002.s088.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"confusion%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":746,"cached_input_tokens":0,"output_tokens":45,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001942,"input_uncached_usd":0.0014919999999999998,"input_cached_usd":0.0,"output_usd":0.00045000000000000004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8153972500003874}
{"item_id":"senseval3.d002.s088.t007","gold_sense_keys":["continue%2:30:10::"],"candidates":[{"index":1,"sense_key":"continue%2:42:01::","synset_id":"continue.v.01"},{"index":2,"sense_key":"continue%2:32:00::","synset_id":"continue.v.02"},{"index":3,"sense_key":"continue%2:42:00::","synset_id":"continue.v.03"},{"index":4,"sense_key":"continue%2:38:00::","synset_id":"proceed.v.02"},{"index":5,"sense_key":"continue%2:41:00::","synset_id":"retain.v.02"},{"index":6,"sense_key":"continue%2:42:12::","synset_id":"continue.v.06"},{"index":7,"sense_key":"continue%2:42:11::","synset_id":"continue.v.07"},{"index":8,"sense_key":"continue%2:42:10::","synset_id":"stay.v.04"},{"index":9,"sense_key":"continue%2:42:06::","synset_id":"cover.v.03"},{"index":10,"sense_key":"continue%2:30:10::","synset_id":"continue.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"continue%2:30:10::","call_ids":["senseval3.d002.s088.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"continue%2:30:10::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1112,"cached_input_tokens":0,"output_tokens":123,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003454,"input_uncached_usd":0.002224,"input_cached_usd":0.0,"output_usd":0.0012300000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6204956249566749}
{"item_id":"senseval3.d002.s091.t001","gold_sense_keys":["seem%2:39:00::"],"candidates":[{"index":1,"sense_key":"seem%2:39:00::","synset_id":"look.v.02"},{"index":2,"sense_key":"seem%2:39:01::","synset_id":"appear.v.04"},{"index":3,"sense_key":"seem%2:42:00::","synset_id":"seem.v.03"},{"index":4,"sense_key":"seem%2:39:02::","synset_id":"seem.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"seem%2:39:00::","call_ids":["senseval3.d002.s091.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"seem%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":721,"cached_input_tokens":0,"output_tokens":175,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003192,"input_uncached_usd":0.001442,"input_cached_usd":0.0,"output_usd":0.00175,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.48807304201182}
{"item_id":"senseval3.d002.s091.t003","gold_sense_keys":["night%1:28:00::"],"candidates":[{"index":1,"sense_key":"night%1:28:00::","synset_id":"night.n.01"},{"index":2,"sense_key":"night%1:28:01::","synset_id":"night.n.02"},{"index":3,"sense_key":"night%1:28:04::","synset_id":"night.n.03"},{"index":4,"sense_key":"night%1:28:02::","synset_id":"night.n.04"},{"index":5,"sense_key":"night%1:26:00::","synset_id":"night.n.05"},{"index":6,"sense_key":"night%1:28:05::","synset_id":"night.n.06"},{"index":7,"sense_key":"night%1:28:03::","synset_id":"night.n.07"},{"index":8,"sense_key":"night%1:18:00::","synset_id":"nox.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"night%1:28:00::","call_ids":["senseval3.d002.s091.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"night%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":820,"cached_input_tokens":0,"output_tokens":151,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00315,"input_uncached_usd":0.00164,"input_cached_usd":0.0,"output_usd":0.00151,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":6.197438000002876}
{"item_id":"senseval3.d002.s091.t004","gold_sense_keys":["do%2:36:01::"],"candidates":[{"index":1,"sense_key":"do%2:41:01::","synset_id":"make.v.01"},{"index":2,"sense_key":"do%2:36:01::","synset_id":"perform.v.01"},{"index":3,"sense_key":"do%2:41:04::","synset_id":"do.v.03"},{"index":4,"sense_key":"do%2:42:00::","synset_id":"do.v.04"},{"index":5,"sense_key":"do%2:36:02::","synset_id":"cause.v.01"},{"index":6,"sense_key":"do%2:41:02::","synset_id":"practice.v.01"},{"index":7,"sense_key":"do%2:42:01::","synset_id":"suffice.v.01"},{"index":8,"sense_key":"do%2:36:00::","synset_id":"do.v.08"},{"index":9,"sense_key":"do%2:29:09::","synset_id":"act.v.02"},{"index":10,"sense_key":"do%2:42:02::","synset_id":"serve.v.09"},{"index":11,"sense_key":"do%2:41:03::","synset_id":"do.v.11"},{"index":12,"sense_key":"do%2:29:04::","synset_id":"dress.v.16"},{"index":13,"sense_key":"do%2:38:00::","synset_id":"do.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"do%2:36:01::","call_ids":["senseval3.d002.s091.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"do%2:36:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1272,"cached_input_tokens":0,"output_tokens":152,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004064,"input_uncached_usd":0.002544,"input_cached_usd":0.0,"output_usd":0.00152,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.443813332938589}
{"item_id":"senseval3.d002.s091.t007","gold_sense_keys":["sound%1:07:00::"],"candidates":[{"index":1,"sense_key":"sound%1:07:00::","synset_id":"sound.n.01"},{"index":2,"sense_key":"sound%1:09:00::","synset_id":"sound.n.02"},{"index":3,"sense_key":"sound%1:19:00::","synset_id":"sound.n.03"},{"index":4,"sense_key":"sound%1:11:00::","synset_id":"sound.n.04"},{"index":5,"sense_key":"sound%1:10:00::","synset_id":"audio.n.01"},{"index":6,"sense_key":"sound%1:10:01::","synset_id":"phone.n.02"},{"index":7,"sense_key":"sound%1:17:01::","synset_id":"strait.n.01"},{"index":8,"sense_key":"sound%1:17:00::","synset_id":"sound.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"sound%1:07:00::","call_ids":["senseval3.d002.s091.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"sound%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":930,"cached_input_tokens":0,"output_tokens":241,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00427,"input_uncached_usd":0.0018599999999999999,"input_cached_usd":0.0,"output_usd":0.0024100000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.729405999998562}
{"item_id":"senseval3.d002.s091.t008","gold_sense_keys":["light%1:06:00::","light%1:19:00::"],"candidates":[{"index":1,"sense_key":"light%1:19:00::","synset_id":"light.n.01"},{"index":2,"sense_key":"light%1:06:00::","synset_id":"light.n.02"},{"index":3,"sense_key":"light%1:09:01::","synset_id":"light.n.03"},{"index":4,"sense_key":"light%1:07:01::","synset_id":"luminosity.n.01"},{"index":5,"sense_key":"light%1:15:00::","synset_id":"light.n.05"},{"index":6,"sense_key":"light%1:26:01::","synset_id":"light.n.06"},{"index":7,"sense_key":"light%1:07:00::","synset_id":"light.n.07"},{"index":8,"sense_key":"light%1:18:00::","synset_id":"light.n.08"},{"index":9,"sense_key":"light%1:26:00::","synset_id":"light.n.09"},{"index":10,"sense_key":"light%1:09:00::","synset_id":"light.n.10"},{"index":11,"sense_key":"light%1:07:02::","synset_id":"sparkle.n.01"},{"index":12,"sense_key":"light%1:09:02::","synset_id":"light.n.12"},{"index":13,"sense_key":"light%1:16:00::","synset_id":"inner_light.n.01"},{"index":14,"sense_key":"light%1:10:00::","synset_id":"light.n.14"},{"index":15,"sense_key":"light%1:06:01::","synset_id":"lighter.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"light%1:06:00::","call_ids":["senseval3.d002.s091.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"light%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1374,"cached_input_tokens":0,"output_tokens":213,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004878,"input_uncached_usd":0.002748,"input_cached_usd":0.0,"output_usd":0.0021300000000000004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.9150102499406785}
{"item_id":"senseval3.d002.s092.t000","gold_sense_keys":["friend%1:18:00::","friend%1:18:02::"],"candidates":[{"index":1,"sense_key":"friend%1:18:00::","synset_id":"friend.n.01"},{"index":2,"sense_key":"friend%1:18:03::","synset_id":"ally.n.02"},{"index":3,"sense_key":"friend%1:18:02::","synset_id":"acquaintance.n.03"},{"index":4,"sense_key":"friend%1:18:04::","synset_id":"supporter.n.01"},{"index":5,"sense_key":"friend%1:18:01::","synset_id":"friend.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"friend%1:18:00::","call_ids":["senseval3.d002.s092.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"friend%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":752,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001624,"input_uncached_usd":0.001504,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.076446459046565}
{"item_id":"senseval3.d002.s092.t001","gold_sense_keys":["underground%5:00:00:subsurface:00"],"candidates":[{"index":1,"sense_key":"underground%5:00:00:subsurface:00","synset_id":"belowground.s.01"},{"index":2,"sense_key":"underground%5:00:00:covert:00","synset_id":"clandestine.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"underground%5:00:00:subsurface:00","call_ids":["senseval3.d002.s092.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"underground%5:00:00:subsurface:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":603,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0013260000000000001,"input_uncached_usd":0.001206,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.164111708989367}
{"item_id":"senseval3.d002.s092.t002","gold_sense_keys":["computer%1:06:00::"],"candidates":[{"index":1,"sense_key":"computer%1:06:00::","synset_id":"computer.n.01"},{"index":2,"sense_key":"computer%1:18:00::","synset_id":"calculator.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"computer%1:06:00::","call_ids":["senseval3.d002.s092.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"computer%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":547,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001214,"input_uncached_usd":0.001094,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5537914580199867}
{"item_id":"senseval3.d002.s092.t003","gold_sense_keys":["center%1:06:00::"],"candidates":[{"index":1,"sense_key":"center%1:15:01::","synset_id":"center.n.01"},{"index":2,"sense_key":"center%1:06:02::","synset_id":"center_field.n.01"},{"index":3,"sense_key":"center%1:06:00::","synset_id":"center.n.03"},{"index":4,"sense_key":"center%1:15:00::","synset_id":"center.n.04"},{"index":5,"sense_key":"center%1:09:00::","synset_id":"kernel.n.03"},{"index":6,"sense_key":"center%1:09:01::","synset_id":"center.n.06"},{"index":7,"sense_key":"center%1:08:00::","synset_id":"center.n.07"},{"index":8,"sense_key":"center%1:14:01::","synset_id":"center.n.08"},{"index":9,"sense_key":"center%1:18:01::","synset_id":"center.n.09"},{"index":10,"sense_key":"center%1:18:00::","synset_id":"center.n.10"},{"index":11,"sense_key":"center%1:15:02::","synset_id":"center.n.11"},{"index":12,"sense_key":"center%1:14:00::","synset_id":"center.n.12"},{"index":13,"sense_key":"center%1:18:02::","synset_id":"center.n.13"},{"index":14,"sense_key":"center%1:13:00::","synset_id":"center.n.14"},{"index":15,"sense_key":"center%1:06:01::","synset_id":"plaza.n.02"},{"index":16,"sense_key":"center%1:04:03::","synset_id":"center.n.16"},{"index":17,"sense_key":"center%1:04:02::","synset_id":"center.n.17"},{"index":18,"sense_key":"center%1:04:01::","synset_id":"center.n.18"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"center%1:06:00::","call_ids":["senseval3.d002.s092.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"center%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1561,"cached_input_tokens":0,"output_tokens":115,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004272,"input_uncached_usd":0.0031219999999999998,"input_cached_usd":0.0,"output_usd":0.0011500000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6020888750208542}
{"item_id":"senseval3.d002.s092.t006","gold_sense_keys":["hit%2:35:05::"],"candidates":[{"index":1,"sense_key":"hit%2:35:03::","synset_id":"hit.v.01"},{"index":2,"sense_key":"hit%2:35:01::","synset_id":"hit.v.02"},{"index":3,"sense_key":"hit%2:35:00::","synset_id":"hit.v.03"},{"index":4,"sense_key":"hit%2:38:00::","synset_id":"reach.v.01"},{"index":5,"sense_key":"hit%2:35:05::","synset_id":"hit.v.05"},{"index":6,"sense_key":"hit%2:33:01::","synset_id":"shoot.v.01"},{"index":7,"sense_key":"hit%2:40:05::","synset_id":"stumble.v.03"},{"index":8,"sense_key":"hit%2:33:02::","synset_id":"score.v.01"},{"index":9,"sense_key":"hit%2:39:00::","synset_id":"hit.v.09"},{"index":10,"sense_key":"hit%2:33:00::","synset_id":"strike.v.04"},{"index":11,"sense_key":"hit%2:41:00::","synset_id":"murder.v.01"},{"index":12,"sense_key":"hit%2:38:02::","synset_id":"hit.v.12"},{"index":13,"sense_key":"hit%2:38:07::","synset_id":"reach.v.02"},{"index":14,"sense_key":"hit%2:35:02::","synset_id":"strike.v.10"},{"index":15,"sense_key":"hit%2:34:00::","synset_id":"hit.v.15"},{"index":16,"sense_key":"hit%2:33:03::","synset_id":"hit.v.16"},{"index":17,"sense_key":"hit%2:32:12::","synset_id":"hit.v.17"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"hit%2:35:05::","call_ids":["senseval3.d002.s092.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"hit%2:35:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1502,"cached_input_tokens":0,"output_tokens":86,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003864,"input_uncached_usd":0.0030039999999999997,"input_cached_usd":0.0,"output_usd":0.0008600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.706150084035471}
{"item_id":"senseval3.d002.s093.t001","gold_sense_keys":["computer%1:06:00::"],"candidates":[{"index":1,"sense_key":"computer%1:06:00::","synset_id":"computer.n.01"},{"index":2,"sense_key":"computer%1:18:00::","synset_id":"calculator.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"computer%1:06:00::","call_ids":["senseval3.d002.s093.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"computer%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":548,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001216,"input_uncached_usd":0.001096,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4366310829063877}
{"item_id":"senseval3.d002.s093.t005","gold_sense_keys":["slide%2:38:01::"],"candidates":[{"index":1,"sense_key":"slide%2:38:01::","synset_id":"skid.v.04"},{"index":2,"sense_key":"slide%2:38:00::","synset_id":"slither.v.01"},{"index":3,"sense_key":"slide%2:38:02::","synset_id":"slide.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"slide%2:38:02::","call_ids":["senseval3.d002.s093.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"slide%2:38:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":613,"cached_input_tokens":0,"output_tokens":144,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002666,"input_uncached_usd":0.0012259999999999999,"input_cached_usd":0.0,"output_usd":0.00144,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9217777089215815}
{"item_id":"senseval3.d002.s093.t006","gold_sense_keys":["floor%1:06:00::"],"candidates":[{"index":1,"sense_key":"floor%1:06:00::","synset_id":"floor.n.01"},{"index":2,"sense_key":"floor%1:06:01::","synset_id":"floor.n.02"},{"index":3,"sense_key":"floor%1:10:00::","synset_id":"floor.n.03"},{"index":4,"sense_key":"floor%1:17:01::","synset_id":"floor.n.04"},{"index":5,"sense_key":"floor%1:17:00::","synset_id":"floor.n.05"},{"index":6,"sense_key":"floor%1:17:02::","synset_id":"floor.n.06"},{"index":7,"sense_key":"floor%1:14:00::","synset_id":"floor.n.07"},{"index":8,"sense_key":"floor%1:07:00::","synset_id":"floor.n.08"},{"index":9,"sense_key":"floor%1:06:03::","synset_id":"floor.n.09"},{"index":10,"sense_key":"floor%1:06:02::","synset_id":"floor.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"floor%1:06:00::","call_ids":["senseval3.d002.s093.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"floor%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":988,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0020959999999999998,"input_uncached_usd":0.001976,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.03904370800592}
{"item_id":"senseval3.d002.s094.t000","gold_sense_keys":["today%1:28:01::"],"candidates":[{"index":1,"sense_key":"today%1:28:00::","synset_id":"today.n.01"},{"index":2,"sense_key":"today%1:28:01::","synset_id":"today.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"today%1:28:01::","call_ids":["senseval3.d002.s094.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"today%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":507,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001134,"input_uncached_usd":0.001014,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.048483916092664}
{"item_id":"senseval3.d002.s094.t002","gold_sense_keys":["people%1:14:00::"],"candidates":[{"index":1,"sense_key":"people%1:14:00::","synset_id":"people.n.01"},{"index":2,"sense_key":"people%1:14:01::","synset_id":"citizenry.n.01"},{"index":3,"sense_key":"people%1:14:02::","synset_id":"people.n.03"},{"index":4,"sense_key":"people%1:14:03::","synset_id":"multitude.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"people%1:14:00::","call_ids":["senseval3.d002.s094.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"people%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":631,"cached_input_tokens":0,"output_tokens":87,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002132,"input_uncached_usd":0.0012619999999999999,"input_cached_usd":0.0,"output_usd":0.0008700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6505294590024278}
{"item_id":"senseval3.d002.s094.t003","gold_sense_keys":["realize%2:31:00::","realize%2:31:01::"],"candidates":[{"index":1,"sense_key":"realize%2:31:01::","synset_id":"recognize.v.02"},{"index":2,"sense_key":"realize%2:31:00::","synset_id":"understand.v.02"},{"index":3,"sense_key":"realize%2:36:00::","synset_id":"realize.v.03"},{"index":4,"sense_key":"realize%2:40:01::","synset_id":"gain.v.08"},{"index":5,"sense_key":"realize%2:40:00::","synset_id":"realize.v.05"},{"index":6,"sense_key":"realize%2:36:07::","synset_id":"realize.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"realize%2:31:01::","call_ids":["senseval3.d002.s094.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"realize%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":795,"cached_input_tokens":0,"output_tokens":171,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0033,"input_uncached_usd":0.0015899999999999998,"input_cached_usd":0.0,"output_usd":0.0017100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.108851124998182}
{"item_id":"senseval3.d002.s094.t004","gold_sense_keys":["hard%3:00:06::"],"candidates":[{"index":1,"sense_key":"hard%3:00:06::","synset_id":"difficult.a.01"},{"index":2,"sense_key":"hard%3:00:02::","synset_id":"hard.a.02"},{"index":3,"sense_key":"hard%3:00:01::","synset_id":"hard.a.03"},{"index":4,"sense_key":"hard%5:00:00:strong:00","synset_id":"hard.s.01"},{"index":5,"sense_key":"hard%5:00:01:effortful:00","synset_id":"arduous.s.01"},{"index":6,"sense_key":"hard%3:00:00::","synset_id":"unvoiced.a.01"},{"index":7,"sense_key":"hard%3:00:05::","synset_id":"hard.a.07"},{"index":8,"sense_key":"hard%3:00:03::","synset_id":"hard.a.08"},{"index":9,"sense_key":"hard%5:00:00:indulgent:00","synset_id":"intemperate.s.01"},{"index":10,"sense_key":"hard%5:00:00:alcoholic:00","synset_id":"hard.s.04"},{"index":11,"sense_key":"hard%5:00:00:bad:00","synset_id":"hard.s.05"},{"index":12,"sense_key":"hard%5:00:00:stale:00","synset_id":"hard.s.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"hard%3:00:06::","call_ids":["senseval3.d002.s094.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"hard%3:00:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1184,"cached_input_tokens":0,"output_tokens":169,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004058,"input_uncached_usd":0.002368,"input_cached_usd":0.0,"output_usd":0.00169,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1218537079403177}
{"item_id":"senseval3.d002.s094.t005","gold_sense_keys":["life%1:09:00::"],"candidates":[{"index":1,"sense_key":"life%1:26:01::","synset_id":"life.n.01"},{"index":2,"sense_key":"life%1:09:00::","synset_id":"life.n.02"},{"index":3,"sense_key":"life%1:26:02::","synset_id":"life.n.03"},{"index":4,"sense_key":"life%1:26:00::","synset_id":"animation.n.01"},{"index":5,"sense_key":"life%1:28:00::","synset_id":"life.n.05"},{"index":6,"sense_key":"life%1:28:01::","synset_id":"life.n.06"},{"index":7,"sense_key":"life%1:28:02::","synset_id":"life.n.07"},{"index":8,"sense_key":"life%1:18:00::","synset_id":"life.n.08"},{"index":9,"sense_key":"life%1:07:01::","synset_id":"liveliness.n.02"},{"index":10,"sense_key":"life%1:03:00::","synset_id":"life.n.10"},{"index":11,"sense_key":"life%1:19:00::","synset_id":"life.n.11"},{"index":12,"sense_key":"life%1:10:00::","synset_id":"biography.n.01"},{"index":13,"sense_key":"life%1:16:00::","synset_id":"life.n.13"},{"index":14,"sense_key":"life%1:28:03::","synset_id":"life_sentence.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"life%1:26:01::","call_ids":["senseval3.d002.s094.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"life%1:26:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1158,"cached_input_tokens":0,"output_tokens":130,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0036160000000000003,"input_uncached_usd":0.002316,"input_cached_usd":0.0,"output_usd":0.0013000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.67387083300855}
{"item_id":"senseval3.d002.s097.t000","gold_sense_keys":["get%2:38:00::"],"candidates":[{"index":1,"sense_key":"get%2:40:00::","synset_id":"get.v.01"},{"index":2,"sense_key":"get%2:30:00::","synset_id":"become.v.01"},{"index":3,"sense_key":"get%2:30:02::","synset_id":"get.v.03"},{"index":4,"sense_key":"get%2:30:03::","synset_id":"receive.v.02"},{"index":5,"sense_key":"get%2:38:00::","synset_id":"arrive.v.01"},{"index":6,"sense_key":"get%2:35:03::","synset_id":"bring.v.04"},{"index":7,"sense_key":"get%2:39:00::","synset_id":"experience.v.03"},{"index":8,"sense_key":"get%2:33:00::","synset_id":"pay_back.v.02"},{"index":9,"sense_key":"get%2:40:01::","synset_id":"have.v.17"},{"index":10,"sense_key":"get%2:32:00::","synset_id":"induce.v.02"},{"index":11,"sense_key":"get%2:35:09::","synset_id":"get.v.11"},{"index":12,"sense_key":"get%2:29:01::","synset_id":"grow.v.08"},{"index":13,"sense_key":"get%2:29:00::","synset_id":"contract.v.04"},{"index":14,"sense_key":"get%2:32:03::","synset_id":"get.v.14"},{"index":15,"sense_key":"get%2:30:01::","synset_id":"make.v.02"},{"index":16,"sense_key":"get%2:32:08::","synset_id":"drive.v.11"},{"index":17,"sense_key":"get%2:31:09::","synset_id":"catch.v.18"},{"index":18,"sense_key":"get%2:35:00::","synset_id":"catch.v.07"},{"index":19,"sense_key":"get%2:35:10::","synset_id":"get.v.19"},{"index":20,"sense_key":"get%2:31:03::","synset_id":"get.v.20"},{"index":21,"sense_key":"get%2:40:14::","synset_id":"get.v.21"},{"index":22,"sense_key":"get%2:40:13::","synset_id":"get.v.22"},{"index":23,"sense_key":"get%2:39:13::","synset_id":"catch.v.21"},{"index":24,"sense_key":"get%2:39:09::","synset_id":"catch.v.22"},{"index":25,"sense_key":"get%2:39:14::","synset_id":"get.v.25"},{"index":26,"sense_key":"get%2:38:15::","synset_id":"scram.v.01"},{"index":27,"sense_key":"get%2:38:12::","synset_id":"get.v.27"},{"index":28,"sense_key":"get%2:37:12::","synset_id":"get.v.28"},{"index":29,"sense_key":"get%2:37:11::","synset_id":"get.v.29"},{"index":30,"sense_key":"get%2:36:14::","synset_id":"catch.v.24"},{"index":31,"sense_key":"get%2:36:00::","synset_id":"draw.v.15"},{"index":32,"sense_key":"get%2:35:12::","synset_id":"get.v.32"},{"index":33,"sense_key":"get%2:31:10::","synset_id":"perplex.v.01"},{"index":34,"sense_key":"get%2:30:12::","synset_id":"get_down.v.07"},{"index":35,"sense_key":"get%2:29:11::","synset_id":"suffer.v.02"},{"index":36,"sense_key":"get%2:29:08::","synset_id":"beget.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"get%2:38:00::","call_ids":["senseval3.d002.s097.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"get%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2729,"cached_input_tokens":0,"output_tokens":47,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.005928,"input_uncached_usd":0.005458,"input_cached_usd":0.0,"output_usd":0.00047000000000000004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2658697080332786}
{"item_id":"senseval3.d002.s097.t001","gold_sense_keys":["home%1:15:04::"],"candidates":[{"index":1,"sense_key":"home%1:15:04::","synset_id":"home.n.01"},{"index":2,"sense_key":"home%1:06:00::","synset_id":"dwelling.n.01"},{"index":3,"sense_key":"home%1:15:02::","synset_id":"home.n.03"},{"index":4,"sense_key":"home%1:06:02::","synset_id":"home_plate.n.01"},{"index":5,"sense_key":"home%1:15:05::","synset_id":"base.n.14"},{"index":6,"sense_key":"home%1:15:03::","synset_id":"home.n.06"},{"index":7,"sense_key":"home%1:26:00::","synset_id":"home.n.07"},{"index":8,"sense_key":"home%1:14:00::","synset_id":"family.n.01"},{"index":9,"sense_key":"home%1:06:01::","synset_id":"home.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"home%1:15:04::","call_ids":["senseval3.d002.s097.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"home%1:15:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":952,"cached_input_tokens":0,"output_tokens":82,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002724,"input_uncached_usd":0.0019039999999999999,"input_cached_usd":0.0,"output_usd":0.0008200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5663621670100838}
{"item_id":"senseval3.d002.s097.t002","gold_sense_keys":["let%2:30:01::","let%2:32:02::","let%2:41:00::"],"candidates":[{"index":1,"sense_key":"let%2:41:00::","synset_id":"let.v.01"},{"index":2,"sense_key":"let%2:32:00::","synset_id":"let.v.02"},{"index":3,"sense_key":"let%2:32:02::","synset_id":"permit.v.01"},{"index":4,"sense_key":"let%2:30:01::","synset_id":"get.v.03"},{"index":5,"sense_key":"let%2:30:00::","synset_id":"let.v.05"},{"index":6,"sense_key":"let%2:40:00::","synset_id":"lease.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"let%2:41:00::","call_ids":["senseval3.d002.s097.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"let%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":754,"cached_input_tokens":0,"output_tokens":169,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0031980000000000003,"input_uncached_usd":0.001508,"input_cached_usd":0.0,"output_usd":0.00169,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2587266250047833}
{"item_id":"senseval3.d002.s097.t003","gold_sense_keys":["dog%1:05:00::"],"candidates":[{"index":1,"sense_key":"dog%1:05:00::","synset_id":"dog.n.01"},{"index":2,"sense_key":"dog%1:18:01::","synset_id":"frump.n.01"},{"index":3,"sense_key":"dog%1:18:00::","synset_id":"dog.n.03"},{"index":4,"sense_key":"dog%1:18:02::","synset_id":"cad.n.01"},{"index":5,"sense_key":"dog%1:13:01::","synset_id":"frank.n.02"},{"index":6,"sense_key":"dog%1:06:00::","synset_id":"pawl.n.01"},{"index":7,"sense_key":"dog%1:06:01::","synset_id":"andiron.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"dog%1:05:00::","call_ids":["senseval3.d002.s097.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"dog%1:05:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":889,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001898,"input_uncached_usd":0.0017779999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9877668749541044}
{"item_id":"senseval3.d002.s097.t004","gold_sense_keys":["house%1:06:00::"],"candidates":[{"index":1,"sense_key":"house%1:06:00::","synset_id":"house.n.01"},{"index":2,"sense_key":"house%1:14:01::","synset_id":"firm.n.01"},{"index":3,"sense_key":"house%1:14:04::","synset_id":"house.n.03"},{"index":4,"sense_key":"house%1:14:03::","synset_id":"house.n.04"},{"index":5,"sense_key":"house%1:14:02::","synset_id":"house.n.05"},{"index":6,"sense_key":"house%1:14:00::","synset_id":"house.n.06"},{"index":7,"sense_key":"house%1:04:00::","synset_id":"house.n.07"},{"index":8,"sense_key":"house%1:15:00::","synset_id":"sign_of_the_zodiac.n.01"},{"index":9,"sense_key":"house%1:14:06::","synset_id":"house.n.09"},{"index":10,"sense_key":"house%1:14:05::","synset_id":"family.n.01"},{"index":11,"sense_key":"house%1:06:02::","synset_id":"theater.n.01"},{"index":12,"sense_key":"house%1:06:01::","synset_id":"house.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"house%1:06:00::","call_ids":["senseval3.d002.s097.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"house%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1034,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002188,"input_uncached_usd":0.002068,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.2822788750054315}
{"item_id":"senseval3.d002.s097.t005","gold_sense_keys":["notice%2:39:00::"],"candidates":[{"index":1,"sense_key":"notice%2:39:04::","synset_id":"detect.v.01"},{"index":2,"sense_key":"notice%2:39:00::","synset_id":"notice.v.02"},{"index":3,"sense_key":"notice%2:32:00::","synset_id":"comment.v.01"},{"index":4,"sense_key":"notice%2:32:01::","synset_id":"notice.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"notice%2:39:00::","call_ids":["senseval3.d002.s097.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"notice%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":681,"cached_input_tokens":0,"output_tokens":106,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002422,"input_uncached_usd":0.001362,"input_cached_usd":0.0,"output_usd":0.0010600000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.6700961670139804}
{"item_id":"senseval3.d002.s097.t006","gold_sense_keys":["sound%1:09:00::"],"candidates":[{"index":1,"sense_key":"sound%1:07:00::","synset_id":"sound.n.01"},{"index":2,"sense_key":"sound%1:09:00::","synset_id":"sound.n.02"},{"index":3,"sense_key":"sound%1:19:00::","synset_id":"sound.n.03"},{"index":4,"sense_key":"sound%1:11:00::","synset_id":"sound.n.04"},{"index":5,"sense_key":"sound%1:10:00::","synset_id":"audio.n.01"},{"index":6,"sense_key":"sound%1:10:01::","synset_id":"phone.n.02"},{"index":7,"sense_key":"sound%1:17:01::","synset_id":"strait.n.01"},{"index":8,"sense_key":"sound%1:17:00::","synset_id":"sound.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"sound%1:09:00::","call_ids":["senseval3.d002.s097.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"sound%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":857,"cached_input_tokens":0,"output_tokens":151,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0032240000000000003,"input_uncached_usd":0.001714,"input_cached_usd":0.0,"output_usd":0.00151,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.419977999990806}
{"item_id":"senseval3.d002.s097.t007","gold_sense_keys":["head%1:08:00::"],"candidates":[{"index":1,"sense_key":"head%1:08:00::","synset_id":"head.n.01"},{"index":2,"sense_key":"head%1:05:00::","synset_id":"head.n.02"},{"index":3,"sense_key":"head%1:09:00::","synset_id":"mind.n.01"},{"index":4,"sense_key":"head%1:18:00::","synset_id":"head.n.04"},{"index":5,"sense_key":"head%1:14:00::","synset_id":"head.n.05"},{"index":6,"sense_key":"head%1:19:00::","synset_id":"head.n.06"},{"index":7,"sense_key":"head%1:15:00::","synset_id":"head.n.07"},{"index":8,"sense_key":"head%1:15:01::","synset_id":"fountainhead.n.02"},{"index":9,"sense_key":"head%1:10:03::","synset_id":"head.n.09"},{"index":10,"sense_key":"head%1:26:00::","synset_id":"head.n.10"},{"index":11,"sense_key":"head%1:23:00::","synset_id":"head.n.11"},{"index":12,"sense_key":"head%1:20:00::","synset_id":"capitulum.n.01"},{"index":13,"sense_key":"head%1:18:01::","synset_id":"principal.n.02"},{"index":14,"sense_key":"head%1:18:03::","synset_id":"head.n.14"},{"index":15,"sense_key":"head%1:18:02::","synset_id":"head.n.15"},{"index":16,"sense_key":"head%1:17:02::","synset_id":"promontory.n.01"},{"index":17,"sense_key":"head%1:17:01::","synset_id":"head.n.17"},{"index":18,"sense_key":"head%1:17:00::","synset_id":"head.n.18"},{"index":19,"sense_key":"head%1:15:02::","synset_id":"forefront.n.01"},{"index":20,"sense_key":"head%1:11:01::","synset_id":"pass.n.09"},{"index":21,"sense_key":"head%1:11:00::","synset_id":"headway.n.02"},{"index":22,"sense_key":"head%1:10:01::","synset_id":"point.n.20"},{"index":23,"sense_key":"head%1:10:02::","synset_id":"question.n.02"},{"index":24,"sense_key":"head%1:10:00::","synset_id":"heading.n.01"},{"index":25,"sense_key":"head%1:08:01::","synset_id":"head.n.25"},{"index":26,"sense_key":"head%1:08:02::","synset_id":"head.n.26"},{"index":27,"sense_key":"head%1:06:05::","synset_id":"read/write_head.n.01"},{"index":28,"sense_key":"head%1:06:04::","synset_id":"head.n.28"},{"index":29,"sense_key":"head%1:06:03::","synset_id":"head.n.29"},{"index":30,"sense_key":"head%1:06:02::","synset_id":"head.n.30"},{"index":31,"sense_key":"head%1:06:00::","synset_id":"head.n.31"},{"index":32,"sense_key":"head%1:06:01::","synset_id":"drumhead.n.01"},{"index":33,"sense_key":"head%1:04:00::","synset_id":"oral_sex.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"head%1:08:00::","call_ids":["senseval3.d002.s097.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"head%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2208,"cached_input_tokens":0,"output_tokens":42,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004836,"input_uncached_usd":0.004416,"input_cached_usd":0.0,"output_usd":0.00042,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8771187079837546}
{"item_id":"senseval3.d002.s097.t009","gold_sense_keys":["walk%2:38:00::"],"candidates":[{"index":1,"sense_key":"walk%2:38:00::","synset_id":"walk.v.01"},{"index":2,"sense_key":"walk%2:38:01::","synset_id":"walk.v.02"},{"index":3,"sense_key":"walk%2:33:00::","synset_id":"walk.v.03"},{"index":4,"sense_key":"walk%2:38:03::","synset_id":"walk.v.04"},{"index":5,"sense_key":"walk%2:33:01::","synset_id":"walk.v.05"},{"index":6,"sense_key":"walk%2:41:00::","synset_id":"walk.v.06"},{"index":7,"sense_key":"walk%2:41:01::","synset_id":"walk.v.07"},{"index":8,"sense_key":"walk%2:38:02::","synset_id":"walk.v.08"},{"index":9,"sense_key":"walk%2:38:05::","synset_id":"walk.v.09"},{"index":10,"sense_key":"walk%2:38:04::","synset_id":"walk.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"walk%2:38:00::","call_ids":["senseval3.d002.s097.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"walk%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":867,"cached_input_tokens":0,"output_tokens":155,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003284,"input_uncached_usd":0.0017339999999999999,"input_cached_usd":0.0,"output_usd":0.0015500000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.93405304197222}
{"item_id":"senseval3.d002.s097.t010","gold_sense_keys":["roof%1:06:00::"],"candidates":[{"index":1,"sense_key":"roof%1:06:00::","synset_id":"roof.n.01"},{"index":2,"sense_key":"roof%1:06:01::","synset_id":"roof.n.02"},{"index":3,"sense_key":"roof%1:17:00::","synset_id":"roof.n.03"},{"index":4,"sense_key":"roof%1:10:00::","synset_id":"ceiling.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"roof%1:06:00::","call_ids":["senseval3.d002.s097.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"roof%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":606,"cached_input_tokens":0,"output_tokens":101,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002222,"input_uncached_usd":0.001212,"input_cached_usd":0.0,"output_usd":0.00101,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5492612919770181}
{"item_id":"senseval3.d002.s098.t000","gold_sense_keys":["notice%2:39:00::"],"candidates":[{"index":1,"sense_key":"notice%2:39:04::","synset_id":"detect.v.01"},{"index":2,"sense_key":"notice%2:39:00::","synset_id":"notice.v.02"},{"index":3,"sense_key":"notice%2:32:00::","synset_id":"comment.v.01"},{"index":4,"sense_key":"notice%2:32:01::","synset_id":"notice.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"notice%2:39:00::","call_ids":["senseval3.d002.s098.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"notice%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":674,"cached_input_tokens":0,"output_tokens":144,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002788,"input_uncached_usd":0.001348,"input_cached_usd":0.0,"output_usd":0.00144,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.12448504101485}
{"item_id":"senseval3.d002.s098.t001","gold_sense_keys":["car%1:06:00::"],"candidates":[{"index":1,"sense_key":"car%1:06:00::","synset_id":"car.n.01"},{"index":2,"sense_key":"car%1:06:01::","synset_id":"car.n.02"},{"index":3,"sense_key":"car%1:06:03::","synset_id":"car.n.03"},{"index":4,"sense_key":"car%1:06:02::","synset_id":"car.n.04"},{"index":5,"sense_key":"car%1:06:04::","synset_id":"cable_car.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"car%1:06:00::","call_ids":["senseval3.d002.s098.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"car%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":711,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001542,"input_uncached_usd":0.0014219999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9535407499643043}
{"item_id":"senseval3.d002.s098.t002","gold_sense_keys":["bounce%2:38:02::"],"candidates":[{"index":1,"sense_key":"bounce%2:38:00::","synset_id":"bounce.v.01"},{"index":2,"sense_key":"bounce%2:35:03::","synset_id":"bounce.v.02"},{"index":3,"sense_key":"bounce%2:38:02::","synset_id":"bounce.v.03"},{"index":4,"sense_key":"bounce%2:40:00::","synset_id":"bounce.v.04"},{"index":5,"sense_key":"bounce%2:38:01::","synset_id":"bounce.v.05"},{"index":6,"sense_key":"bounce%2:40:01::","synset_id":"bounce.v.06"},{"index":7,"sense_key":"bounce%2:35:00::","synset_id":"bounce.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"bounce%2:38:02::","call_ids":["senseval3.d002.s098.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"bounce%2:38:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":733,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001586,"input_uncached_usd":0.0014659999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2804140420630574}
{"item_id":"senseval3.d002.s099.t000","gold_sense_keys":["realize%2:31:00::"],"candidates":[{"index":1,"sense_key":"realize%2:31:01::","synset_id":"recognize.v.02"},{"index":2,"sense_key":"realize%2:31:00::","synset_id":"understand.v.02"},{"index":3,"sense_key":"realize%2:36:00::","synset_id":"realize.v.03"},{"index":4,"sense_key":"realize%2:40:01::","synset_id":"gain.v.08"},{"index":5,"sense_key":"realize%2:40:00::","synset_id":"realize.v.05"},{"index":6,"sense_key":"realize%2:36:07::","synset_id":"realize.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"realize%2:31:00::","call_ids":["senseval3.d002.s099.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"realize%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":827,"cached_input_tokens":0,"output_tokens":125,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002904,"input_uncached_usd":0.0016539999999999999,"input_cached_usd":0.0,"output_usd":0.00125,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2364577499683946}
{"item_id":"senseval3.d002.s099.t001","gold_sense_keys":["happen%2:30:00::"],"candidates":[{"index":1,"sense_key":"happen%2:30:00::","synset_id":"happen.v.01"},{"index":2,"sense_key":"happen%2:30:02::","synset_id":"happen.v.02"},{"index":3,"sense_key":"happen%2:41:00::","synset_id":"happen.v.03"},{"index":4,"sense_key":"happen%2:30:01::","synset_id":"happen.v.04"},{"index":5,"sense_key":"happen%2:40:12::","synset_id":"find.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"happen%2:30:00::","call_ids":["senseval3.d002.s099.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"happen%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":775,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00167,"input_uncached_usd":0.00155,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0623318329453468}
{"item_id":"senseval3.d002.s099.t002","gold_sense_keys":["scream%2:32:01::"],"candidates":[{"index":1,"sense_key":"scream%2:32:01::","synset_id":"shout.v.02"},{"index":2,"sense_key":"scream%2:32:08::","synset_id":"yell.v.02"},{"index":3,"sense_key":"scream%2:39:00::","synset_id":"scream.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"scream%2:32:01::","call_ids":["senseval3.d002.s099.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"scream%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":639,"cached_input_tokens":0,"output_tokens":115,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002428,"input_uncached_usd":0.0012779999999999998,"input_cached_usd":0.0,"output_usd":0.0011500000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.185070624924265}
{"item_id":"senseval3.d002.s099.t003","gold_sense_keys":["house%1:06:00::"],"candidates":[{"index":1,"sense_key":"house%1:06:00::","synset_id":"house.n.01"},{"index":2,"sense_key":"house%1:14:01::","synset_id":"firm.n.01"},{"index":3,"sense_key":"house%1:14:04::","synset_id":"house.n.03"},{"index":4,"sense_key":"house%1:14:03::","synset_id":"house.n.04"},{"index":5,"sense_key":"house%1:14:02::","synset_id":"house.n.05"},{"index":6,"sense_key":"house%1:14:00::","synset_id":"house.n.06"},{"index":7,"sense_key":"house%1:04:00::","synset_id":"house.n.07"},{"index":8,"sense_key":"house%1:15:00::","synset_id":"sign_of_the_zodiac.n.01"},{"index":9,"sense_key":"house%1:14:06::","synset_id":"house.n.09"},{"index":10,"sense_key":"house%1:14:05::","synset_id":"family.n.01"},{"index":11,"sense_key":"house%1:06:02::","synset_id":"theater.n.01"},{"index":12,"sense_key":"house%1:06:01::","synset_id":"house.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"house%1:06:00::","call_ids":["senseval3.d002.s099.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"house%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1077,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002274,"input_uncached_usd":0.002154,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0192449999740347}
{"item_id":"senseval3.d002.s099.t004","gold_sense_keys":["dog%1:05:00::"],"candidates":[{"index":1,"sense_key":"dog%1:05:00::","synset_id":"dog.n.01"},{"index":2,"sense_key":"dog%1:18:01::","synset_id":"frump.n.01"},{"index":3,"sense_key":"dog%1:18:00::","synset_id":"dog.n.03"},{"index":4,"sense_key":"dog%1:18:02::","synset_id":"cad.n.01"},{"index":5,"sense_key":"dog%1:13:01::","synset_id":"frank.n.02"},{"index":6,"sense_key":"dog%1:06:00::","synset_id":"pawl.n.01"},{"index":7,"sense_key":"dog%1:06:01::","synset_id":"andiron.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"dog%1:05:00::","call_ids":["senseval3.d002.s099.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"dog%1:05:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":932,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001984,"input_uncached_usd":0.001864,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2921881249640137}
{"item_id":"senseval3.d002.s100.t001","gold_sense_keys":["door%1:06:00::"],"candidates":[{"index":1,"sense_key":"door%1:06:00::","synset_id":"door.n.01"},{"index":2,"sense_key":"door%1:06:01::","synset_id":"doorway.n.01"},{"index":3,"sense_key":"door%1:07:00::","synset_id":"door.n.03"},{"index":4,"sense_key":"door%1:06:04::","synset_id":"door.n.04"},{"index":5,"sense_key":"door%1:06:03::","synset_id":"door.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"door%1:06:00::","call_ids":["senseval3.d002.s100.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"door%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":778,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001676,"input_uncached_usd":0.0015559999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0719646660145372}
{"item_id":"senseval3.d002.s100.t007","gold_sense_keys":["foot%1:23:00::"],"candidates":[{"index":1,"sense_key":"foot%1:08:01::","synset_id":"foot.n.01"},{"index":2,"sense_key":"foot%1:23:00::","synset_id":"foot.n.02"},{"index":3,"sense_key":"foot%1:15:00::","synset_id":"foot.n.03"},{"index":4,"sense_key":"foot%1:05:01::","synset_id":"animal_foot.n.01"},{"index":5,"sense_key":"foot%1:06:00::","synset_id":"foundation.n.03"},{"index":6,"sense_key":"foot%1:05:00::","synset_id":"foot.n.06"},{"index":7,"sense_key":"foot%1:04:00::","synset_id":"foot.n.07"},{"index":8,"sense_key":"foot%1:18:00::","synset_id":"foot.n.08"},{"index":9,"sense_key":"foot%1:14:00::","synset_id":"infantry.n.01"},{"index":10,"sense_key":"foot%1:10:00::","synset_id":"metrical_foot.n.01"},{"index":11,"sense_key":"foot%1:06:01::","synset_id":"foot.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"foot%1:23:00::","call_ids":["senseval3.d002.s100.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"foot%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1125,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00278,"input_uncached_usd":0.00225,"input_cached_usd":0.0,"output_usd":0.0005300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8941420409828424}
{"item_id":"senseval3.d002.s100.t008","gold_sense_keys":["dog%1:05:00::"],"candidates":[{"index":1,"sense_key":"dog%1:05:00::","synset_id":"dog.n.01"},{"index":2,"sense_key":"dog%1:18:01::","synset_id":"frump.n.01"},{"index":3,"sense_key":"dog%1:18:00::","synset_id":"dog.n.03"},{"index":4,"sense_key":"dog%1:18:02::","synset_id":"cad.n.01"},{"index":5,"sense_key":"dog%1:13:01::","synset_id":"frank.n.02"},{"index":6,"sense_key":"dog%1:06:00::","synset_id":"pawl.n.01"},{"index":7,"sense_key":"dog%1:06:01::","synset_id":"andiron.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"dog%1:05:00::","call_ids":["senseval3.d002.s100.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"dog%1:05:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":960,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0020399999999999997,"input_uncached_usd":0.0019199999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.448969625052996}
{"item_id":"senseval3.d002.s100.t009","gold_sense_keys":["come%2:38:00::","come%2:38:04::"],"candidates":[{"index":1,"sense_key":"come%2:38:00::","synset_id":"come.v.01"},{"index":2,"sense_key":"come%2:38:04::","synset_id":"arrive.v.01"},{"index":3,"sense_key":"come%2:30:01::","synset_id":"come.v.03"},{"index":4,"sense_key":"come%2:30:03::","synset_id":"come.v.04"},{"index":5,"sense_key":"come%2:42:04::","synset_id":"come.v.05"},{"index":6,"sense_key":"come%2:30:02::","synset_id":"come.v.06"},{"index":7,"sense_key":"come%2:42:13::","synset_id":"issue_forth.v.01"},{"index":8,"sense_key":"come%2:42:09::","synset_id":"hail.v.02"},{"index":9,"sense_key":"come%2:42:03::","synset_id":"come.v.09"},{"index":10,"sense_key":"come%2:42:05::","synset_id":"come.v.10"},{"index":11,"sense_key":"come%2:38:03::","synset_id":"come.v.11"},{"index":12,"sense_key":"come%2:42:00::","synset_id":"fall.v.04"},{"index":13,"sense_key":"come%2:30:13::","synset_id":"come.v.13"},{"index":14,"sense_key":"come%2:42:12::","synset_id":"total.v.01"},{"index":15,"sense_key":"come%2:42:11::","synset_id":"come.v.15"},{"index":16,"sense_key":"come%2:38:02::","synset_id":"come.v.16"},{"index":17,"sense_key":"come%2:31:14::","synset_id":"occur.v.02"},{"index":18,"sense_key":"come%2:42:15::","synset_id":"derive.v.05"},{"index":19,"sense_key":"come%2:42:02::","synset_id":"do.v.04"},{"index":20,"sense_key":"come%2:39:00::","synset_id":"come.v.20"},{"index":21,"sense_key":"come%2:31:13::","synset_id":"come.v.21"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"come%2:38:00::","call_ids":["senseval3.d002.s100.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"come%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1632,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003794,"input_uncached_usd":0.003264,"input_cached_usd":0.0,"output_usd":0.0005300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.457966750022024}
{"item_id":"senseval3.d002.s100.t011","gold_sense_keys":["dog%1:05:00::"],"candidates":[{"index":1,"sense_key":"dog%1:05:00::","synset_id":"dog.n.01"},{"index":2,"sense_key":"dog%1:18:01::","synset_id":"frump.n.01"},{"index":3,"sense_key":"dog%1:18:00::","synset_id":"dog.n.03"},{"index":4,"sense_key":"dog%1:18:02::","synset_id":"cad.n.01"},{"index":5,"sense_key":"dog%1:13:01::","synset_id":"frank.n.02"},{"index":6,"sense_key":"dog%1:06:00::","synset_id":"pawl.n.01"},{"index":7,"sense_key":"dog%1:06:01::","synset_id":"andiron.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"dog%1:05:00::","call_ids":["senseval3.d002.s100.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"dog%1:05:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":959,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002038,"input_uncached_usd":0.001918,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.870079500018619}
{"item_id":"senseval3.d002.s100.t012","gold_sense_keys":["stand%2:35:00::","stand%2:42:03::"],"candidates":[{"index":1,"sense_key":"stand%2:35:00::","synset_id":"stand.v.01"},{"index":2,"sense_key":"stand%2:42:04::","synset_id":"stand.v.02"},{"index":3,"sense_key":"stand%2:42:03::","synset_id":"stand.v.03"},{"index":4,"sense_key":"stand%2:42:00::","synset_id":"stand.v.04"},{"index":5,"sense_key":"stand%2:31:00::","synset_id":"digest.v.03"},{"index":6,"sense_key":"stand%2:31:01::","synset_id":"stand.v.06"},{"index":7,"sense_key":"stand%2:42:06::","synset_id":"stand.v.07"},{"index":8,"sense_key":"stand%2:42:02::","synset_id":"stand.v.08"},{"index":9,"sense_key":"stand%2:42:01::","synset_id":"stand.v.09"},{"index":10,"sense_key":"stand%2:35:01::","synset_id":"stand.v.10"},{"index":11,"sense_key":"stand%2:33:00::","synset_id":"resist.v.04"},{"index":12,"sense_key":"stand%2:35:10::","synset_id":"stand.v.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"stand%2:35:00::","call_ids":["senseval3.d002.s100.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"stand%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1112,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0023439999999999997,"input_uncached_usd":0.002224,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.150717332959175}
{"item_id":"senseval3.d002.s100.t014","gold_sense_keys":["watch%2:39:00::","watch%2:39:02::","watch%2:41:00::"],"candidates":[{"index":1,"sense_key":"watch%2:39:00::","synset_id":"watch.v.01"},{"index":2,"sense_key":"watch%2:41:00::","synset_id":"watch.v.02"},{"index":3,"sense_key":"watch%2:39:05::","synset_id":"watch.v.03"},{"index":4,"sense_key":"watch%2:39:02::","synset_id":"watch.v.04"},{"index":5,"sense_key":"watch%2:39:01::","synset_id":"watch.v.05"},{"index":6,"sense_key":"watch%2:39:03::","synset_id":"watch.v.06"},{"index":7,"sense_key":"watch%2:32:09::","synset_id":"determine.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"watch%2:39:02::","call_ids":["senseval3.d002.s100.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"watch%2:39:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":896,"cached_input_tokens":0,"output_tokens":125,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003042,"input_uncached_usd":0.001792,"input_cached_usd":0.0,"output_usd":0.00125,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7787266250234097}
{"item_id":"senseval3.d002.s100.t015","gold_sense_keys":["outside%3:00:04::"],"candidates":[{"index":1,"sense_key":"outside%3:00:00::","synset_id":"outside.a.01"},{"index":2,"sense_key":"outside%5:00:00:extrinsic:00","synset_id":"external.s.01"},{"index":3,"sense_key":"outside%5:00:00:external:00","synset_id":"outside.s.02"},{"index":4,"sense_key":"outside%3:00:04::","synset_id":"outdoor.a.01"},{"index":5,"sense_key":"outside%5:00:00:extramural:00","synset_id":"outside.s.03"},{"index":6,"sense_key":"outside%5:00:00:exterior:00","synset_id":"outside.s.04"},{"index":7,"sense_key":"outside%5:00:00:foreign:02","synset_id":"external.s.02"},{"index":8,"sense_key":"outside%5:00:00:unlikely:00","synset_id":"outside.s.06"},{"index":9,"sense_key":"outside%5:00:00:outer:00","synset_id":"outside.s.07"},{"index":10,"sense_key":"outside%5:00:00:inaccurate:00","synset_id":"away.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"outside%3:00:04::","call_ids":["senseval3.d002.s100.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"outside%3:00:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1061,"cached_input_tokens":0,"output_tokens":94,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0030619999999999996,"input_uncached_usd":0.0021219999999999998,"input_cached_usd":0.0,"output_usd":0.0009400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.748997832997702}
{"item_id":"senseval3.d002.s100.t016","gold_sense_keys":["dance%2:38:02::"],"candidates":[{"index":1,"sense_key":"dance%2:38:01::","synset_id":"dance.v.01"},{"index":2,"sense_key":"dance%2:36:00::","synset_id":"dance.v.02"},{"index":3,"sense_key":"dance%2:38:02::","synset_id":"dance.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"dance%2:38:02::","call_ids":["senseval3.d002.s100.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"dance%2:38:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":653,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0020759999999999997,"input_uncached_usd":0.0013059999999999999,"input_cached_usd":0.0,"output_usd":0.0007700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7528496249578893}
{"item_id":"senseval3.d002.s101.t000","gold_sense_keys":["realize%2:31:00::"],"candidates":[{"index":1,"sense_key":"realize%2:31:01::","synset_id":"recognize.v.02"},{"index":2,"sense_key":"realize%2:31:00::","synset_id":"understand.v.02"},{"index":3,"sense_key":"realize%2:36:00::","synset_id":"realize.v.03"},{"index":4,"sense_key":"realize%2:40:01::","synset_id":"gain.v.08"},{"index":5,"sense_key":"realize%2:40:00::","synset_id":"realize.v.05"},{"index":6,"sense_key":"realize%2:36:07::","synset_id":"realize.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"realize%2:31:00::","call_ids":["senseval3.d002.s101.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"realize%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":860,"cached_input_tokens":0,"output_tokens":115,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00287,"input_uncached_usd":0.00172,"input_cached_usd":0.0,"output_usd":0.0011500000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.796256916015409}
{"item_id":"senseval3.d002.s101.t001","gold_sense_keys":["go%2:38:00::"],"candidates":[{"index":1,"sense_key":"go%2:38:00::","synset_id":"travel.v.01"},{"index":2,"sense_key":"go%2:41:00::","synset_id":"go.v.02"},{"index":3,"sense_key":"go%2:38:01::","synset_id":"go.v.03"},{"index":4,"sense_key":"go%2:30:04::","synset_id":"become.v.01"},{"index":5,"sense_key":"go%2:42:06::","synset_id":"go.v.05"},{"index":6,"sense_key":"go%2:42:02::","synset_id":"run.v.05"},{"index":7,"sense_key":"go%2:42:00::","synset_id":"run.v.03"},{"index":8,"sense_key":"go%2:30:01::","synset_id":"proceed.v.04"},{"index":9,"sense_key":"go%2:30:08::","synset_id":"go.v.09"},{"index":10,"sense_key":"go%2:42:07::","synset_id":"go.v.10"},{"index":11,"sense_key":"go%2:39:00::","synset_id":"sound.v.02"},{"index":12,"sense_key":"go%2:35:00::","synset_id":"function.v.01"},{"index":13,"sense_key":"go%2:34:00::","synset_id":"run_low.v.01"},{"index":14,"sense_key":"go%2:30:02::","synset_id":"move.v.13"},{"index":15,"sense_key":"go%2:42:14::","synset_id":"survive.v.01"},{"index":16,"sense_key":"go%2:42:03::","synset_id":"go.v.16"},{"index":17,"sense_key":"go%2:30:00::","synset_id":"die.v.01"},{"index":18,"sense_key":"go%2:42:04::","synset_id":"belong.v.03"},{"index":19,"sense_key":"go%2:42:10::","synset_id":"go.v.19"},{"index":20,"sense_key":"go%2:38:09::","synset_id":"start.v.09"},{"index":21,"sense_key":"go%2:33:00::","synset_id":"move.v.15"},{"index":22,"sense_key":"go%2:42:13::","synset_id":"go.v.22"},{"index":23,"sense_key":"go%2:42:12::","synset_id":"go.v.23"},{"index":24,"sense_key":"go%2:42:11::","synset_id":"blend.v.02"},{"index":25,"sense_key":"go%2:42:08::","synset_id":"go.v.25"},{"index":26,"sense_key":"go%2:42:05::","synset_id":"fit.v.02"},{"index":27,"sense_key":"go%2:35:13::","synset_id":"rifle.v.02"},{"index":28,"sense_key":"go%2:34:09::","synset_id":"go.v.28"},{"index":29,"sense_key":"go%2:31:12::","synset_id":"plump.v.04"},{"index":30,"sense_key":"go%2:30:03::","synset_id":"fail.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"go%2:38:00::","call_ids":["senseval3.d002.s101.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"go%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2365,"cached_input_tokens":0,"output_tokens":122,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00595,"input_uncached_usd":0.00473,"input_cached_usd":0.0,"output_usd":0.0012200000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7771059999940917}
{"item_id":"senseval3.d002.s101.t002","gold_sense_keys":["stand%2:35:00::","stand%2:42:03::"],"candidates":[{"index":1,"sense_key":"stand%2:35:00::","synset_id":"stand.v.01"},{"index":2,"sense_key":"stand%2:42:04::","synset_id":"stand.v.02"},{"index":3,"sense_key":"stand%2:42:03::","synset_id":"stand.v.03"},{"index":4,"sense_key":"stand%2:42:00::","synset_id":"stand.v.04"},{"index":5,"sense_key":"stand%2:31:00::","synset_id":"digest.v.03"},{"index":6,"sense_key":"stand%2:31:01::","synset_id":"stand.v.06"},{"index":7,"sense_key":"stand%2:42:06::","synset_id":"stand.v.07"},{"index":8,"sense_key":"stand%2:42:02::","synset_id":"stand.v.08"},{"index":9,"sense_key":"stand%2:42:01::","synset_id":"stand.v.09"},{"index":10,"sense_key":"stand%2:35:01::","synset_id":"stand.v.10"},{"index":11,"sense_key":"stand%2:33:00::","synset_id":"resist.v.04"},{"index":12,"sense_key":"stand%2:35:10::","synset_id":"stand.v.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"stand%2:35:00::","call_ids":["senseval3.d002.s101.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"stand%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1117,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0023539999999999998,"input_uncached_usd":0.002234,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0053443750366569}
{"item_id":"senseval3.d002.s101.t004","gold_sense_keys":["house%1:06:00::"],"candidates":[{"index":1,"sense_key":"house%1:06:00::","synset_id":"house.n.01"},{"index":2,"sense_key":"house%1:14:01::","synset_id":"firm.n.01"},{"index":3,"sense_key":"house%1:14:04::","synset_id":"house.n.03"},{"index":4,"sense_key":"house%1:14:03::","synset_id":"house.n.04"},{"index":5,"sense_key":"house%1:14:02::","synset_id":"house.n.05"},{"index":6,"sense_key":"house%1:14:00::","synset_id":"house.n.06"},{"index":7,"sense_key":"house%1:04:00::","synset_id":"house.n.07"},{"index":8,"sense_key":"house%1:15:00::","synset_id":"sign_of_the_zodiac.n.01"},{"index":9,"sense_key":"house%1:14:06::","synset_id":"house.n.09"},{"index":10,"sense_key":"house%1:14:05::","synset_id":"family.n.01"},{"index":11,"sense_key":"house%1:06:02::","synset_id":"theater.n.01"},{"index":12,"sense_key":"house%1:06:01::","synset_id":"house.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"house%1:06:00::","call_ids":["senseval3.d002.s101.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"house%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1110,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0023399999999999996,"input_uncached_usd":0.0022199999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.077968499972485}
{"item_id":"senseval3.d002.s101.t005","gold_sense_keys":["wait%2:42:00::"],"candidates":[{"index":1,"sense_key":"wait%2:42:00::","synset_id":"wait.v.01"},{"index":2,"sense_key":"wait%2:42:01::","synset_id":"wait.v.02"},{"index":3,"sense_key":"wait%2:31:00::","synset_id":"expect.v.03"},{"index":4,"sense_key":"wait%2:41:00::","synset_id":"wait.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"wait%2:42:00::","call_ids":["senseval3.d002.s101.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"wait%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":726,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001572,"input_uncached_usd":0.001452,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2247407080139965}
{"item_id":"senseval3.d002.s101.t006","gold_sense_keys":["pray%2:32:00::"],"candidates":[{"index":1,"sense_key":"pray%2:32:00::","synset_id":"pray.v.01"},{"index":2,"sense_key":"pray%2:32:01::","synset_id":"beg.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"pray%2:32:00::","call_ids":["senseval3.d002.s101.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"pray%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":578,"cached_input_tokens":0,"output_tokens":91,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002066,"input_uncached_usd":0.001156,"input_cached_usd":0.0,"output_usd":0.0009100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.048803999903612}
{"item_id":"senseval3.d002.s101.t007","gold_sense_keys":["come%2:38:04::"],"candidates":[{"index":1,"sense_key":"come%2:38:00::","synset_id":"come.v.01"},{"index":2,"sense_key":"come%2:38:04::","synset_id":"arrive.v.01"},{"index":3,"sense_key":"come%2:30:01::","synset_id":"come.v.03"},{"index":4,"sense_key":"come%2:30:03::","synset_id":"come.v.04"},{"index":5,"sense_key":"come%2:42:04::","synset_id":"come.v.05"},{"index":6,"sense_key":"come%2:30:02::","synset_id":"come.v.06"},{"index":7,"sense_key":"come%2:42:13::","synset_id":"issue_forth.v.01"},{"index":8,"sense_key":"come%2:42:09::","synset_id":"hail.v.02"},{"index":9,"sense_key":"come%2:42:03::","synset_id":"come.v.09"},{"index":10,"sense_key":"come%2:42:05::","synset_id":"come.v.10"},{"index":11,"sense_key":"come%2:38:03::","synset_id":"come.v.11"},{"index":12,"sense_key":"come%2:42:00::","synset_id":"fall.v.04"},{"index":13,"sense_key":"come%2:30:13::","synset_id":"come.v.13"},{"index":14,"sense_key":"come%2:42:12::","synset_id":"total.v.01"},{"index":15,"sense_key":"come%2:42:11::","synset_id":"come.v.15"},{"index":16,"sense_key":"come%2:38:02::","synset_id":"come.v.16"},{"index":17,"sense_key":"come%2:31:14::","synset_id":"occur.v.02"},{"index":18,"sense_key":"come%2:42:15::","synset_id":"derive.v.05"},{"index":19,"sense_key":"come%2:42:02::","synset_id":"do.v.04"},{"index":20,"sense_key":"come%2:39:00::","synset_id":"come.v.20"},{"index":21,"sense_key":"come%2:31:13::","synset_id":"come.v.21"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"come%2:38:04::","call_ids":["senseval3.d002.s101.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"come%2:38:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1637,"cached_input_tokens":0,"output_tokens":90,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004174,"input_uncached_usd":0.003274,"input_cached_usd":0.0,"output_usd":0.0009000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8317002080148086}
{"item_id":"senseval3.d002.s101.t008","gold_sense_keys":["home%1:15:04::"],"candidates":[{"index":1,"sense_key":"home%1:15:04::","synset_id":"home.n.01"},{"index":2,"sense_key":"home%1:06:00::","synset_id":"dwelling.n.01"},{"index":3,"sense_key":"home%1:15:02::","synset_id":"home.n.03"},{"index":4,"sense_key":"home%1:06:02::","synset_id":"home_plate.n.01"},{"index":5,"sense_key":"home%1:15:05::","synset_id":"base.n.14"},{"index":6,"sense_key":"home%1:15:03::","synset_id":"home.n.06"},{"index":7,"sense_key":"home%1:26:00::","synset_id":"home.n.07"},{"index":8,"sense_key":"home%1:14:00::","synset_id":"family.n.01"},{"index":9,"sense_key":"home%1:06:01::","synset_id":"home.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"home%1:15:04::","call_ids":["senseval3.d002.s101.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"home%1:15:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1028,"cached_input_tokens":0,"output_tokens":85,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002906,"input_uncached_usd":0.002056,"input_cached_usd":0.0,"output_usd":0.0008500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9230612079845741}
{"item_id":"senseval3.d002.s101.t010","gold_sense_keys":["get%2:38:00::"],"candidates":[{"index":1,"sense_key":"get%2:40:00::","synset_id":"get.v.01"},{"index":2,"sense_key":"get%2:30:00::","synset_id":"become.v.01"},{"index":3,"sense_key":"get%2:30:02::","synset_id":"get.v.03"},{"index":4,"sense_key":"get%2:30:03::","synset_id":"receive.v.02"},{"index":5,"sense_key":"get%2:38:00::","synset_id":"arrive.v.01"},{"index":6,"sense_key":"get%2:35:03::","synset_id":"bring.v.04"},{"index":7,"sense_key":"get%2:39:00::","synset_id":"experience.v.03"},{"index":8,"sense_key":"get%2:33:00::","synset_id":"pay_back.v.02"},{"index":9,"sense_key":"get%2:40:01::","synset_id":"have.v.17"},{"index":10,"sense_key":"get%2:32:00::","synset_id":"induce.v.02"},{"index":11,"sense_key":"get%2:35:09::","synset_id":"get.v.11"},{"index":12,"sense_key":"get%2:29:01::","synset_id":"grow.v.08"},{"index":13,"sense_key":"get%2:29:00::","synset_id":"contract.v.04"},{"index":14,"sense_key":"get%2:32:03::","synset_id":"get.v.14"},{"index":15,"sense_key":"get%2:30:01::","synset_id":"make.v.02"},{"index":16,"sense_key":"get%2:32:08::","synset_id":"drive.v.11"},{"index":17,"sense_key":"get%2:31:09::","synset_id":"catch.v.18"},{"index":18,"sense_key":"get%2:35:00::","synset_id":"catch.v.07"},{"index":19,"sense_key":"get%2:35:10::","synset_id":"get.v.19"},{"index":20,"sense_key":"get%2:31:03::","synset_id":"get.v.20"},{"index":21,"sense_key":"get%2:40:14::","synset_id":"get.v.21"},{"index":22,"sense_key":"get%2:40:13::","synset_id":"get.v.22"},{"index":23,"sense_key":"get%2:39:13::","synset_id":"catch.v.21"},{"index":24,"sense_key":"get%2:39:09::","synset_id":"catch.v.22"},{"index":25,"sense_key":"get%2:39:14::","synset_id":"get.v.25"},{"index":26,"sense_key":"get%2:38:15::","synset_id":"scram.v.01"},{"index":27,"sense_key":"get%2:38:12::","synset_id":"get.v.27"},{"index":28,"sense_key":"get%2:37:12::","synset_id":"get.v.28"},{"index":29,"sense_key":"get%2:37:11::","synset_id":"get.v.29"},{"index":30,"sense_key":"get%2:36:14::","synset_id":"catch.v.24"},{"index":31,"sense_key":"get%2:36:00::","synset_id":"draw.v.15"},{"index":32,"sense_key":"get%2:35:12::","synset_id":"get.v.32"},{"index":33,"sense_key":"get%2:31:10::","synset_id":"perplex.v.01"},{"index":34,"sense_key":"get%2:30:12::","synset_id":"get_down.v.07"},{"index":35,"sense_key":"get%2:29:11::","synset_id":"suffer.v.02"},{"index":36,"sense_key":"get%2:29:08::","synset_id":"beget.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"get%2:38:00::","call_ids":["senseval3.d002.s101.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"get%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2805,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00573,"input_uncached_usd":0.0056099999999999995,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.007693958003074}
{"item_id":"senseval3.d002.s102.t000","gold_sense_keys":["life%1:26:02::","life%1:28:01::"],"candidates":[{"index":1,"sense_key":"life%1:26:01::","synset_id":"life.n.01"},{"index":2,"sense_key":"life%1:09:00::","synset_id":"life.n.02"},{"index":3,"sense_key":"life%1:26:02::","synset_id":"life.n.03"},{"index":4,"sense_key":"life%1:26:00::","synset_id":"animation.n.01"},{"index":5,"sense_key":"life%1:28:00::","synset_id":"life.n.05"},{"index":6,"sense_key":"life%1:28:01::","synset_id":"life.n.06"},{"index":7,"sense_key":"life%1:28:02::","synset_id":"life.n.07"},{"index":8,"sense_key":"life%1:18:00::","synset_id":"life.n.08"},{"index":9,"sense_key":"life%1:07:01::","synset_id":"liveliness.n.02"},{"index":10,"sense_key":"life%1:03:00::","synset_id":"life.n.10"},{"index":11,"sense_key":"life%1:19:00::","synset_id":"life.n.11"},{"index":12,"sense_key":"life%1:10:00::","synset_id":"biography.n.01"},{"index":13,"sense_key":"life%1:16:00::","synset_id":"life.n.13"},{"index":14,"sense_key":"life%1:28:03::","synset_id":"life_sentence.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"life%1:28:01::","call_ids":["senseval3.d002.s102.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"life%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1247,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0031739999999999997,"input_uncached_usd":0.0024939999999999997,"input_cached_usd":0.0,"output_usd":0.00068,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.210905250045471}
{"item_id":"senseval3.d002.s103.t000","gold_sense_keys":["see%2:39:00::","see%2:39:03::"],"candidates":[{"index":1,"sense_key":"see%2:39:00::","synset_id":"see.v.01"},{"index":2,"sense_key":"see%2:31:01::","synset_id":"understand.v.02"},{"index":3,"sense_key":"see%2:39:02::","synset_id":"witness.v.02"},{"index":4,"sense_key":"see%2:36:00::","synset_id":"visualize.v.01"},{"index":5,"sense_key":"see%2:31:00::","synset_id":"see.v.05"},{"index":6,"sense_key":"see%2:31:02::","synset_id":"learn.v.02"},{"index":7,"sense_key":"see%2:39:03::","synset_id":"watch.v.03"},{"index":8,"sense_key":"see%2:38:00::","synset_id":"meet.v.01"},{"index":9,"sense_key":"see%2:32:00::","synset_id":"determine.v.08"},{"index":10,"sense_key":"see%2:31:03::","synset_id":"see.v.10"},{"index":11,"sense_key":"see%2:41:11::","synset_id":"see.v.11"},{"index":12,"sense_key":"see%2:41:12::","synset_id":"see.v.12"},{"index":13,"sense_key":"see%2:41:10::","synset_id":"visit.v.01"},{"index":14,"sense_key":"see%2:41:14::","synset_id":"attend.v.02"},{"index":15,"sense_key":"see%2:35:09::","synset_id":"see.v.15"},{"index":16,"sense_key":"see%2:41:00::","synset_id":"go_steady.v.01"},{"index":17,"sense_key":"see%2:39:05::","synset_id":"see.v.17"},{"index":18,"sense_key":"see%2:31:08::","synset_id":"see.v.18"},{"index":19,"sense_key":"see%2:39:11::","synset_id":"see.v.19"},{"index":20,"sense_key":"see%2:39:13::","synset_id":"examine.v.02"},{"index":21,"sense_key":"see%2:39:12::","synset_id":"experience.v.01"},{"index":22,"sense_key":"see%2:38:03::","synset_id":"see.v.22"},{"index":23,"sense_key":"see%2:33:00::","synset_id":"see.v.23"},{"index":24,"sense_key":"see%2:31:13::","synset_id":"interpret.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"see%2:39:00::","call_ids":["senseval3.d002.s103.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"see%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2009,"cached_input_tokens":0,"output_tokens":144,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.005458,"input_uncached_usd":0.004018,"input_cached_usd":0.0,"output_usd":0.00144,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.901441708090715}
{"item_id":"senseval3.d002.s103.t001","gold_sense_keys":["picture%1:06:00::","picture%1:06:02::"],"candidates":[{"index":1,"sense_key":"picture%1:06:00::","synset_id":"picture.n.01"},{"index":2,"sense_key":"picture%1:06:01::","synset_id":"painting.n.01"},{"index":3,"sense_key":"picture%1:09:00::","synset_id":"mental_picture.n.01"},{"index":4,"sense_key":"picture%1:26:00::","synset_id":"picture.n.04"},{"index":5,"sense_key":"picture%1:10:00::","synset_id":"picture.n.05"},{"index":6,"sense_key":"picture%1:10:02::","synset_id":"movie.n.01"},{"index":7,"sense_key":"picture%1:10:01::","synset_id":"video.n.01"},{"index":8,"sense_key":"picture%1:10:03::","synset_id":"word_picture.n.01"},{"index":9,"sense_key":"picture%1:09:01::","synset_id":"picture.n.09"},{"index":10,"sense_key":"picture%1:06:02::","synset_id":"photograph.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"picture%1:06:00::","call_ids":["senseval3.d002.s103.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"picture%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1248,"cached_input_tokens":0,"output_tokens":212,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004616,"input_uncached_usd":0.002496,"input_cached_usd":0.0,"output_usd":0.0021200000000000004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.9277034170227125}
{"item_id":"senseval3.d002.s103.t002","gold_sense_keys":["begin%2:30:00::"],"candidates":[{"index":1,"sense_key":"begin%2:30:00::","synset_id":"get_down.v.07"},{"index":2,"sense_key":"begin%2:42:00::","synset_id":"begin.v.02"},{"index":3,"sense_key":"begin%2:30:01::","synset_id":"begin.v.03"},{"index":4,"sense_key":"begin%2:32:03::","synset_id":"begin.v.04"},{"index":5,"sense_key":"begin%2:42:02::","synset_id":"begin.v.05"},{"index":6,"sense_key":"begin%2:42:04::","synset_id":"begin.v.06"},{"index":7,"sense_key":"begin%2:42:03::","synset_id":"begin.v.07"},{"index":8,"sense_key":"begin%2:41:04::","synset_id":"begin.v.08"},{"index":9,"sense_key":"begin%2:41:00::","synset_id":"begin.v.09"},{"index":10,"sense_key":"begin%2:32:04::","synset_id":"begin.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"begin%2:30:00::","call_ids":["senseval3.d002.s103.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"begin%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1167,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0024539999999999996,"input_uncached_usd":0.0023339999999999997,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4560950830345973}
{"item_id":"senseval3.d002.s103.t003","gold_sense_keys":["cry%2:29:00::"],"candidates":[{"index":1,"sense_key":"cry%2:32:01::","synset_id":"shout.v.02"},{"index":2,"sense_key":"cry%2:29:00::","synset_id":"cry.v.02"},{"index":3,"sense_key":"cry%2:32:07::","synset_id":"exclaim.v.01"},{"index":4,"sense_key":"cry%2:32:02::","synset_id":"cry.v.04"},{"index":5,"sense_key":"cry%2:34:00::","synset_id":"cry.v.05"},{"index":6,"sense_key":"cry%2:32:03::","synset_id":"cry.v.06"},{"index":7,"sense_key":"cry%2:29:02::","synset_id":"cry.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cry%2:29:00::","call_ids":["senseval3.d002.s103.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cry%2:29:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":936,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001992,"input_uncached_usd":0.001872,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3545007080538198}
{"item_id":"senseval3.d002.s106.t000","gold_sense_keys":["evening%1:28:00::"],"candidates":[{"index":1,"sense_key":"evening%1:28:00::","synset_id":"evening.n.01"},{"index":2,"sense_key":"evening%1:28:01::","synset_id":"evening.n.02"},{"index":3,"sense_key":"evening%1:28:02::","synset_id":"evening.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"evening%1:28:02::","call_ids":["senseval3.d002.s106.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"evening%1:28:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":583,"cached_input_tokens":0,"output_tokens":114,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002306,"input_uncached_usd":0.001166,"input_cached_usd":0.0,"output_usd":0.0011400000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7091481669340283}
{"item_id":"senseval3.d002.s106.t002","gold_sense_keys":["people%1:14:00::"],"candidates":[{"index":1,"sense_key":"people%1:14:00::","synset_id":"people.n.01"},{"index":2,"sense_key":"people%1:14:01::","synset_id":"citizenry.n.01"},{"index":3,"sense_key":"people%1:14:02::","synset_id":"people.n.03"},{"index":4,"sense_key":"people%1:14:03::","synset_id":"multitude.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"people%1:14:00::","call_ids":["senseval3.d002.s106.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"people%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":630,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00138,"input_uncached_usd":0.0012599999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1927303329575807}
{"item_id":"senseval3.d002.s106.t003","gold_sense_keys":["dog%1:05:00::"],"candidates":[{"index":1,"sense_key":"dog%1:05:00::","synset_id":"dog.n.01"},{"index":2,"sense_key":"dog%1:18:01::","synset_id":"frump.n.01"},{"index":3,"sense_key":"dog%1:18:00::","synset_id":"dog.n.03"},{"index":4,"sense_key":"dog%1:18:02::","synset_id":"cad.n.01"},{"index":5,"sense_key":"dog%1:13:01::","synset_id":"frank.n.02"},{"index":6,"sense_key":"dog%1:06:00::","synset_id":"pawl.n.01"},{"index":7,"sense_key":"dog%1:06:01::","synset_id":"andiron.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"dog%1:05:00::","call_ids":["senseval3.d002.s106.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"dog%1:05:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":899,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001918,"input_uncached_usd":0.001798,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3896630000090227}
{"item_id":"senseval3.d002.s106.t004","gold_sense_keys":["walk_around%2:38:00::"],"candidates":[{"index":1,"sense_key":"walk_around%2:38:00::","synset_id":"perambulate.v.02"},{"index":2,"sense_key":"walk_around%2:38:01::","synset_id":"circumambulate.v.01"},{"index":3,"sense_key":"walk_around%2:41:00::","synset_id":"walk_around.v.03"},{"index":4,"sense_key":"walk_around%2:41:01::","synset_id":"walk_around.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"walk_around%2:38:00::","call_ids":["senseval3.d002.s106.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"walk_around%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":638,"cached_input_tokens":0,"output_tokens":110,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002376,"input_uncached_usd":0.001276,"input_cached_usd":0.0,"output_usd":0.0011,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.807140416931361}
{"item_id":"senseval3.d002.s106.t005","gold_sense_keys":["drink%2:34:00::"],"candidates":[{"index":1,"sense_key":"drink%2:34:00::","synset_id":"drink.v.01"},{"index":2,"sense_key":"drink%2:34:01::","synset_id":"drink.v.02"},{"index":3,"sense_key":"drink%2:34:12::","synset_id":"toast.v.02"},{"index":4,"sense_key":"drink%2:31:00::","synset_id":"drink_in.v.01"},{"index":5,"sense_key":"drink%2:34:02::","synset_id":"drink.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"drink%2:34:00::","call_ids":["senseval3.d002.s106.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"drink%2:34:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":724,"cached_input_tokens":0,"output_tokens":238,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003828,"input_uncached_usd":0.0014479999999999999,"input_cached_usd":0.0,"output_usd":0.00238,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.4193273750133812}
{"item_id":"senseval3.d002.s109.t000","gold_sense_keys":["sit_down%2:38:00::"],"candidates":[{"index":1,"sense_key":"sit_down%2:38:00::","synset_id":"sit_down.v.01"},{"index":2,"sense_key":"sit_down%2:35:00::","synset_id":"seat.v.01"},{"index":3,"sense_key":"sit_down%2:35:03::","synset_id":"sit.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"sit_down%2:38:00::","call_ids":["senseval3.d002.s109.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"sit_down%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":510,"cached_input_tokens":0,"output_tokens":87,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00189,"input_uncached_usd":0.0010199999999999999,"input_cached_usd":0.0,"output_usd":0.0008700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4772992499638349}
{"item_id":"senseval3.d002.s109.t001","gold_sense_keys":["meet%2:38:06::","meet%2:41:01::"],"candidates":[{"index":1,"sense_key":"meet%2:38:01::","synset_id":"meet.v.01"},{"index":2,"sense_key":"meet%2:41:01::","synset_id":"meet.v.02"},{"index":3,"sense_key":"meet%2:42:02::","synset_id":"converge.v.01"},{"index":4,"sense_key":"meet%2:34:00::","synset_id":"meet.v.04"},{"index":5,"sense_key":"meet%2:42:01::","synset_id":"meet.v.05"},{"index":6,"sense_key":"meet%2:41:02::","synset_id":"meet.v.06"},{"index":7,"sense_key":"meet%2:41:00::","synset_id":"meet.v.07"},{"index":8,"sense_key":"meet%2:41:03::","synset_id":"meet.v.08"},{"index":9,"sense_key":"meet%2:38:06::","synset_id":"meet.v.09"},{"index":10,"sense_key":"meet%2:33:00::","synset_id":"meet.v.10"},{"index":11,"sense_key":"meet%2:42:00::","synset_id":"meet.v.11"},{"index":12,"sense_key":"meet%2:39:00::","synset_id":"suffer.v.10"},{"index":13,"sense_key":"meet%2:35:00::","synset_id":"touch.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"meet%2:41:01::","call_ids":["senseval3.d002.s109.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"meet%2:41:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1156,"cached_input_tokens":0,"output_tokens":116,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003472,"input_uncached_usd":0.002312,"input_cached_usd":0.0,"output_usd":0.00116,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1424806660506874}
{"item_id":"senseval3.d002.s109.t002","gold_sense_keys":["new%3:00:00::","new%5:00:00:other:00"],"candidates":[{"index":1,"sense_key":"new%3:00:00::","synset_id":"new.a.01"},{"index":2,"sense_key":"new%5:00:00:original:00","synset_id":"fresh.s.03"},{"index":3,"sense_key":"new%5:00:00:inexperienced:00","synset_id":"raw.s.11"},{"index":4,"sense_key":"new%5:00:00:unprecedented:00","synset_id":"new.s.03"},{"index":5,"sense_key":"new%5:00:00:other:00","synset_id":"new.s.04"},{"index":6,"sense_key":"new%3:00:09::","synset_id":"new.a.06"},{"index":7,"sense_key":"new%5:00:02:original:00","synset_id":"newfangled.s.01"},{"index":8,"sense_key":"new%5:00:02:late:01","synset_id":"new.s.06"},{"index":9,"sense_key":"new%5:00:01:late:01","synset_id":"modern.s.04"},{"index":10,"sense_key":"new%5:00:00:early:02","synset_id":"new.s.08"},{"index":11,"sense_key":"new%5:00:00:unaccustomed:00","synset_id":"new.s.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"new%3:00:00::","call_ids":["senseval3.d002.s109.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"new%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1058,"cached_input_tokens":0,"output_tokens":99,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003106,"input_uncached_usd":0.002116,"input_cached_usd":0.0,"output_usd":0.00099,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6040458750212565}
{"item_id":"senseval3.d002.s109.t004","gold_sense_keys":["client%1:18:01::"],"candidates":[{"index":1,"sense_key":"client%1:18:00::","synset_id":"client.n.01"},{"index":2,"sense_key":"client%1:18:01::","synset_id":"customer.n.01"},{"index":3,"sense_key":"client%1:06:00::","synset_id":"node.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"client%1:18:01::","call_ids":["senseval3.d002.s109.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"client%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":493,"cached_input_tokens":0,"output_tokens":104,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002026,"input_uncached_usd":0.000986,"input_cached_usd":0.0,"output_usd":0.0010400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7569045840064064}
{"item_id":"senseval3.d002.s109.t005","gold_sense_keys":["couple%1:14:01::"],"candidates":[{"index":1,"sense_key":"couple%1:14:00::","synset_id":"couple.n.01"},{"index":2,"sense_key":"couple%1:14:01::","synset_id":"couple.n.02"},{"index":3,"sense_key":"couple%1:23:01::","synset_id":"couple.n.03"},{"index":4,"sense_key":"couple%1:23:00::","synset_id":"couple.n.04"},{"index":5,"sense_key":"couple%1:17:00::","synset_id":"couple.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"couple%1:14:00::","call_ids":["senseval3.d002.s109.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"couple%1:14:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":630,"cached_input_tokens":0,"output_tokens":245,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00371,"input_uncached_usd":0.0012599999999999998,"input_cached_usd":0.0,"output_usd":0.0024500000000000004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.565090874908492}
{"item_id":"senseval3.d002.s109.t006","gold_sense_keys":["building%1:06:00::"],"candidates":[{"index":1,"sense_key":"building%1:06:00::","synset_id":"building.n.01"},{"index":2,"sense_key":"building%1:04:00::","synset_id":"construction.n.01"},{"index":3,"sense_key":"building%1:04:01::","synset_id":"construction.n.07"},{"index":4,"sense_key":"building%1:14:00::","synset_id":"building.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"building%1:06:00::","call_ids":["senseval3.d002.s109.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"building%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":619,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001358,"input_uncached_usd":0.001238,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9464772921055555}
{"item_id":"senseval3.d002.s109.t007","gold_sense_keys":["start%2:30:00::"],"candidates":[{"index":1,"sense_key":"start%2:30:00::","synset_id":"get_down.v.07"},{"index":2,"sense_key":"start%2:30:01::","synset_id":"begin.v.03"},{"index":3,"sense_key":"start%2:38:02::","synset_id":"depart.v.03"},{"index":4,"sense_key":"start%2:42:00::","synset_id":"begin.v.02"},{"index":5,"sense_key":"start%2:36:00::","synset_id":"originate.v.02"},{"index":6,"sense_key":"start%2:36:01::","synset_id":"start.v.06"},{"index":7,"sense_key":"start%2:38:04::","synset_id":"startle.v.02"},{"index":8,"sense_key":"start%2:38:01::","synset_id":"start.v.08"},{"index":9,"sense_key":"start%2:38:00::","synset_id":"start.v.09"},{"index":10,"sense_key":"start%2:41:00::","synset_id":"start.v.10"},{"index":11,"sense_key":"start%2:33:00::","synset_id":"start.v.11"},{"index":12,"sense_key":"start%2:42:03::","synset_id":"begin.v.07"},{"index":13,"sense_key":"start%2:41:04::","synset_id":"begin.v.08"},{"index":14,"sense_key":"start%2:38:03::","synset_id":"start.v.14"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"start%2:30:00::","call_ids":["senseval3.d002.s109.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"start%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1252,"cached_input_tokens":0,"output_tokens":131,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0038139999999999997,"input_uncached_usd":0.0025039999999999997,"input_cached_usd":0.0,"output_usd":0.0013100000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.7265011669369414}
{"item_id":"senseval3.d002.s109.t008","gold_sense_keys":["shake%2:29:00::","shake%2:38:00::","shake%2:38:01::","shake%2:38:02::"],"candidates":[{"index":1,"sense_key":"shake%2:38:00::","synset_id":"shake.v.01"},{"index":2,"sense_key":"shake%2:29:00::","synset_id":"shake.v.02"},{"index":3,"sense_key":"shake%2:38:01::","synset_id":"judder.v.01"},{"index":4,"sense_key":"shake%2:38:02::","synset_id":"rock.v.01"},{"index":5,"sense_key":"shake%2:30:00::","synset_id":"shake.v.05"},{"index":6,"sense_key":"shake%2:37:00::","synset_id":"stimulate.v.03"},{"index":7,"sense_key":"shake%2:38:04::","synset_id":"shake.v.07"},{"index":8,"sense_key":"shake%2:30:01::","synset_id":"shake.v.08"},{"index":9,"sense_key":"shake%2:32:00::","synset_id":"shake.v.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"shake%2:38:00::","call_ids":["senseval3.d002.s109.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"shake%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":930,"cached_input_tokens":0,"output_tokens":305,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00491,"input_uncached_usd":0.0018599999999999999,"input_cached_usd":0.0,"output_usd":0.00305,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.0124945839634165}
{"item_id":"senseval3.d002.s110.t000","gold_sense_keys":["flimsy%5:00:01:weak:00"],"candidates":[{"index":1,"sense_key":"flimsy%5:00:01:weak:00","synset_id":"flimsy.s.01"},{"index":2,"sense_key":"flimsy%3:00:02::","synset_id":"unconvincing.a.01"},{"index":3,"sense_key":"flimsy%5:00:03:insignificant:00","synset_id":"flimsy.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"flimsy%5:00:01:weak:00","call_ids":["senseval3.d002.s110.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"flimsy%5:00:01:weak:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":608,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001336,"input_uncached_usd":0.0012159999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.7937032909831032}
{"item_id":"senseval3.d002.s110.t001","gold_sense_keys":["structure%1:06:00::"],"candidates":[{"index":1,"sense_key":"structure%1:06:00::","synset_id":"structure.n.01"},{"index":2,"sense_key":"structure%1:07:00::","synset_id":"structure.n.02"},{"index":3,"sense_key":"structure%1:09:00::","synset_id":"structure.n.03"},{"index":4,"sense_key":"structure%1:08:00::","synset_id":"structure.n.04"},{"index":5,"sense_key":"structure%1:14:00::","synset_id":"social_organization.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"structure%1:06:00::","call_ids":["senseval3.d002.s110.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"structure%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":776,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001672,"input_uncached_usd":0.001552,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.072063292027451}
{"item_id":"senseval3.d002.s110.t002","gold_sense_keys":["build%2:36:00::"],"candidates":[{"index":1,"sense_key":"build%2:36:00::","synset_id":"construct.v.01"},{"index":2,"sense_key":"build%2:30:00::","synset_id":"build_up.v.02"},{"index":3,"sense_key":"build%2:36:09::","synset_id":"build.v.03"},{"index":4,"sense_key":"build%2:30:02::","synset_id":"build.v.04"},{"index":5,"sense_key":"build%2:41:00::","synset_id":"build.v.05"},{"index":6,"sense_key":"build%2:36:04::","synset_id":"build.v.06"},{"index":7,"sense_key":"build%2:36:10::","synset_id":"build.v.07"},{"index":8,"sense_key":"build%2:31:03::","synset_id":"build.v.08"},{"index":9,"sense_key":"build%2:30:05::","synset_id":"build_up.v.04"},{"index":10,"sense_key":"build%2:30:10::","synset_id":"build.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"build%2:36:00::","call_ids":["senseval3.d002.s110.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"build%2:36:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":929,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0024879999999999998,"input_uncached_usd":0.0018579999999999998,"input_cached_usd":0.0,"output_usd":0.00063,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1722960420884192}
{"item_id":"senseval3.d002.s110.t003","gold_sense_keys":["support%1:06:01::"],"candidates":[{"index":1,"sense_key":"support%1:04:04::","synset_id":"support.n.01"},{"index":2,"sense_key":"support%1:04:00::","synset_id":"support.n.02"},{"index":3,"sense_key":"support%1:09:00::","synset_id":"support.n.03"},{"index":4,"sense_key":"support%1:04:03::","synset_id":"support.n.04"},{"index":5,"sense_key":"support%1:04:02::","synset_id":"documentation.n.03"},{"index":6,"sense_key":"support%1:21:00::","synset_id":"support.n.06"},{"index":7,"sense_key":"support%1:06:01::","synset_id":"support.n.07"},{"index":8,"sense_key":"support%1:04:01::","synset_id":"support.n.08"},{"index":9,"sense_key":"support%1:10:01::","synset_id":"accompaniment.n.02"},{"index":10,"sense_key":"support%1:06:00::","synset_id":"support.n.10"},{"index":11,"sense_key":"support%1:21:01::","synset_id":"support.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"support%1:06:00::","call_ids":["senseval3.d002.s110.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"support%1:06:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1122,"cached_input_tokens":0,"output_tokens":172,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0039640000000000005,"input_uncached_usd":0.002244,"input_cached_usd":0.0,"output_usd":0.0017200000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.7339350410038605}
{"item_id":"senseval3.d002.s110.t004","gold_sense_keys":["rock%2:38:00::"],"candidates":[{"index":1,"sense_key":"rock%2:38:00::","synset_id":"rock.v.01"},{"index":2,"sense_key":"rock%2:38:06::","synset_id":"rock.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"rock%2:38:00::","call_ids":["senseval3.d002.s110.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"rock%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":511,"cached_input_tokens":0,"output_tokens":91,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0019320000000000001,"input_uncached_usd":0.001022,"input_cached_usd":0.0,"output_usd":0.0009100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.468042584019713}
{"item_id":"senseval3.d002.s111.t000","gold_sense_keys":["stop%2:42:00::"],"candidates":[{"index":1,"sense_key":"stop%2:38:00::","synset_id":"stop.v.01"},{"index":2,"sense_key":"stop%2:42:00::","synset_id":"discontinue.v.01"},{"index":3,"sense_key":"stop%2:41:00::","synset_id":"stop.v.03"},{"index":4,"sense_key":"stop%2:38:02::","synset_id":"stop.v.04"},{"index":5,"sense_key":"stop%2:38:01::","synset_id":"stop.v.05"},{"index":6,"sense_key":"stop%2:30:05::","synset_id":"break.v.10"},{"index":7,"sense_key":"stop%2:33:00::","synset_id":"check.v.18"},{"index":8,"sense_key":"stop%2:35:00::","synset_id":"intercept.v.01"},{"index":9,"sense_key":"stop%2:42:13::","synset_id":"end.v.01"},{"index":10,"sense_key":"stop%2:35:12::","synset_id":"barricade.v.01"},{"index":11,"sense_key":"stop%2:30:12::","synset_id":"hold_on.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"stop%2:42:00::","call_ids":["senseval3.d002.s111.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"stop%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1168,"cached_input_tokens":0,"output_tokens":82,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003156,"input_uncached_usd":0.002336,"input_cached_usd":0.0,"output_usd":0.0008200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.899021582910791}
{"item_id":"senseval3.d002.s111.t001","gold_sense_keys":["breathe%2:29:00::"],"candidates":[{"index":1,"sense_key":"breathe%2:29:00::","synset_id":"breathe.v.01"},{"index":2,"sense_key":"breathe%2:42:00::","synset_id":"breathe.v.02"},{"index":3,"sense_key":"breathe%2:40:00::","synset_id":"breathe.v.03"},{"index":4,"sense_key":"breathe%2:42:01::","synset_id":"breathe.v.04"},{"index":5,"sense_key":"breathe%2:32:01::","synset_id":"breathe.v.05"},{"index":6,"sense_key":"breathe%2:32:03::","synset_id":"breathe.v.06"},{"index":7,"sense_key":"breathe%2:32:00::","synset_id":"rest.v.02"},{"index":8,"sense_key":"breathe%2:30:03::","synset_id":"breathe.v.08"},{"index":9,"sense_key":"breathe%2:29:01::","synset_id":"emit.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"breathe%2:29:00::","call_ids":["senseval3.d002.s111.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"breathe%2:29:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":889,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001898,"input_uncached_usd":0.0017779999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1226479160832241}
{"item_id":"senseval3.d002.s111.t002","gold_sense_keys":["moment%1:28:01::"],"candidates":[{"index":1,"sense_key":"moment%1:28:00::","synset_id":"moment.n.01"},{"index":2,"sense_key":"moment%1:28:01::","synset_id":"moment.n.02"},{"index":3,"sense_key":"moment%1:28:02::","synset_id":"here_and_now.n.01"},{"index":4,"sense_key":"moment%1:07:00::","synset_id":"consequence.n.03"},{"index":5,"sense_key":"moment%1:19:00::","synset_id":"moment.n.05"},{"index":6,"sense_key":"moment%1:09:00::","synset_id":"moment.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"moment%1:28:01::","call_ids":["senseval3.d002.s111.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"moment%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":754,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001628,"input_uncached_usd":0.001508,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1931409169919789}
{"item_id":"senseval3.d002.s111.t003","gold_sense_keys":["keep%2:42:07::"],"candidates":[{"index":1,"sense_key":"keep%2:42:00::","synset_id":"keep.v.01"},{"index":2,"sense_key":"keep%2:42:07::","synset_id":"continue.v.01"},{"index":3,"sense_key":"keep%2:40:00::","synset_id":"keep.v.03"},{"index":4,"sense_key":"keep%2:41:01::","synset_id":"prevent.v.02"},{"index":5,"sense_key":"keep%2:41:03::","synset_id":"observe.v.09"},{"index":6,"sense_key":"keep%2:31:00::","synset_id":"observe.v.08"},{"index":7,"sense_key":"keep%2:40:01::","synset_id":"keep.v.07"},{"index":8,"sense_key":"keep%2:32:00::","synset_id":"keep.v.08"},{"index":9,"sense_key":"keep%2:42:01::","synset_id":"keep.v.09"},{"index":10,"sense_key":"keep%2:41:02::","synset_id":"retain.v.02"},{"index":11,"sense_key":"keep%2:34:00::","synset_id":"sustain.v.04"},{"index":12,"sense_key":"keep%2:42:03::","synset_id":"keep.v.12"},{"index":13,"sense_key":"keep%2:41:05::","synset_id":"observe.v.06"},{"index":14,"sense_key":"keep%2:41:00::","synset_id":"restrain.v.01"},{"index":15,"sense_key":"keep%2:42:02::","synset_id":"keep.v.15"},{"index":16,"sense_key":"keep%2:42:12::","synset_id":"keep.v.16"},{"index":17,"sense_key":"keep%2:40:09::","synset_id":"keep_open.v.01"},{"index":18,"sense_key":"keep%2:40:13::","synset_id":"keep.v.18"},{"index":19,"sense_key":"keep%2:40:12::","synset_id":"keep.v.19"},{"index":20,"sense_key":"keep%2:40:10::","synset_id":"keep.v.20"},{"index":21,"sense_key":"keep%2:35:10::","synset_id":"keep.v.21"},{"index":22,"sense_key":"keep%2:30:10::","synset_id":"preserve.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"keep%2:42:07::","call_ids":["senseval3.d002.s111.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"keep%2:42:07::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1751,"cached_input_tokens":0,"output_tokens":45,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003952,"input_uncached_usd":0.003502,"input_cached_usd":0.0,"output_usd":0.00045000000000000004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2589301250409335}
{"item_id":"senseval3.d002.s111.t004","gold_sense_keys":["come%2:30:01::"],"candidates":[{"index":1,"sense_key":"come%2:38:00::","synset_id":"come.v.01"},{"index":2,"sense_key":"come%2:38:04::","synset_id":"arrive.v.01"},{"index":3,"sense_key":"come%2:30:01::","synset_id":"come.v.03"},{"index":4,"sense_key":"come%2:30:03::","synset_id":"come.v.04"},{"index":5,"sense_key":"come%2:42:04::","synset_id":"come.v.05"},{"index":6,"sense_key":"come%2:30:02::","synset_id":"come.v.06"},{"index":7,"sense_key":"come%2:42:13::","synset_id":"issue_forth.v.01"},{"index":8,"sense_key":"come%2:42:09::","synset_id":"hail.v.02"},{"index":9,"sense_key":"come%2:42:03::","synset_id":"come.v.09"},{"index":10,"sense_key":"come%2:42:05::","synset_id":"come.v.10"},{"index":11,"sense_key":"come%2:38:03::","synset_id":"come.v.11"},{"index":12,"sense_key":"come%2:42:00::","synset_id":"fall.v.04"},{"index":13,"sense_key":"come%2:30:13::","synset_id":"come.v.13"},{"index":14,"sense_key":"come%2:42:12::","synset_id":"total.v.01"},{"index":15,"sense_key":"come%2:42:11::","synset_id":"come.v.15"},{"index":16,"sense_key":"come%2:38:02::","synset_id":"come.v.16"},{"index":17,"sense_key":"come%2:31:14::","synset_id":"occur.v.02"},{"index":18,"sense_key":"come%2:42:15::","synset_id":"derive.v.05"},{"index":19,"sense_key":"come%2:42:02::","synset_id":"do.v.04"},{"index":20,"sense_key":"come%2:39:00::","synset_id":"come.v.20"},{"index":21,"sense_key":"come%2:31:13::","synset_id":"come.v.21"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"come%2:30:01::","call_ids":["senseval3.d002.s111.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"come%2:30:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1580,"cached_input_tokens":0,"output_tokens":201,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00517,"input_uncached_usd":0.00316,"input_cached_usd":0.0,"output_usd":0.00201,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0683821249986067}
{"item_id":"senseval3.d002.s112.t001","gold_sense_keys":["say%2:32:00::"],"candidates":[{"index":1,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":6,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":9,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":10,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":11,"sense_key":"say%2:32:09::","synset_id":"say.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"say%2:32:00::","call_ids":["senseval3.d002.s112.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"say%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1020,"cached_input_tokens":0,"output_tokens":82,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0028599999999999997,"input_uncached_usd":0.0020399999999999997,"input_cached_usd":0.0,"output_usd":0.0008200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9395547920139506}
{"item_id":"senseval3.d002.s112.t004","gold_sense_keys":["session%1:04:00::"],"candidates":[{"index":1,"sense_key":"session%1:10:00::","synset_id":"session.n.01"},{"index":2,"sense_key":"session%1:28:00::","synset_id":"school_term.n.01"},{"index":3,"sense_key":"session%1:04:00::","synset_id":"session.n.03"},{"index":4,"sense_key":"session%1:14:00::","synset_id":"seance.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"session%1:04:00::","call_ids":["senseval3.d002.s112.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"session%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":578,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001736,"input_uncached_usd":0.001156,"input_cached_usd":0.0,"output_usd":0.00058,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1929912919877097}
{"item_id":"senseval3.d002.s116.t000","gold_sense_keys":["trouble%1:09:00::"],"candidates":[{"index":1,"sense_key":"trouble%1:09:00::","synset_id":"trouble.n.01"},{"index":2,"sense_key":"trouble%1:11:01::","synset_id":"fuss.n.02"},{"index":3,"sense_key":"trouble%1:11:00::","synset_id":"trouble.n.03"},{"index":4,"sense_key":"trouble%1:04:02::","synset_id":"trouble.n.04"},{"index":5,"sense_key":"trouble%1:12:03::","synset_id":"worry.n.02"},{"index":6,"sense_key":"trouble%1:26:02::","synset_id":"trouble.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"trouble%1:09:00::","call_ids":["senseval3.d002.s116.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"trouble%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":721,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002042,"input_uncached_usd":0.001442,"input_cached_usd":0.0,"output_usd":0.0006000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2411109999520704}
{"item_id":"senseval3.d002.s116.t005","gold_sense_keys":["spend%2:42:00::"],"candidates":[{"index":1,"sense_key":"spend%2:42:00::","synset_id":"spend.v.01"},{"index":2,"sense_key":"spend%2:40:00::","synset_id":"spend.v.02"},{"index":3,"sense_key":"spend%2:40:01::","synset_id":"spend.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"spend%2:42:00::","call_ids":["senseval3.d002.s116.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"spend%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":499,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001118,"input_uncached_usd":0.000998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8308512920048088}
{"item_id":"senseval3.d002.s116.t007","gold_sense_keys":["hour%1:28:00::"],"candidates":[{"index":1,"sense_key":"hour%1:28:00::","synset_id":"hour.n.01"},{"index":2,"sense_key":"hour%1:28:01::","synset_id":"hour.n.02"},{"index":3,"sense_key":"hour%1:28:02::","synset_id":"hour.n.03"},{"index":4,"sense_key":"hour%1:07:00::","synset_id":"hour.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"hour%1:28:00::","call_ids":["senseval3.d002.s116.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"hour%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":577,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001274,"input_uncached_usd":0.0011539999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.7727893750416115}
{"item_id":"senseval3.d002.s116.t008","gold_sense_keys":["know%2:31:01::"],"candidates":[{"index":1,"sense_key":"know%2:31:01::","synset_id":"know.v.01"},{"index":2,"sense_key":"know%2:31:03::","synset_id":"know.v.02"},{"index":3,"sense_key":"know%2:31:02::","synset_id":"know.v.03"},{"index":4,"sense_key":"know%2:31:00::","synset_id":"know.v.04"},{"index":5,"sense_key":"know%2:31:04::","synset_id":"know.v.05"},{"index":6,"sense_key":"know%2:31:06::","synset_id":"acknowledge.v.06"},{"index":7,"sense_key":"know%2:31:15::","synset_id":"know.v.07"},{"index":8,"sense_key":"know%2:35:00::","synset_id":"sleep_together.v.01"},{"index":9,"sense_key":"know%2:31:11::","synset_id":"know.v.09"},{"index":10,"sense_key":"know%2:31:12::","synset_id":"know.v.10"},{"index":11,"sense_key":"know%2:31:14::","synset_id":"know.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"know%2:31:01::","call_ids":["senseval3.d002.s116.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"know%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1083,"cached_input_tokens":0,"output_tokens":71,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002876,"input_uncached_usd":0.002166,"input_cached_usd":0.0,"output_usd":0.00071,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3498489171033725}
{"item_id":"senseval3.d002.s119.t002","gold_sense_keys":["start%2:30:00::"],"candidates":[{"index":1,"sense_key":"start%2:30:00::","synset_id":"get_down.v.07"},{"index":2,"sense_key":"start%2:30:01::","synset_id":"begin.v.03"},{"index":3,"sense_key":"start%2:38:02::","synset_id":"depart.v.03"},{"index":4,"sense_key":"start%2:42:00::","synset_id":"begin.v.02"},{"index":5,"sense_key":"start%2:36:00::","synset_id":"originate.v.02"},{"index":6,"sense_key":"start%2:36:01::","synset_id":"start.v.06"},{"index":7,"sense_key":"start%2:38:04::","synset_id":"startle.v.02"},{"index":8,"sense_key":"start%2:38:01::","synset_id":"start.v.08"},{"index":9,"sense_key":"start%2:38:00::","synset_id":"start.v.09"},{"index":10,"sense_key":"start%2:41:00::","synset_id":"start.v.10"},{"index":11,"sense_key":"start%2:33:00::","synset_id":"start.v.11"},{"index":12,"sense_key":"start%2:42:03::","synset_id":"begin.v.07"},{"index":13,"sense_key":"start%2:41:04::","synset_id":"begin.v.08"},{"index":14,"sense_key":"start%2:38:03::","synset_id":"start.v.14"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"start%2:30:00::","call_ids":["senseval3.d002.s119.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"start%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1257,"cached_input_tokens":0,"output_tokens":261,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.005124,"input_uncached_usd":0.0025139999999999997,"input_cached_usd":0.0,"output_usd":0.0026100000000000003,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.039390999940224}
{"item_id":"senseval3.d002.s119.t003","gold_sense_keys":["roll%2:38:10::"],"candidates":[{"index":1,"sense_key":"roll%2:38:00::","synset_id":"roll.v.01"},{"index":2,"sense_key":"roll%2:38:01::","synset_id":"wheel.v.03"},{"index":3,"sense_key":"roll%2:42:00::","synset_id":"roll.v.03"},{"index":4,"sense_key":"roll%2:35:05::","synset_id":"roll_out.v.01"},{"index":5,"sense_key":"roll%2:39:00::","synset_id":"roll.v.05"},{"index":6,"sense_key":"roll%2:35:00::","synset_id":"wind.v.03"},{"index":7,"sense_key":"roll%2:35:01::","synset_id":"roll.v.07"},{"index":8,"sense_key":"roll%2:30:10::","synset_id":"roll.v.08"},{"index":9,"sense_key":"roll%2:29:06::","synset_id":"roll.v.09"},{"index":10,"sense_key":"roll%2:40:00::","synset_id":"hustle.v.03"},{"index":11,"sense_key":"roll%2:38:10::","synset_id":"roll.v.11"},{"index":12,"sense_key":"roll%2:38:09::","synset_id":"roll.v.12"},{"index":13,"sense_key":"roll%2:38:12::","synset_id":"roll.v.13"},{"index":14,"sense_key":"roll%2:38:11::","synset_id":"roll.v.14"},{"index":15,"sense_key":"roll%2:32:00::","synset_id":"roll.v.15"},{"index":16,"sense_key":"roll%2:30:08::","synset_id":"seethe.v.04"},{"index":17,"sense_key":"roll%2:30:06::","synset_id":"roll.v.17"},{"index":18,"sense_key":"roll%2:30:07::","synset_id":"roll.v.18"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"roll%2:38:10::","call_ids":["senseval3.d002.s119.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"roll%2:38:10::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1464,"cached_input_tokens":0,"output_tokens":146,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004388,"input_uncached_usd":0.002928,"input_cached_usd":0.0,"output_usd":0.0014600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7718864170601591}
{"item_id":"senseval3.d002.s119.t004","gold_sense_keys":["ocean%1:17:00::"],"candidates":[{"index":1,"sense_key":"ocean%1:17:00::","synset_id":"ocean.n.01"},{"index":2,"sense_key":"ocean%1:23:00::","synset_id":"ocean.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ocean%1:17:00::","call_ids":["senseval3.d002.s119.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ocean%1:17:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":465,"cached_input_tokens":0,"output_tokens":82,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00175,"input_uncached_usd":0.0009299999999999999,"input_cached_usd":0.0,"output_usd":0.0008200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6696750830160454}
{"item_id":"senseval3.d002.s119.t005","gold_sense_keys":["wave%1:11:01::"],"candidates":[{"index":1,"sense_key":"wave%1:11:01::","synset_id":"wave.n.01"},{"index":2,"sense_key":"wave%1:04:02::","synset_id":"wave.n.02"},{"index":3,"sense_key":"wave%1:11:00::","synset_id":"wave.n.03"},{"index":4,"sense_key":"wave%1:11:02::","synset_id":"wave.n.04"},{"index":5,"sense_key":"wave%1:10:01::","synset_id":"wave.n.05"},{"index":6,"sense_key":"wave%1:08:00::","synset_id":"wave.n.06"},{"index":7,"sense_key":"wave%1:25:00::","synset_id":"wave.n.07"},{"index":8,"sense_key":"wave%1:19:00::","synset_id":"wave.n.08"},{"index":9,"sense_key":"wave%1:18:00::","synset_id":"wave.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"wave%1:11:01::","call_ids":["senseval3.d002.s119.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"wave%1:11:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":850,"cached_input_tokens":0,"output_tokens":50,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0021999999999999997,"input_uncached_usd":0.0017,"input_cached_usd":0.0,"output_usd":0.0005,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.730841042008251}
{"item_id":"senseval3.d002.s120.t000","gold_sense_keys":["run%2:38:00::"],"candidates":[{"index":1,"sense_key":"run%2:38:00::","synset_id":"run.v.01"},{"index":2,"sense_key":"run%2:38:04::","synset_id":"scat.v.01"},{"index":3,"sense_key":"run%2:42:00::","synset_id":"run.v.03"},{"index":4,"sense_key":"run%2:41:02::","synset_id":"operate.v.01"},{"index":5,"sense_key":"run%2:42:04::","synset_id":"run.v.05"},{"index":6,"sense_key":"run%2:38:01::","synset_id":"run.v.06"},{"index":7,"sense_key":"run%2:35:00::","synset_id":"function.v.01"},{"index":8,"sense_key":"run%2:42:07::","synset_id":"range.v.01"},{"index":9,"sense_key":"run%2:33:06::","synset_id":"campaign.v.01"},{"index":10,"sense_key":"run%2:36:01::","synset_id":"play.v.18"},{"index":11,"sense_key":"run%2:38:11::","synset_id":"run.v.11"},{"index":12,"sense_key":"run%2:42:01::","synset_id":"tend.v.01"},{"index":13,"sense_key":"run%2:35:04::","synset_id":"run.v.13"},{"index":14,"sense_key":"run%2:30:03::","synset_id":"run.v.14"},{"index":15,"sense_key":"run%2:30:07::","synset_id":"run.v.15"},{"index":16,"sense_key":"run%2:42:02::","synset_id":"run.v.16"},{"index":17,"sense_key":"run%2:42:03::","synset_id":"prevail.v.03"},{"index":18,"sense_key":"run%2:42:06::","synset_id":"run.v.18"},{"index":19,"sense_key":"run%2:41:00::","synset_id":"run.v.19"},{"index":20,"sense_key":"run%2:36:02::","synset_id":"carry.v.15"},{"index":21,"sense_key":"run%2:36:00::","synset_id":"run.v.21"},{"index":22,"sense_key":"run%2:35:01::","synset_id":"guide.v.05"},{"index":23,"sense_key":"run%2:42:08::","synset_id":"run.v.23"},{"index":24,"sense_key":"run%2:41:01::","synset_id":"run.v.24"},{"index":25,"sense_key":"run%2:40:06::","synset_id":"run.v.25"},{"index":26,"sense_key":"run%2:38:10::","synset_id":"run.v.26"},{"index":27,"sense_key":"run%2:38:08::","synset_id":"run.v.27"},{"index":28,"sense_key":"run%2:38:03::","synset_id":"run.v.28"},{"index":29,"sense_key":"run%2:38:05::","synset_id":"run.v.29"},{"index":30,"sense_key":"run%2:42:05::","synset_id":"run.v.30"},{"index":31,"sense_key":"run%2:41:04::","synset_id":"run.v.31"},{"index":32,"sense_key":"run%2:38:02::","synset_id":"run.v.32"},{"index":33,"sense_key":"run%2:38:06::","synset_id":"run.v.33"},{"index":34,"sense_key":"run%2:38:13::","synset_id":"run.v.34"},{"index":35,"sense_key":"run%2:38:09::","synset_id":"ply.v.03"},{"index":36,"sense_key":"run%2:33:02::","synset_id":"hunt.v.01"},{"index":37,"sense_key":"run%2:33:01::","synset_id":"race.v.02"},{"index":38,"sense_key":"run%2:30:02::","synset_id":"move.v.13"},{"index":39,"sense_key":"run%2:30:00::","synset_id":"melt.v.01"},{"index":40,"sense_key":"run%2:30:01::","synset_id":"ladder.v.01"},{"index":41,"sense_key":"run%2:30:04::","synset_id":"run.v.41"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"run%2:38:00::","call_ids":["senseval3.d002.s120.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"run%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2728,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.005576,"input_uncached_usd":0.005456,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.7838082499802113}
{"item_id":"senseval3.d002.s120.t001","gold_sense_keys":["house%1:06:00::"],"candidates":[{"index":1,"sense_key":"house%1:06:00::","synset_id":"house.n.01"},{"index":2,"sense_key":"house%1:14:01::","synset_id":"firm.n.01"},{"index":3,"sense_key":"house%1:14:04::","synset_id":"house.n.03"},{"index":4,"sense_key":"house%1:14:03::","synset_id":"house.n.04"},{"index":5,"sense_key":"house%1:14:02::","synset_id":"house.n.05"},{"index":6,"sense_key":"house%1:14:00::","synset_id":"house.n.06"},{"index":7,"sense_key":"house%1:04:00::","synset_id":"house.n.07"},{"index":8,"sense_key":"house%1:15:00::","synset_id":"sign_of_the_zodiac.n.01"},{"index":9,"sense_key":"house%1:14:06::","synset_id":"house.n.09"},{"index":10,"sense_key":"house%1:14:05::","synset_id":"family.n.01"},{"index":11,"sense_key":"house%1:06:02::","synset_id":"theater.n.01"},{"index":12,"sense_key":"house%1:06:01::","synset_id":"house.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"house%1:06:00::","call_ids":["senseval3.d002.s120.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"house%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1031,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002182,"input_uncached_usd":0.002062,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9080870829056948}
{"item_id":"senseval3.d002.s120.t002","gold_sense_keys":["get%2:35:03::"],"candidates":[{"index":1,"sense_key":"get%2:40:00::","synset_id":"get.v.01"},{"index":2,"sense_key":"get%2:30:00::","synset_id":"become.v.01"},{"index":3,"sense_key":"get%2:30:02::","synset_id":"get.v.03"},{"index":4,"sense_key":"get%2:30:03::","synset_id":"receive.v.02"},{"index":5,"sense_key":"get%2:38:00::","synset_id":"arrive.v.01"},{"index":6,"sense_key":"get%2:35:03::","synset_id":"bring.v.04"},{"index":7,"sense_key":"get%2:39:00::","synset_id":"experience.v.03"},{"index":8,"sense_key":"get%2:33:00::","synset_id":"pay_back.v.02"},{"index":9,"sense_key":"get%2:40:01::","synset_id":"have.v.17"},{"index":10,"sense_key":"get%2:32:00::","synset_id":"induce.v.02"},{"index":11,"sense_key":"get%2:35:09::","synset_id":"get.v.11"},{"index":12,"sense_key":"get%2:29:01::","synset_id":"grow.v.08"},{"index":13,"sense_key":"get%2:29:00::","synset_id":"contract.v.04"},{"index":14,"sense_key":"get%2:32:03::","synset_id":"get.v.14"},{"index":15,"sense_key":"get%2:30:01::","synset_id":"make.v.02"},{"index":16,"sense_key":"get%2:32:08::","synset_id":"drive.v.11"},{"index":17,"sense_key":"get%2:31:09::","synset_id":"catch.v.18"},{"index":18,"sense_key":"get%2:35:00::","synset_id":"catch.v.07"},{"index":19,"sense_key":"get%2:35:10::","synset_id":"get.v.19"},{"index":20,"sense_key":"get%2:31:03::","synset_id":"get.v.20"},{"index":21,"sense_key":"get%2:40:14::","synset_id":"get.v.21"},{"index":22,"sense_key":"get%2:40:13::","synset_id":"get.v.22"},{"index":23,"sense_key":"get%2:39:13::","synset_id":"catch.v.21"},{"index":24,"sense_key":"get%2:39:09::","synset_id":"catch.v.22"},{"index":25,"sense_key":"get%2:39:14::","synset_id":"get.v.25"},{"index":26,"sense_key":"get%2:38:15::","synset_id":"scram.v.01"},{"index":27,"sense_key":"get%2:38:12::","synset_id":"get.v.27"},{"index":28,"sense_key":"get%2:37:12::","synset_id":"get.v.28"},{"index":29,"sense_key":"get%2:37:11::","synset_id":"get.v.29"},{"index":30,"sense_key":"get%2:36:14::","synset_id":"catch.v.24"},{"index":31,"sense_key":"get%2:36:00::","synset_id":"draw.v.15"},{"index":32,"sense_key":"get%2:35:12::","synset_id":"get.v.32"},{"index":33,"sense_key":"get%2:31:10::","synset_id":"perplex.v.01"},{"index":34,"sense_key":"get%2:30:12::","synset_id":"get_down.v.07"},{"index":35,"sense_key":"get%2:29:11::","synset_id":"suffer.v.02"},{"index":36,"sense_key":"get%2:29:08::","synset_id":"beget.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"get%2:35:03::","call_ids":["senseval3.d002.s120.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"get%2:35:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2726,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.006062,"input_uncached_usd":0.005451999999999999,"input_cached_usd":0.0,"output_usd":0.0006100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.257313709007576}
{"item_id":"senseval3.d002.s120.t003","gold_sense_keys":["next%5:00:00:succeeding:00"],"candidates":[{"index":1,"sense_key":"next%5:00:00:succeeding:00","synset_id":"following.s.02"},{"index":2,"sense_key":"next%5:00:00:close:01","synset_id":"adjacent.s.01"},{"index":3,"sense_key":"next%5:00:00:incoming:00","synset_id":"future.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"next%5:00:00:succeeding:00","call_ids":["senseval3.d002.s120.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"next%5:00:00:succeeding:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":571,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001262,"input_uncached_usd":0.001142,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5367336249910295}
{"item_id":"senseval3.d002.s120.t004","gold_sense_keys":["tremor%1:11:00::"],"candidates":[{"index":1,"sense_key":"tremor%1:04:00::","synset_id":"tremor.n.01"},{"index":2,"sense_key":"tremor%1:11:00::","synset_id":"tremor.n.02"},{"index":3,"sense_key":"tremor%1:26:00::","synset_id":"tremor.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"tremor%1:11:00::","call_ids":["senseval3.d002.s120.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"tremor%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":528,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001176,"input_uncached_usd":0.0010559999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.599987416062504}
{"item_id":"senseval3.d002.s120.t005","gold_sense_keys":["throw%2:35:00::"],"candidates":[{"index":1,"sense_key":"throw%2:35:00::","synset_id":"throw.v.01"},{"index":2,"sense_key":"throw%2:38:00::","synset_id":"throw.v.02"},{"index":3,"sense_key":"throw%2:35:07::","synset_id":"shed.v.01"},{"index":4,"sense_key":"throw%2:35:02::","synset_id":"throw.v.04"},{"index":5,"sense_key":"throw%2:32:00::","synset_id":"give.v.07"},{"index":6,"sense_key":"throw%2:35:01::","synset_id":"throw.v.06"},{"index":7,"sense_key":"throw%2:36:02::","synset_id":"project.v.10"},{"index":8,"sense_key":"throw%2:38:03::","synset_id":"throw.v.08"},{"index":9,"sense_key":"throw%2:37:00::","synset_id":"bewilder.v.02"},{"index":10,"sense_key":"throw%2:32:03::","synset_id":"hurl.v.03"},{"index":11,"sense_key":"throw%2:36:08::","synset_id":"hold.v.03"},{"index":12,"sense_key":"throw%2:36:00::","synset_id":"throw.v.12"},{"index":13,"sense_key":"throw%2:35:06::","synset_id":"throw.v.13"},{"index":14,"sense_key":"throw%2:35:08::","synset_id":"throw.v.14"},{"index":15,"sense_key":"throw%2:31:00::","synset_id":"confuse.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"throw%2:35:00::","call_ids":["senseval3.d002.s120.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"throw%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1292,"cached_input_tokens":0,"output_tokens":125,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0038339999999999997,"input_uncached_usd":0.002584,"input_cached_usd":0.0,"output_usd":0.00125,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9949633330106735}
{"item_id":"senseval3.d002.s120.t006","gold_sense_keys":["air%1:15:00::"],"candidates":[{"index":1,"sense_key":"air%1:27:00::","synset_id":"air.n.01"},{"index":2,"sense_key":"air%1:15:00::","synset_id":"air.n.02"},{"index":3,"sense_key":"air%1:07:00::","synset_id":"air.n.03"},{"index":4,"sense_key":"air%1:19:00::","synset_id":"breeze.n.01"},{"index":5,"sense_key":"air%1:15:01::","synset_id":"atmosphere.n.03"},{"index":6,"sense_key":"air%1:27:01::","synset_id":"air.n.06"},{"index":7,"sense_key":"air%1:10:01::","synset_id":"tune.n.01"},{"index":8,"sense_key":"air%1:10:02::","synset_id":"air.n.08"},{"index":9,"sense_key":"air%1:04:00::","synset_id":"air_travel.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"air%1:15:00::","call_ids":["senseval3.d002.s120.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"air%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1010,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00214,"input_uncached_usd":0.00202,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1110325829358771}
{"item_id":"senseval3.d002.s120.t007","gold_sense_keys":["bounce%2:35:03::"],"candidates":[{"index":1,"sense_key":"bounce%2:38:00::","synset_id":"bounce.v.01"},{"index":2,"sense_key":"bounce%2:35:03::","synset_id":"bounce.v.02"},{"index":3,"sense_key":"bounce%2:38:02::","synset_id":"bounce.v.03"},{"index":4,"sense_key":"bounce%2:40:00::","synset_id":"bounce.v.04"},{"index":5,"sense_key":"bounce%2:38:01::","synset_id":"bounce.v.05"},{"index":6,"sense_key":"bounce%2:40:01::","synset_id":"bounce.v.06"},{"index":7,"sense_key":"bounce%2:35:00::","synset_id":"bounce.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"bounce%2:38:02::","call_ids":["senseval3.d002.s120.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"bounce%2:38:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":736,"cached_input_tokens":0,"output_tokens":191,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0033820000000000005,"input_uncached_usd":0.001472,"input_cached_usd":0.0,"output_usd":0.0019100000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.19513945793733}
{"item_id":"senseval3.d002.s120.t008","gold_sense_keys":["try%2:41:00::"],"candidates":[{"index":1,"sense_key":"try%2:41:00::","synset_id":"try.v.01"},{"index":2,"sense_key":"try%2:41:01::","synset_id":"test.v.01"},{"index":3,"sense_key":"try%2:41:06::","synset_id":"judge.v.05"},{"index":4,"sense_key":"try%2:34:00::","synset_id":"sample.v.01"},{"index":5,"sense_key":"try%2:41:05::","synset_id":"hear.v.03"},{"index":6,"sense_key":"try%2:37:01::","synset_id":"try.v.06"},{"index":7,"sense_key":"try%2:37:00::","synset_id":"try.v.07"},{"index":8,"sense_key":"try%2:30:00::","synset_id":"try.v.08"},{"index":9,"sense_key":"try%2:29:00::","synset_id":"try_on.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"try%2:41:00::","call_ids":["senseval3.d002.s120.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"try%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":984,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0020879999999999996,"input_uncached_usd":0.0019679999999999997,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9483433340210468}
{"item_id":"senseval3.d002.s121.t000","gold_sense_keys":["fine%5:00:00:satisfactory:00"],"candidates":[{"index":1,"sense_key":"fine%5:00:00:satisfactory:00","synset_id":"all_right.s.01"},{"index":2,"sense_key":"fine%5:00:00:precise:00","synset_id":"fine.s.02"},{"index":3,"sense_key":"fine%5:00:00:thin:01","synset_id":"fine.s.03"},{"index":4,"sense_key":"fine%5:00:00:elegant:00","synset_id":"fine.s.04"},{"index":5,"sense_key":"fine%3:00:00::","synset_id":"fine.a.05"},{"index":6,"sense_key":"fine%5:00:00:pure:02","synset_id":"fine.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"fine%5:00:00:satisfactory:00","call_ids":["senseval3.d002.s121.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"fine%5:00:00:satisfactory:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":761,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001642,"input_uncached_usd":0.001522,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.058696125051938}
{"item_id":"senseval3.d002.s122.t001","gold_sense_keys":["full%3:00:00::"],"candidates":[{"index":1,"sense_key":"full%3:00:00::","synset_id":"full.a.01"},{"index":2,"sense_key":"full%5:00:01:whole:00","synset_id":"entire.s.01"},{"index":3,"sense_key":"full%5:00:00:complete:00","synset_id":"full.s.02"},{"index":4,"sense_key":"full%5:00:00:nourished:00","synset_id":"full.s.03"},{"index":5,"sense_key":"full%3:00:01::","synset_id":"full.a.05"},{"index":6,"sense_key":"full%5:00:00:ample:00","synset_id":"full.s.04"},{"index":7,"sense_key":"full%5:00:00:high:02","synset_id":"broad.s.04"},{"index":8,"sense_key":"full%5:00:02:ample:00","synset_id":"wide.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"full%3:00:00::","call_ids":["senseval3.d002.s122.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"full%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":810,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00174,"input_uncached_usd":0.00162,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1280719169881195}
{"item_id":"senseval3.d002.s122.t002","gold_sense_keys":["crystal%1:06:01::"],"candidates":[{"index":1,"sense_key":"crystal%1:27:00::","synset_id":"crystal.n.01"},{"index":2,"sense_key":"crystal%1:06:02::","synset_id":"crystal.n.02"},{"index":3,"sense_key":"crystal%1:17:00::","synset_id":"crystal.n.03"},{"index":4,"sense_key":"crystal%1:27:01::","synset_id":"quartz_glass.n.01"},{"index":5,"sense_key":"crystal%1:06:01::","synset_id":"crystal.n.05"},{"index":6,"sense_key":"crystal%1:06:00::","synset_id":"crystal.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"crystal%1:06:01::","call_ids":["senseval3.d002.s122.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"crystal%1:06:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":659,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001968,"input_uncached_usd":0.001318,"input_cached_usd":0.0,"output_usd":0.0006500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3336454579839483}
{"item_id":"senseval3.d002.s123.t000","gold_sense_keys":["book%1:06:00::"],"candidates":[{"index":1,"sense_key":"book%1:10:00::","synset_id":"book.n.01"},{"index":2,"sense_key":"book%1:06:00::","synset_id":"book.n.02"},{"index":3,"sense_key":"book%1:10:02::","synset_id":"record.n.05"},{"index":4,"sense_key":"book%1:10:01::","synset_id":"script.n.01"},{"index":5,"sense_key":"book%1:21:00::","synset_id":"ledger.n.01"},{"index":6,"sense_key":"book%1:14:01::","synset_id":"book.n.06"},{"index":7,"sense_key":"book%1:14:00::","synset_id":"book.n.07"},{"index":8,"sense_key":"book%1:10:05::","synset_id":"koran.n.01"},{"index":9,"sense_key":"book%1:10:04::","synset_id":"bible.n.01"},{"index":10,"sense_key":"book%1:10:03::","synset_id":"book.n.10"},{"index":11,"sense_key":"book%1:06:02::","synset_id":"book.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"book%1:06:00::","call_ids":["senseval3.d002.s123.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"book%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1067,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0027340000000000003,"input_uncached_usd":0.002134,"input_cached_usd":0.0,"output_usd":0.0006000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6186438750009984}
{"item_id":"senseval3.d002.s123.t001","gold_sense_keys":["tape%1:06:01::","tape%1:06:04::"],"candidates":[{"index":1,"sense_key":"tape%1:06:00::","synset_id":"tape.n.01"},{"index":2,"sense_key":"tape%1:06:01::","synset_id":"tape.n.02"},{"index":3,"sense_key":"tape%1:15:00::","synset_id":"tape.n.03"},{"index":4,"sense_key":"tape%1:06:03::","synset_id":"tape.n.04"},{"index":5,"sense_key":"tape%1:06:04::","synset_id":"magnetic_tape.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"tape%1:06:04::","call_ids":["senseval3.d002.s123.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"tape%1:06:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":747,"cached_input_tokens":0,"output_tokens":90,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002394,"input_uncached_usd":0.0014939999999999999,"input_cached_usd":0.0,"output_usd":0.0009000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1921679580118507}
{"item_id":"senseval3.d002.s123.t002","gold_sense_keys":["room%1:06:00::"],"candidates":[{"index":1,"sense_key":"room%1:06:00::","synset_id":"room.n.01"},{"index":2,"sense_key":"room%1:23:00::","synset_id":"room.n.02"},{"index":3,"sense_key":"room%1:26:00::","synset_id":"room.n.03"},{"index":4,"sense_key":"room%1:14:00::","synset_id":"room.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"room%1:06:00::","call_ids":["senseval3.d002.s123.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"room%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":553,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001226,"input_uncached_usd":0.001106,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0965337080415338}
{"item_id":"senseval3.d002.s124.t000","gold_sense_keys":["thing%1:06:00::"],"candidates":[{"index":1,"sense_key":"thing%1:26:00::","synset_id":"thing.n.01"},{"index":2,"sense_key":"thing%1:04:00::","synset_id":"thing.n.02"},{"index":3,"sense_key":"thing%1:09:00::","synset_id":"thing.n.03"},{"index":4,"sense_key":"thing%1:06:00::","synset_id":"thing.n.04"},{"index":5,"sense_key":"thing%1:11:00::","synset_id":"thing.n.05"},{"index":6,"sense_key":"thing%1:09:02::","synset_id":"matter.n.01"},{"index":7,"sense_key":"thing%1:10:00::","synset_id":"thing.n.07"},{"index":8,"sense_key":"thing%1:06:01::","synset_id":"thing.n.08"},{"index":9,"sense_key":"thing%1:07:00::","synset_id":"thing.n.09"},{"index":10,"sense_key":"thing%1:09:01::","synset_id":"thing.n.10"},{"index":11,"sense_key":"thing%1:12:00::","synset_id":"thing.n.11"},{"index":12,"sense_key":"thing%1:03:00::","synset_id":"thing.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":12,"chosen_sense_key":"thing%1:03:00::","call_ids":["senseval3.d002.s124.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":12,"predicted_sense_key":"thing%1:03:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":976,"cached_input_tokens":0,"output_tokens":193,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003882,"input_uncached_usd":0.001952,"input_cached_usd":0.0,"output_usd":0.00193,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.6592576670227572}
{"item_id":"senseval3.d002.s124.t001","gold_sense_keys":["house%1:06:00::"],"candidates":[{"index":1,"sense_key":"house%1:06:00::","synset_id":"house.n.01"},{"index":2,"sense_key":"house%1:14:01::","synset_id":"firm.n.01"},{"index":3,"sense_key":"house%1:14:04::","synset_id":"house.n.03"},{"index":4,"sense_key":"house%1:14:03::","synset_id":"house.n.04"},{"index":5,"sense_key":"house%1:14:02::","synset_id":"house.n.05"},{"index":6,"sense_key":"house%1:14:00::","synset_id":"house.n.06"},{"index":7,"sense_key":"house%1:04:00::","synset_id":"house.n.07"},{"index":8,"sense_key":"house%1:15:00::","synset_id":"sign_of_the_zodiac.n.01"},{"index":9,"sense_key":"house%1:14:06::","synset_id":"house.n.09"},{"index":10,"sense_key":"house%1:14:05::","synset_id":"family.n.01"},{"index":11,"sense_key":"house%1:06:02::","synset_id":"theater.n.01"},{"index":12,"sense_key":"house%1:06:01::","synset_id":"house.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"house%1:06:00::","call_ids":["senseval3.d002.s124.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"house%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1066,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0022519999999999997,"input_uncached_usd":0.0021319999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2708302079699934}
{"item_id":"senseval3.d002.s124.t002","gold_sense_keys":["structure%1:06:00::"],"candidates":[{"index":1,"sense_key":"structure%1:06:00::","synset_id":"structure.n.01"},{"index":2,"sense_key":"structure%1:07:00::","synset_id":"structure.n.02"},{"index":3,"sense_key":"structure%1:09:00::","synset_id":"structure.n.03"},{"index":4,"sense_key":"structure%1:08:00::","synset_id":"structure.n.04"},{"index":5,"sense_key":"structure%1:14:00::","synset_id":"social_organization.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"structure%1:06:00::","call_ids":["senseval3.d002.s124.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"structure%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":801,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001722,"input_uncached_usd":0.001602,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7406989579321817}
{"item_id":"senseval3.d002.s124.t003","gold_sense_keys":["fine%5:00:00:satisfactory:00"],"candidates":[{"index":1,"sense_key":"fine%5:00:00:satisfactory:00","synset_id":"all_right.s.01"},{"index":2,"sense_key":"fine%5:00:00:precise:00","synset_id":"fine.s.02"},{"index":3,"sense_key":"fine%5:00:00:thin:01","synset_id":"fine.s.03"},{"index":4,"sense_key":"fine%5:00:00:elegant:00","synset_id":"fine.s.04"},{"index":5,"sense_key":"fine%3:00:00::","synset_id":"fine.a.05"},{"index":6,"sense_key":"fine%5:00:00:pure:02","synset_id":"fine.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"fine%5:00:00:satisfactory:00","call_ids":["senseval3.d002.s124.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"fine%5:00:00:satisfactory:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":800,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00172,"input_uncached_usd":0.0015999999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3903156659798697}
{"item_id":"senseval3.d002.s125.t000","gold_sense_keys":["stand%2:35:00::","stand%2:42:03::"],"candidates":[{"index":1,"sense_key":"stand%2:35:00::","synset_id":"stand.v.01"},{"index":2,"sense_key":"stand%2:42:04::","synset_id":"stand.v.02"},{"index":3,"sense_key":"stand%2:42:03::","synset_id":"stand.v.03"},{"index":4,"sense_key":"stand%2:42:00::","synset_id":"stand.v.04"},{"index":5,"sense_key":"stand%2:31:00::","synset_id":"digest.v.03"},{"index":6,"sense_key":"stand%2:31:01::","synset_id":"stand.v.06"},{"index":7,"sense_key":"stand%2:42:06::","synset_id":"stand.v.07"},{"index":8,"sense_key":"stand%2:42:02::","synset_id":"stand.v.08"},{"index":9,"sense_key":"stand%2:42:01::","synset_id":"stand.v.09"},{"index":10,"sense_key":"stand%2:35:01::","synset_id":"stand.v.10"},{"index":11,"sense_key":"stand%2:33:00::","synset_id":"resist.v.04"},{"index":12,"sense_key":"stand%2:35:10::","synset_id":"stand.v.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"stand%2:35:00::","call_ids":["senseval3.d002.s125.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"stand%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1055,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0022299999999999998,"input_uncached_usd":0.00211,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4177470409777015}
{"item_id":"senseval3.d002.s125.t002","gold_sense_keys":["wait%2:42:00::"],"candidates":[{"index":1,"sense_key":"wait%2:42:00::","synset_id":"wait.v.01"},{"index":2,"sense_key":"wait%2:42:01::","synset_id":"wait.v.02"},{"index":3,"sense_key":"wait%2:31:00::","synset_id":"expect.v.03"},{"index":4,"sense_key":"wait%2:41:00::","synset_id":"wait.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"wait%2:42:00::","call_ids":["senseval3.d002.s125.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"wait%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":664,"cached_input_tokens":0,"output_tokens":97,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002298,"input_uncached_usd":0.001328,"input_cached_usd":0.0,"output_usd":0.00097,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6292614160338417}
{"item_id":"senseval3.d002.s125.t003","gold_sense_keys":["tremor%1:11:00::"],"candidates":[{"index":1,"sense_key":"tremor%1:04:00::","synset_id":"tremor.n.01"},{"index":2,"sense_key":"tremor%1:11:00::","synset_id":"tremor.n.02"},{"index":3,"sense_key":"tremor%1:26:00::","synset_id":"tremor.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"tremor%1:11:00::","call_ids":["senseval3.d002.s125.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"tremor%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":545,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0012100000000000001,"input_uncached_usd":0.00109,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4864534999942407}
{"item_id":"senseval3.d002.s125.t004","gold_sense_keys":["notice%2:39:00::"],"candidates":[{"index":1,"sense_key":"notice%2:39:04::","synset_id":"detect.v.01"},{"index":2,"sense_key":"notice%2:39:00::","synset_id":"notice.v.02"},{"index":3,"sense_key":"notice%2:32:00::","synset_id":"comment.v.01"},{"index":4,"sense_key":"notice%2:32:01::","synset_id":"notice.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"notice%2:39:00::","call_ids":["senseval3.d002.s125.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"notice%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":695,"cached_input_tokens":0,"output_tokens":142,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00281,"input_uncached_usd":0.00139,"input_cached_usd":0.0,"output_usd":0.00142,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1499974579783157}
{"item_id":"senseval3.d002.s125.t006","gold_sense_keys":["emerge%2:30:00::","emerge%2:30:02::","emerge%2:38:00::"],"candidates":[{"index":1,"sense_key":"emerge%2:30:00::","synset_id":"emerge.v.01"},{"index":2,"sense_key":"emerge%2:30:02::","synset_id":"issue.v.04"},{"index":3,"sense_key":"emerge%2:30:01::","synset_id":"emerge.v.03"},{"index":4,"sense_key":"emerge%2:38:00::","synset_id":"emerge.v.04"},{"index":5,"sense_key":"emerge%2:42:00::","synset_id":"come_forth.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"emerge%2:30:00::","call_ids":["senseval3.d002.s125.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"emerge%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":668,"cached_input_tokens":0,"output_tokens":102,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002356,"input_uncached_usd":0.001336,"input_cached_usd":0.0,"output_usd":0.00102,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1750506670214236}
{"item_id":"senseval3.d002.s125.t007","gold_sense_keys":["ground%1:27:00::"],"candidates":[{"index":1,"sense_key":"ground%1:17:00::","synset_id":"land.n.04"},{"index":2,"sense_key":"ground%1:16:00::","synset_id":"reason.n.01"},{"index":3,"sense_key":"ground%1:27:00::","synset_id":"earth.n.02"},{"index":4,"sense_key":"ground%1:24:00::","synset_id":"footing.n.02"},{"index":5,"sense_key":"ground%1:15:00::","synset_id":"ground.n.05"},{"index":6,"sense_key":"ground%1:09:00::","synset_id":"background.n.02"},{"index":7,"sense_key":"ground%1:17:01::","synset_id":"land.n.02"},{"index":8,"sense_key":"ground%1:09:01::","synset_id":"ground.n.08"},{"index":9,"sense_key":"ground%1:06:02::","synset_id":"ground.n.09"},{"index":10,"sense_key":"ground%1:06:01::","synset_id":"ground.n.10"},{"index":11,"sense_key":"ground%1:06:00::","synset_id":"flat_coat.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"ground%1:27:00::","call_ids":["senseval3.d002.s125.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"ground%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1140,"cached_input_tokens":0,"output_tokens":206,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00434,"input_uncached_usd":0.00228,"input_cached_usd":0.0,"output_usd":0.00206,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0830600829795003}
{"item_id":"senseval3.d002.s128.t000","gold_sense_keys":["amazing%5:00:00:surprising:00"],"candidates":[{"index":1,"sense_key":"amazing%5:00:00:surprising:00","synset_id":"amazing.s.01"},{"index":2,"sense_key":"amazing%5:00:00:impressive:00","synset_id":"amazing.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"amazing%5:00:00:surprising:00","call_ids":["senseval3.d002.s128.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"amazing%5:00:00:surprising:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":546,"cached_input_tokens":0,"output_tokens":86,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001952,"input_uncached_usd":0.0010919999999999999,"input_cached_usd":0.0,"output_usd":0.0008600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.931556083029136}
{"item_id":"senseval3.d002.s128.t001","gold_sense_keys":["second%1:28:00::"],"candidates":[{"index":1,"sense_key":"second%1:28:00::","synset_id":"second.n.01"},{"index":2,"sense_key":"second%1:28:01::","synset_id":"moment.n.02"},{"index":3,"sense_key":"second%1:04:00::","synset_id":"second_base.n.02"},{"index":4,"sense_key":"second%1:28:02::","synset_id":"moment.n.01"},{"index":5,"sense_key":"second%1:24:00::","synset_id":"second.n.05"},{"index":6,"sense_key":"second%1:23:00::","synset_id":"second.n.06"},{"index":7,"sense_key":"second%1:18:00::","synset_id":"second.n.07"},{"index":8,"sense_key":"second%1:10:00::","synset_id":"second.n.08"},{"index":9,"sense_key":"second%1:06:00::","synset_id":"second_gear.n.01"},{"index":10,"sense_key":"second%1:06:01::","synset_id":"irregular.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"second%1:28:00::","call_ids":["senseval3.d002.s128.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"second%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":941,"cached_input_tokens":0,"output_tokens":238,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004262,"input_uncached_usd":0.001882,"input_cached_usd":0.0,"output_usd":0.00238,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.5283437919570133}
{"item_id":"senseval3.d002.s128.t002","gold_sense_keys":["change%2:30:01::"],"candidates":[{"index":1,"sense_key":"change%2:30:01::","synset_id":"change.v.01"},{"index":2,"sense_key":"change%2:30:00::","synset_id":"change.v.02"},{"index":3,"sense_key":"change%2:30:02::","synset_id":"change.v.03"},{"index":4,"sense_key":"change%2:30:05::","synset_id":"switch.v.03"},{"index":5,"sense_key":"change%2:30:03::","synset_id":"change.v.05"},{"index":6,"sense_key":"change%2:30:04::","synset_id":"change.v.06"},{"index":7,"sense_key":"change%2:40:00::","synset_id":"exchange.v.01"},{"index":8,"sense_key":"change%2:38:00::","synset_id":"transfer.v.06"},{"index":9,"sense_key":"change%2:30:08::","synset_id":"deepen.v.04"},{"index":10,"sense_key":"change%2:30:06::","synset_id":"change.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"change%2:30:01::","call_ids":["senseval3.d002.s128.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"change%2:30:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1063,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002776,"input_uncached_usd":0.002126,"input_cached_usd":0.0,"output_usd":0.0006500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4485218330519274}
{"item_id":"senseval3.d002.s128.t003","gold_sense_keys":["life%1:26:02::"],"candidates":[{"index":1,"sense_key":"life%1:26:01::","synset_id":"life.n.01"},{"index":2,"sense_key":"life%1:09:00::","synset_id":"life.n.02"},{"index":3,"sense_key":"life%1:26:02::","synset_id":"life.n.03"},{"index":4,"sense_key":"life%1:26:00::","synset_id":"animation.n.01"},{"index":5,"sense_key":"life%1:28:00::","synset_id":"life.n.05"},{"index":6,"sense_key":"life%1:28:01::","synset_id":"life.n.06"},{"index":7,"sense_key":"life%1:28:02::","synset_id":"life.n.07"},{"index":8,"sense_key":"life%1:18:00::","synset_id":"life.n.08"},{"index":9,"sense_key":"life%1:07:01::","synset_id":"liveliness.n.02"},{"index":10,"sense_key":"life%1:03:00::","synset_id":"life.n.10"},{"index":11,"sense_key":"life%1:19:00::","synset_id":"life.n.11"},{"index":12,"sense_key":"life%1:10:00::","synset_id":"biography.n.01"},{"index":13,"sense_key":"life%1:16:00::","synset_id":"life.n.13"},{"index":14,"sense_key":"life%1:28:03::","synset_id":"life_sentence.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"life%1:26:02::","call_ids":["senseval3.d002.s128.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"life%1:26:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1122,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002884,"input_uncached_usd":0.002244,"input_cached_usd":0.0,"output_usd":0.00064,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8501123329624534}
{"item_id":"senseval3.d002.s131.t000","gold_sense_keys":["guess%2:31:00::"],"candidates":[{"index":1,"sense_key":"guess%2:31:00::","synset_id":"think.v.02"},{"index":2,"sense_key":"guess%2:32:00::","synset_id":"guess.v.02"},{"index":3,"sense_key":"guess%2:31:01::","synset_id":"estimate.v.01"},{"index":4,"sense_key":"guess%2:31:02::","synset_id":"guess.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"guess%2:31:00::","call_ids":["senseval3.d002.s131.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"guess%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":654,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001428,"input_uncached_usd":0.001308,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8164469579933211}
{"item_id":"senseval3.d002.s131.t001","gold_sense_keys":["live%2:42:04::","live%2:42:06::"],"candidates":[{"index":1,"sense_key":"live%2:42:08::","synset_id":"populate.v.01"},{"index":2,"sense_key":"live%2:42:06::","synset_id":"live.v.02"},{"index":3,"sense_key":"live%2:42:04::","synset_id":"survive.v.01"},{"index":4,"sense_key":"live%2:42:01::","synset_id":"exist.v.02"},{"index":5,"sense_key":"live%2:42:00::","synset_id":"be.v.11"},{"index":6,"sense_key":"live%2:31:00::","synset_id":"know.v.05"},{"index":7,"sense_key":"live%2:42:07::","synset_id":"live.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"live%2:42:06::","call_ids":["senseval3.d002.s131.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"live%2:42:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":810,"cached_input_tokens":0,"output_tokens":84,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00246,"input_uncached_usd":0.00162,"input_cached_usd":0.0,"output_usd":0.00084,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3770628749625757}
{"item_id":"senseval3.d002.s131.t002","gold_sense_keys":["wait%2:31:00::"],"candidates":[{"index":1,"sense_key":"wait%2:42:00::","synset_id":"wait.v.01"},{"index":2,"sense_key":"wait%2:42:01::","synset_id":"wait.v.02"},{"index":3,"sense_key":"wait%2:31:00::","synset_id":"expect.v.03"},{"index":4,"sense_key":"wait%2:41:00::","synset_id":"wait.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"wait%2:31:00::","call_ids":["senseval3.d002.s131.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"wait%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":590,"cached_input_tokens":0,"output_tokens":148,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00266,"input_uncached_usd":0.0011799999999999998,"input_cached_usd":0.0,"output_usd":0.0014800000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0896681669401005}
{"item_id":"senseval3.d002.s132.t000","gold_sense_keys":["hard%3:00:06::"],"candidates":[{"index":1,"sense_key":"hard%3:00:06::","synset_id":"difficult.a.01"},{"index":2,"sense_key":"hard%3:00:02::","synset_id":"hard.a.02"},{"index":3,"sense_key":"hard%3:00:01::","synset_id":"hard.a.03"},{"index":4,"sense_key":"hard%5:00:00:strong:00","synset_id":"hard.s.01"},{"index":5,"sense_key":"hard%5:00:01:effortful:00","synset_id":"arduous.s.01"},{"index":6,"sense_key":"hard%3:00:00::","synset_id":"unvoiced.a.01"},{"index":7,"sense_key":"hard%3:00:05::","synset_id":"hard.a.07"},{"index":8,"sense_key":"hard%3:00:03::","synset_id":"hard.a.08"},{"index":9,"sense_key":"hard%5:00:00:indulgent:00","synset_id":"intemperate.s.01"},{"index":10,"sense_key":"hard%5:00:00:alcoholic:00","synset_id":"hard.s.04"},{"index":11,"sense_key":"hard%5:00:00:bad:00","synset_id":"hard.s.05"},{"index":12,"sense_key":"hard%5:00:00:stale:00","synset_id":"hard.s.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"hard%3:00:06::","call_ids":["senseval3.d002.s132.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"hard%3:00:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1119,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002908,"input_uncached_usd":0.002238,"input_cached_usd":0.0,"output_usd":0.00067,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3080828340025619}
{"item_id":"senseval3.d002.s132.t001","gold_sense_keys":["accept%2:31:00::"],"candidates":[{"index":1,"sense_key":"accept%2:31:00::","synset_id":"accept.v.01"},{"index":2,"sense_key":"accept%2:40:00::","synset_id":"accept.v.02"},{"index":3,"sense_key":"accept%2:32:00::","synset_id":"accept.v.03"},{"index":4,"sense_key":"accept%2:31:01::","synset_id":"accept.v.04"},{"index":5,"sense_key":"accept%2:40:03::","synset_id":"accept.v.05"},{"index":6,"sense_key":"accept%2:40:01::","synset_id":"bear.v.06"},{"index":7,"sense_key":"accept%2:31:03::","synset_id":"accept.v.07"},{"index":8,"sense_key":"accept%2:42:00::","synset_id":"accept.v.08"},{"index":9,"sense_key":"accept%2:40:07::","synset_id":"accept.v.09"},{"index":10,"sense_key":"accept%2:40:12::","synset_id":"take.v.20"},{"index":11,"sense_key":"accept%2:31:08::","synset_id":"accept.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"accept%2:31:00::","call_ids":["senseval3.d002.s132.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"accept%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1015,"cached_input_tokens":0,"output_tokens":232,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00435,"input_uncached_usd":0.00203,"input_cached_usd":0.0,"output_usd":0.00232,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.0300623750081286}
{"item_id":"senseval3.d002.s132.t002","gold_sense_keys":["take%2:40:06::"],"candidates":[{"index":1,"sense_key":"take%2:41:04::","synset_id":"take.v.01"},{"index":2,"sense_key":"take%2:40:06::","synset_id":"take.v.02"},{"index":3,"sense_key":"take%2:38:09::","synset_id":"lead.v.01"},{"index":4,"sense_key":"take%2:35:00::","synset_id":"take.v.04"},{"index":5,"sense_key":"take%2:30:01::","synset_id":"assume.v.03"},{"index":6,"sense_key":"take%2:31:07::","synset_id":"take.v.06"},{"index":7,"sense_key":"take%2:38:10::","synset_id":"bring.v.01"},{"index":8,"sense_key":"take%2:40:00::","synset_id":"take.v.08"},{"index":9,"sense_key":"take%2:38:02::","synset_id":"take.v.09"},{"index":10,"sense_key":"take%2:31:01::","synset_id":"choose.v.01"},{"index":11,"sense_key":"take%2:40:05::","synset_id":"accept.v.02"},{"index":12,"sense_key":"take%2:41:00::","synset_id":"fill.v.04"},{"index":13,"sense_key":"take%2:31:03::","synset_id":"consider.v.03"},{"index":14,"sense_key":"take%2:42:00::","synset_id":"necessitate.v.01"},{"index":15,"sense_key":"take%2:39:00::","synset_id":"take.v.15"},{"index":16,"sense_key":"take%2:32:02::","synset_id":"film.v.01"},{"index":17,"sense_key":"take%2:30:00::","synset_id":"remove.v.01"},{"index":18,"sense_key":"take%2:34:00::","synset_id":"consume.v.02"},{"index":19,"sense_key":"take%2:31:09::","synset_id":"take.v.19"},{"index":20,"sense_key":"take%2:40:15::","synset_id":"take.v.20"},{"index":21,"sense_key":"take%2:40:01::","synset_id":"take.v.21"},{"index":22,"sense_key":"take%2:38:00::","synset_id":"assume.v.05"},{"index":23,"sense_key":"take%2:40:07::","synset_id":"accept.v.05"},{"index":24,"sense_key":"take%2:30:08::","synset_id":"take.v.24"},{"index":25,"sense_key":"take%2:31:02::","synset_id":"learn.v.04"},{"index":26,"sense_key":"take%2:32:00::","synset_id":"claim.v.05"},{"index":27,"sense_key":"take%2:38:05::","synset_id":"take.v.27"},{"index":28,"sense_key":"take%2:33:09::","synset_id":"aim.v.01"},{"index":29,"sense_key":"take%2:30:09::","synset_id":"take.v.29"},{"index":30,"sense_key":"take%2:42:10::","synset_id":"carry.v.02"},{"index":31,"sense_key":"take%2:40:03::","synset_id":"lease.v.04"},{"index":32,"sense_key":"take%2:40:04::","synset_id":"subscribe.v.05"},{"index":33,"sense_key":"take%2:40:02::","synset_id":"take.v.33"},{"index":34,"sense_key":"take%2:40:08::","synset_id":"take.v.34"},{"index":35,"sense_key":"take%2:35:14::","synset_id":"take.v.35"},{"index":36,"sense_key":"take%2:32:09::","synset_id":"claim.v.04"},{"index":37,"sense_key":"take%2:42:03::","synset_id":"accept.v.08"},{"index":38,"sense_key":"take%2:42:15::","synset_id":"contain.v.05"},{"index":39,"sense_key":"take%2:41:13::","synset_id":"take.v.39"},{"index":40,"sense_key":"take%2:38:11::","synset_id":"drive.v.16"},{"index":41,"sense_key":"take%2:33:08::","synset_id":"take.v.41"},{"index":42,"sense_key":"take%2:29:08::","synset_id":"contract.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"take%2:40:06::","call_ids":["senseval3.d002.s132.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"take%2:40:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2934,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.005988,"input_uncached_usd":0.005868,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9650367919821292}
{"item_id":"senseval3.d002.s132.t003","gold_sense_keys":["second%1:28:00::"],"candidates":[{"index":1,"sense_key":"second%1:28:00::","synset_id":"second.n.01"},{"index":2,"sense_key":"second%1:28:01::","synset_id":"moment.n.02"},{"index":3,"sense_key":"second%1:04:00::","synset_id":"second_base.n.02"},{"index":4,"sense_key":"second%1:28:02::","synset_id":"moment.n.01"},{"index":5,"sense_key":"second%1:24:00::","synset_id":"second.n.05"},{"index":6,"sense_key":"second%1:23:00::","synset_id":"second.n.06"},{"index":7,"sense_key":"second%1:18:00::","synset_id":"second.n.07"},{"index":8,"sense_key":"second%1:10:00::","synset_id":"second.n.08"},{"index":9,"sense_key":"second%1:06:00::","synset_id":"second_gear.n.01"},{"index":10,"sense_key":"second%1:06:01::","synset_id":"irregular.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"second%1:28:00::","call_ids":["senseval3.d002.s132.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"second%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":914,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001948,"input_uncached_usd":0.001828,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0491012920392677}
{"item_id":"senseval3.d002.s133.t000","gold_sense_keys":["wonder%2:31:00::"],"candidates":[{"index":1,"sense_key":"wonder%2:31:00::","synset_id":"wonder.v.01"},{"index":2,"sense_key":"wonder%2:32:01::","synset_id":"wonder.v.02"},{"index":3,"sense_key":"wonder%2:32:12::","synset_id":"wonder.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"wonder%2:31:00::","call_ids":["senseval3.d002.s133.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"wonder%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":533,"cached_input_tokens":0,"output_tokens":103,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0020959999999999998,"input_uncached_usd":0.0010659999999999999,"input_cached_usd":0.0,"output_usd":0.00103,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2278515840880573}
{"item_id":"senseval3.d002.s133.t001","gold_sense_keys":["able%3:00:00::"],"candidates":[{"index":1,"sense_key":"able%3:00:00::","synset_id":"able.a.01"},{"index":2,"sense_key":"able%5:00:00:competent:00","synset_id":"able.s.01"},{"index":3,"sense_key":"able%5:00:00:capable:00","synset_id":"able.s.02"},{"index":4,"sense_key":"able%5:00:00:fit:01","synset_id":"able.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"able%3:00:00::","call_ids":["senseval3.d002.s133.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"able%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":587,"cached_input_tokens":0,"output_tokens":124,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002414,"input_uncached_usd":0.001174,"input_cached_usd":0.0,"output_usd":0.00124,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2293535409262404}
{"item_id":"senseval3.d002.s133.t002","gold_sense_keys":["relax%2:29:00::"],"candidates":[{"index":1,"sense_key":"relax%2:29:00::","synset_id":"relax.v.01"},{"index":2,"sense_key":"relax%2:30:01::","synset_id":"relax.v.02"},{"index":3,"sense_key":"relax%2:30:02::","synset_id":"loosen.v.07"},{"index":4,"sense_key":"relax%2:29:01::","synset_id":"relax.v.04"},{"index":5,"sense_key":"relax%2:41:02::","synset_id":"relax.v.05"},{"index":6,"sense_key":"relax%2:41:01::","synset_id":"relax.v.06"},{"index":7,"sense_key":"relax%2:41:00::","synset_id":"relax.v.07"},{"index":8,"sense_key":"relax%2:30:05::","synset_id":"slack.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"relax%2:29:00::","call_ids":["senseval3.d002.s133.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"relax%2:29:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":882,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001884,"input_uncached_usd":0.001764,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5017781250644475}
{"item_id":"senseval3.d002.s136.t000","gold_sense_keys":["flesh%1:08:01::"],"candidates":[{"index":1,"sense_key":"flesh%1:08:02::","synset_id":"flesh.n.01"},{"index":2,"sense_key":"flesh%1:08:01::","synset_id":"human_body.n.01"},{"index":3,"sense_key":"flesh%1:20:00::","synset_id":"pulp.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"flesh%1:08:01::","call_ids":["senseval3.d002.s136.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"flesh%1:08:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":520,"cached_input_tokens":0,"output_tokens":106,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0021000000000000003,"input_uncached_usd":0.00104,"input_cached_usd":0.0,"output_usd":0.0010600000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8849502080120146}
{"item_id":"senseval3.d002.s136.t001","gold_sense_keys":["go%2:30:04::"],"candidates":[{"index":1,"sense_key":"go%2:38:00::","synset_id":"travel.v.01"},{"index":2,"sense_key":"go%2:41:00::","synset_id":"go.v.02"},{"index":3,"sense_key":"go%2:38:01::","synset_id":"go.v.03"},{"index":4,"sense_key":"go%2:30:04::","synset_id":"become.v.01"},{"index":5,"sense_key":"go%2:42:06::","synset_id":"go.v.05"},{"index":6,"sense_key":"go%2:42:02::","synset_id":"run.v.05"},{"index":7,"sense_key":"go%2:42:00::","synset_id":"run.v.03"},{"index":8,"sense_key":"go%2:30:01::","synset_id":"proceed.v.04"},{"index":9,"sense_key":"go%2:30:08::","synset_id":"go.v.09"},{"index":10,"sense_key":"go%2:42:07::","synset_id":"go.v.10"},{"index":11,"sense_key":"go%2:39:00::","synset_id":"sound.v.02"},{"index":12,"sense_key":"go%2:35:00::","synset_id":"function.v.01"},{"index":13,"sense_key":"go%2:34:00::","synset_id":"run_low.v.01"},{"index":14,"sense_key":"go%2:30:02::","synset_id":"move.v.13"},{"index":15,"sense_key":"go%2:42:14::","synset_id":"survive.v.01"},{"index":16,"sense_key":"go%2:42:03::","synset_id":"go.v.16"},{"index":17,"sense_key":"go%2:30:00::","synset_id":"die.v.01"},{"index":18,"sense_key":"go%2:42:04::","synset_id":"belong.v.03"},{"index":19,"sense_key":"go%2:42:10::","synset_id":"go.v.19"},{"index":20,"sense_key":"go%2:38:09::","synset_id":"start.v.09"},{"index":21,"sense_key":"go%2:33:00::","synset_id":"move.v.15"},{"index":22,"sense_key":"go%2:42:13::","synset_id":"go.v.22"},{"index":23,"sense_key":"go%2:42:12::","synset_id":"go.v.23"},{"index":24,"sense_key":"go%2:42:11::","synset_id":"blend.v.02"},{"index":25,"sense_key":"go%2:42:08::","synset_id":"go.v.25"},{"index":26,"sense_key":"go%2:42:05::","synset_id":"fit.v.02"},{"index":27,"sense_key":"go%2:35:13::","synset_id":"rifle.v.02"},{"index":28,"sense_key":"go%2:34:09::","synset_id":"go.v.28"},{"index":29,"sense_key":"go%2:31:12::","synset_id":"plump.v.04"},{"index":30,"sense_key":"go%2:30:03::","synset_id":"fail.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"go%2:30:04::","call_ids":["senseval3.d002.s136.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"go%2:30:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2245,"cached_input_tokens":0,"output_tokens":78,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00527,"input_uncached_usd":0.00449,"input_cached_usd":0.0,"output_usd":0.0007800000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.3874022079398856}
{"item_id":"senseval3.d002.s136.t003","gold_sense_keys":["flight%1:04:01::"],"candidates":[{"index":1,"sense_key":"flight%1:14:01::","synset_id":"flight.n.01"},{"index":2,"sense_key":"flight%1:04:00::","synset_id":"flight.n.02"},{"index":3,"sense_key":"flight%1:06:00::","synset_id":"flight.n.03"},{"index":4,"sense_key":"flight%1:04:01::","synset_id":"escape.n.01"},{"index":5,"sense_key":"flight%1:14:00::","synset_id":"flight.n.05"},{"index":6,"sense_key":"flight%1:09:00::","synset_id":"flight.n.06"},{"index":7,"sense_key":"flight%1:19:00::","synset_id":"trajectory.n.01"},{"index":8,"sense_key":"flight%1:14:02::","synset_id":"flight.n.08"},{"index":9,"sense_key":"flight%1:04:02::","synset_id":"flight.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"flight%1:04:01::","call_ids":["senseval3.d002.s136.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"flight%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":774,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002168,"input_uncached_usd":0.001548,"input_cached_usd":0.0,"output_usd":0.00062,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5553953750059009}
{"item_id":"senseval3.d002.s136.t004","gold_sense_keys":["fight%1:04:02::"],"candidates":[{"index":1,"sense_key":"fight%1:04:01::","synset_id":"battle.n.01"},{"index":2,"sense_key":"fight%1:04:02::","synset_id":"fight.n.02"},{"index":3,"sense_key":"fight%1:07:00::","synset_id":"competitiveness.n.01"},{"index":4,"sense_key":"fight%1:10:00::","synset_id":"fight.n.04"},{"index":5,"sense_key":"fight%1:04:03::","synset_id":"fight.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"fight%1:04:02::","call_ids":["senseval3.d002.s136.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"fight%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":677,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002084,"input_uncached_usd":0.001354,"input_cached_usd":0.0,"output_usd":0.0007300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8091976250289008}
{"item_id":"senseval3.d002.s137.t000","gold_sense_keys":["nausea%1:26:00::"],"candidates":[{"index":1,"sense_key":"nausea%1:26:00::","synset_id":"nausea.n.01"},{"index":2,"sense_key":"nausea%1:12:00::","synset_id":"nausea.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"nausea%1:26:00::","call_ids":["senseval3.d002.s137.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"nausea%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":406,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001422,"input_uncached_usd":0.000812,"input_cached_usd":0.0,"output_usd":0.0006100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2967899170471355}
{"item_id":"senseval3.d002.s137.t001","gold_sense_keys":["seem%2:39:01::"],"candidates":[{"index":1,"sense_key":"seem%2:39:00::","synset_id":"look.v.02"},{"index":2,"sense_key":"seem%2:39:01::","synset_id":"appear.v.04"},{"index":3,"sense_key":"seem%2:42:00::","synset_id":"seem.v.03"},{"index":4,"sense_key":"seem%2:39:02::","synset_id":"seem.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"seem%2:39:00::","call_ids":["senseval3.d002.s137.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"seem%2:39:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":583,"cached_input_tokens":0,"output_tokens":169,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002856,"input_uncached_usd":0.001166,"input_cached_usd":0.0,"output_usd":0.00169,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.365105791017413}
{"item_id":"senseval3.d002.s137.t002","gold_sense_keys":["commonplace%5:00:02:ordinary:00"],"candidates":[{"index":1,"sense_key":"commonplace%5:00:02:ordinary:00","synset_id":"commonplace.s.01"},{"index":2,"sense_key":"commonplace%5:00:00:unexciting:00","synset_id":"commonplace.s.02"},{"index":3,"sense_key":"commonplace%5:00:00:unoriginal:00","synset_id":"banal.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"commonplace%5:00:02:ordinary:00","call_ids":["senseval3.d002.s137.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"commonplace%5:00:02:ordinary:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":584,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001288,"input_uncached_usd":0.001168,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3466999999945983}
{"item_id":"senseval3.d002.s137.t003","gold_sense_keys":["symptom%1:26:00::"],"candidates":[{"index":1,"sense_key":"symptom%1:26:00::","synset_id":"symptom.n.01"},{"index":2,"sense_key":"symptom%1:10:00::","synset_id":"symptom.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"symptom%1:10:00::","call_ids":["senseval3.d002.s137.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"symptom%1:10:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":429,"cached_input_tokens":0,"output_tokens":223,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003088,"input_uncached_usd":0.0008579999999999999,"input_cached_usd":0.0,"output_usd":0.00223,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.242529458948411}
{"item_id":"senseval3.d002.s139.t000","gold_sense_keys":["walk%2:38:00::"],"candidates":[{"index":1,"sense_key":"walk%2:38:00::","synset_id":"walk.v.01"},{"index":2,"sense_key":"walk%2:38:01::","synset_id":"walk.v.02"},{"index":3,"sense_key":"walk%2:33:00::","synset_id":"walk.v.03"},{"index":4,"sense_key":"walk%2:38:03::","synset_id":"walk.v.04"},{"index":5,"sense_key":"walk%2:33:01::","synset_id":"walk.v.05"},{"index":6,"sense_key":"walk%2:41:00::","synset_id":"walk.v.06"},{"index":7,"sense_key":"walk%2:41:01::","synset_id":"walk.v.07"},{"index":8,"sense_key":"walk%2:38:02::","synset_id":"walk.v.08"},{"index":9,"sense_key":"walk%2:38:05::","synset_id":"walk.v.09"},{"index":10,"sense_key":"walk%2:38:04::","synset_id":"walk.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"walk%2:38:00::","call_ids":["senseval3.d002.s139.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"walk%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":867,"cached_input_tokens":0,"output_tokens":164,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0033740000000000003,"input_uncached_usd":0.0017339999999999999,"input_cached_usd":0.0,"output_usd":0.0016400000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.684563749935478}
{"item_id":"senseval3.d002.s139.t002","gold_sense_keys":["minute%1:28:00::"],"candidates":[{"index":1,"sense_key":"minute%1:28:00::","synset_id":"minute.n.01"},{"index":2,"sense_key":"minute%1:28:01::","synset_id":"moment.n.02"},{"index":3,"sense_key":"minute%1:28:02::","synset_id":"moment.n.01"},{"index":4,"sense_key":"minute%1:23:00::","synset_id":"minute.n.04"},{"index":5,"sense_key":"minute%1:10:00::","synset_id":"minute.n.05"},{"index":6,"sense_key":"minute%1:07:00::","synset_id":"hour.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"minute%1:28:00::","call_ids":["senseval3.d002.s139.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"minute%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":741,"cached_input_tokens":0,"output_tokens":97,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002452,"input_uncached_usd":0.001482,"input_cached_usd":0.0,"output_usd":0.00097,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5455871670274064}
{"item_id":"senseval3.d002.s139.t003","gold_sense_keys":["morning%1:28:00::"],"candidates":[{"index":1,"sense_key":"morning%1:28:00::","synset_id":"morning.n.01"},{"index":2,"sense_key":"morning%1:10:00::","synset_id":"good_morning.n.01"},{"index":3,"sense_key":"morning%1:28:01::","synset_id":"dawn.n.01"},{"index":4,"sense_key":"morning%1:11:00::","synset_id":"dawn.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"morning%1:28:00::","call_ids":["senseval3.d002.s139.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"morning%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":624,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001368,"input_uncached_usd":0.001248,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0634972079424188}
{"item_id":"senseval3.d002.s140.t000","gold_sense_keys":["next%5:00:00:close:01"],"candidates":[{"index":1,"sense_key":"next%5:00:00:succeeding:00","synset_id":"following.s.02"},{"index":2,"sense_key":"next%5:00:00:close:01","synset_id":"adjacent.s.01"},{"index":3,"sense_key":"next%5:00:00:incoming:00","synset_id":"future.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"next%5:00:00:close:01","call_ids":["senseval3.d002.s140.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"next%5:00:00:close:01","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":572,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0017140000000000002,"input_uncached_usd":0.001144,"input_cached_usd":0.0,"output_usd":0.0005700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1834826250560582}
{"item_id":"senseval3.d002.s140.t004","gold_sense_keys":["blue%5:00:00:chromatic:00"],"candidates":[{"index":1,"sense_key":"blue%5:00:00:chromatic:00","synset_id":"blue.s.01"},{"index":2,"sense_key":"blue%5:00:00:northern:02","synset_id":"blue.s.02"},{"index":3,"sense_key":"blue%5:00:00:dejected:00","synset_id":"gloomy.s.02"},{"index":4,"sense_key":"blue%5:00:02:dirty:02","synset_id":"blasphemous.s.02"},{"index":5,"sense_key":"blue%5:00:00:sexy:00","synset_id":"blue.s.05"},{"index":6,"sense_key":"blue%5:00:00:noble:02","synset_id":"aristocratic.s.01"},{"index":7,"sense_key":"blue%5:00:00:nonindulgent:00","synset_id":"blue.s.07"},{"index":8,"sense_key":"blue%5:00:00:depressing:00","synset_id":"blue.s.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"blue%5:00:00:chromatic:00","call_ids":["senseval3.d002.s140.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"blue%5:00:00:chromatic:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1054,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002228,"input_uncached_usd":0.002108,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8507142090238631}
{"item_id":"senseval3.d002.s140.t006","gold_sense_keys":["sit_up%2:38:01::"],"candidates":[{"index":1,"sense_key":"sit_up%2:29:00::","synset_id":"stay_up.v.01"},{"index":2,"sense_key":"sit_up%2:38:01::","synset_id":"sit_up.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"sit_up%2:38:01::","call_ids":["senseval3.d002.s140.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"sit_up%2:38:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":506,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001132,"input_uncached_usd":0.0010119999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.287106208037585}
{"item_id":"senseval3.d002.s140.t007","gold_sense_keys":["say%2:32:15::"],"candidates":[{"index":1,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":6,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":9,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":10,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":11,"sense_key":"say%2:32:09::","synset_id":"say.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"say%2:32:15::","call_ids":["senseval3.d002.s140.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"say%2:32:15::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1042,"cached_input_tokens":0,"output_tokens":85,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002934,"input_uncached_usd":0.002084,"input_cached_usd":0.0,"output_usd":0.0008500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5381031669676304}
{"item_id":"senseval3.d002.s140.t009","gold_sense_keys":["woman%1:18:00::"],"candidates":[{"index":1,"sense_key":"woman%1:18:00::","synset_id":"woman.n.01"},{"index":2,"sense_key":"woman%1:18:01::","synset_id":"woman.n.02"},{"index":3,"sense_key":"woman%1:18:02::","synset_id":"charwoman.n.01"},{"index":4,"sense_key":"woman%1:14:00::","synset_id":"womanhood.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"woman%1:18:00::","call_ids":["senseval3.d002.s140.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"woman%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":667,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001454,"input_uncached_usd":0.001334,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3848250829614699}
{"item_id":"senseval3.d002.s140.t010","gold_sense_keys":["smile%2:29:00::"],"candidates":[{"index":1,"sense_key":"smile%2:29:00::","synset_id":"smile.v.01"},{"index":2,"sense_key":"smile%2:32:00::","synset_id":"smile.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"smile%2:29:00::","call_ids":["senseval3.d002.s140.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"smile%2:29:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":470,"cached_input_tokens":0,"output_tokens":76,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0017000000000000001,"input_uncached_usd":0.00094,"input_cached_usd":0.0,"output_usd":0.00076,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.357100084074773}
{"item_id":"senseval3.d002.s140.t011","gold_sense_keys":["say%2:32:15::"],"candidates":[{"index":1,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":6,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":9,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":10,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":11,"sense_key":"say%2:32:09::","synset_id":"say.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"say%2:32:15::","call_ids":["senseval3.d002.s140.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"say%2:32:15::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1042,"cached_input_tokens":0,"output_tokens":136,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003444,"input_uncached_usd":0.002084,"input_cached_usd":0.0,"output_usd":0.00136,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9272807089146227}
{"item_id":"senseval3.d002.s140.t012","gold_sense_keys":["great%5:00:00:good:01"],"candidates":[{"index":1,"sense_key":"great%5:00:01:large:00","synset_id":"great.s.01"},{"index":2,"sense_key":"great%5:00:01:important:00","synset_id":"great.s.02"},{"index":3,"sense_key":"great%5:00:01:extraordinary:00","synset_id":"great.s.03"},{"index":4,"sense_key":"great%5:00:00:good:01","synset_id":"bang-up.s.01"},{"index":5,"sense_key":"great%5:00:00:uppercase:00","synset_id":"capital.s.03"},{"index":6,"sense_key":"great%5:00:00:pregnant:00","synset_id":"big.s.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"great%5:00:00:good:01","call_ids":["senseval3.d002.s140.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"great%5:00:00:good:01","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":792,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001704,"input_uncached_usd":0.001584,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.979376208037138}
{"item_id":"senseval3.d002.s140.t013","gold_sense_keys":["alive%3:00:01::"],"candidates":[{"index":1,"sense_key":"alive%3:00:01::","synset_id":"alive.a.01"},{"index":2,"sense_key":"alive%5:00:00:lively:00","synset_id":"alive.s.01"},{"index":3,"sense_key":"alive%3:00:04::","synset_id":"animated.a.01"},{"index":4,"sense_key":"alive%5:00:00:sensitive:02","synset_id":"alive.s.02"},{"index":5,"sense_key":"alive%5:00:00:existent:00","synset_id":"active.s.03"},{"index":6,"sense_key":"alive%5:00:00:aware:00","synset_id":"alert.s.02"},{"index":7,"sense_key":"alive%5:00:00:active:05","synset_id":"alive.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"alive%3:00:01::","call_ids":["senseval3.d002.s140.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"alive%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":792,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002214,"input_uncached_usd":0.001584,"input_cached_usd":0.0,"output_usd":0.00063,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3703839590307325}
{"item_id":"senseval3.d002.s141.t000","gold_sense_keys":["agree%2:32:00::"],"candidates":[{"index":1,"sense_key":"agree%2:32:00::","synset_id":"agree.v.01"},{"index":2,"sense_key":"agree%2:32:01::","synset_id":"agree.v.02"},{"index":3,"sense_key":"agree%2:42:00::","synset_id":"match.v.01"},{"index":4,"sense_key":"agree%2:42:01::","synset_id":"harmonize.v.01"},{"index":5,"sense_key":"agree%2:42:03::","synset_id":"agree.v.05"},{"index":6,"sense_key":"agree%2:40:02::","synset_id":"agree.v.06"},{"index":7,"sense_key":"agree%2:32:04::","synset_id":"agree.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"agree%2:32:00::","call_ids":["senseval3.d002.s141.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"agree%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":849,"cached_input_tokens":0,"output_tokens":76,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002458,"input_uncached_usd":0.0016979999999999999,"input_cached_usd":0.0,"output_usd":0.00076,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9940135420765728}
{"item_id":"semeval2013.d000.s000.t000","gold_sense_keys":["group%1:03:00::"],"candidates":[{"index":1,"sense_key":"group%1:03:00::","synset_id":"group.n.01"},{"index":2,"sense_key":"group%1:27:00::","synset_id":"group.n.02"},{"index":3,"sense_key":"group%1:09:00::","synset_id":"group.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"group%1:03:00::","call_ids":["semeval2013.d000.s000.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"group%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":509,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0011380000000000001,"input_uncached_usd":0.001018,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.385209457948804}
{"item_id":"semeval2013.d000.s000.t001","gold_sense_keys":["plan%1:09:00::"],"candidates":[{"index":1,"sense_key":"plan%1:09:00::","synset_id":"plan.n.01"},{"index":2,"sense_key":"plan%1:09:01::","synset_id":"design.n.02"},{"index":3,"sense_key":"plan%1:06:00::","synset_id":"plan.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"plan%1:09:00::","call_ids":["semeval2013.d000.s000.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"plan%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":553,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001226,"input_uncached_usd":0.001106,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4185187919065356}
{"item_id":"semeval2013.d000.s000.t002","gold_sense_keys":["emission%1:04:00::","emission%1:27:00::"],"candidates":[{"index":1,"sense_key":"emission%1:04:00::","synset_id":"emission.n.01"},{"index":2,"sense_key":"emission%1:27:00::","synset_id":"discharge.n.03"},{"index":3,"sense_key":"emission%1:22:00::","synset_id":"emission.n.03"},{"index":4,"sense_key":"emission%1:22:01::","synset_id":"discharge.n.04"},{"index":5,"sense_key":"emission%1:11:00::","synset_id":"emission.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"emission%1:27:00::","call_ids":["semeval2013.d000.s000.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"emission%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":574,"cached_input_tokens":0,"output_tokens":84,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001988,"input_uncached_usd":0.001148,"input_cached_usd":0.0,"output_usd":0.00084,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5497157500358298}
{"item_id":"semeval2013.d000.s001.t000","gold_sense_keys":["climate%1:26:00::"],"candidates":[{"index":1,"sense_key":"climate%1:26:00::","synset_id":"climate.n.01"},{"index":2,"sense_key":"climate%1:26:01::","synset_id":"climate.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"climate%1:26:00::","call_ids":["semeval2013.d000.s001.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"climate%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":541,"cached_input_tokens":0,"output_tokens":76,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0018419999999999999,"input_uncached_usd":0.0010819999999999998,"input_cached_usd":0.0,"output_usd":0.00076,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.613152917008847}
{"item_id":"semeval2013.d000.s001.t001","gold_sense_keys":["conference%1:10:00::","conference%1:14:00::"],"candidates":[{"index":1,"sense_key":"conference%1:14:00::","synset_id":"conference.n.01"},{"index":2,"sense_key":"conference%1:14:01::","synset_id":"league.n.01"},{"index":3,"sense_key":"conference%1:10:00::","synset_id":"conference.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"conference%1:14:00::","call_ids":["semeval2013.d000.s001.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"conference%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":554,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0012280000000000001,"input_uncached_usd":0.001108,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1505597500363365}
{"item_id":"semeval2013.d000.s001.t004","gold_sense_keys":["focus%1:09:00::"],"candidates":[{"index":1,"sense_key":"focus%1:09:00::","synset_id":"focus.n.01"},{"index":2,"sense_key":"focus%1:07:01::","synset_id":"focus.n.02"},{"index":3,"sense_key":"focus%1:07:03::","synset_id":"focus.n.03"},{"index":4,"sense_key":"focus%1:26:00::","synset_id":"focus.n.04"},{"index":5,"sense_key":"focus%1:26:01::","synset_id":"stress.n.03"},{"index":6,"sense_key":"focus%1:19:00::","synset_id":"focus.n.06"},{"index":7,"sense_key":"focus%1:15:00::","synset_id":"focus.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"focus%1:09:00::","call_ids":["semeval2013.d000.s001.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"focus%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":814,"cached_input_tokens":0,"output_tokens":145,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003078,"input_uncached_usd":0.0016279999999999999,"input_cached_usd":0.0,"output_usd":0.0014500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0254170410335064}
{"item_id":"semeval2013.d000.s001.t007","gold_sense_keys":["document%1:10:00::"],"candidates":[{"index":1,"sense_key":"document%1:10:00::","synset_id":"document.n.01"},{"index":2,"sense_key":"document%1:06:00::","synset_id":"document.n.02"},{"index":3,"sense_key":"document%1:21:00::","synset_id":"document.n.03"},{"index":4,"sense_key":"document%1:10:01::","synset_id":"text_file.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"document%1:10:00::","call_ids":["semeval2013.d000.s001.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"document%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":591,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001302,"input_uncached_usd":0.0011819999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.760919834021479}
{"item_id":"semeval2013.d000.s001.t008","gold_sense_keys":["reduction%1:04:00::"],"candidates":[{"index":1,"sense_key":"reduction%1:04:00::","synset_id":"decrease.n.04"},{"index":2,"sense_key":"reduction%1:22:00::","synset_id":"reduction.n.02"},{"index":3,"sense_key":"reduction%1:04:02::","synset_id":"reduction.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"reduction%1:04:00::","call_ids":["semeval2013.d000.s001.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"reduction%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":583,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001286,"input_uncached_usd":0.001166,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9885982080595568}
{"item_id":"semeval2013.d000.s001.t009","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["semeval2013.d000.s001.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":660,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00144,"input_uncached_usd":0.00132,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4464966669911519}
{"item_id":"semeval2013.d000.s001.t010","gold_sense_keys":["burden%1:09:01::"],"candidates":[{"index":1,"sense_key":"burden%1:09:01::","synset_id":"burden.n.01"},{"index":2,"sense_key":"burden%1:06:01::","synset_id":"load.n.01"},{"index":3,"sense_key":"burden%1:10:00::","synset_id":"effect.n.04"},{"index":4,"sense_key":"burden%1:09:00::","synset_id":"burden.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"burden%1:09:01::","call_ids":["semeval2013.d000.s001.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"burden%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":612,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0013440000000000001,"input_uncached_usd":0.001224,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8549420409835875}
{"item_id":"semeval2013.d000.s001.t011","gold_sense_keys":["term%1:28:00::"],"candidates":[{"index":1,"sense_key":"term%1:10:00::","synset_id":"term.n.01"},{"index":2,"sense_key":"term%1:28:00::","synset_id":"term.n.02"},{"index":3,"sense_key":"term%1:10:02::","synset_id":"condition.n.07"},{"index":4,"sense_key":"term%1:09:00::","synset_id":"term.n.04"},{"index":5,"sense_key":"term%1:10:01::","synset_id":"term.n.05"},{"index":6,"sense_key":"term%1:28:01::","synset_id":"term.n.06"},{"index":7,"sense_key":"term%1:06:00::","synset_id":"terminus.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"term%1:28:00::","call_ids":["semeval2013.d000.s001.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"term%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":822,"cached_input_tokens":0,"output_tokens":38,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0020239999999999998,"input_uncached_usd":0.0016439999999999998,"input_cached_usd":0.0,"output_usd":0.00038,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5172790000215173}
{"item_id":"semeval2013.d000.s002.t000","gold_sense_keys":["text%1:10:00::","text%1:10:01::"],"candidates":[{"index":1,"sense_key":"text%1:10:00::","synset_id":"text.n.01"},{"index":2,"sense_key":"text%1:10:03::","synset_id":"text.n.02"},{"index":3,"sense_key":"text%1:10:02::","synset_id":"textbook.n.01"},{"index":4,"sense_key":"text%1:10:01::","synset_id":"text.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"text%1:10:00::","call_ids":["semeval2013.d000.s002.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"text%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":735,"cached_input_tokens":0,"output_tokens":121,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00268,"input_uncached_usd":0.00147,"input_cached_usd":0.0,"output_usd":0.0012100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9664594581117854}
{"item_id":"semeval2013.d000.s002.t001","gold_sense_keys":["basis%1:09:00::"],"candidates":[{"index":1,"sense_key":"basis%1:24:00::","synset_id":"footing.n.02"},{"index":2,"sense_key":"basis%1:09:00::","synset_id":"basis.n.02"},{"index":3,"sense_key":"basis%1:24:01::","synset_id":"basis.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"basis%1:09:00::","call_ids":["semeval2013.d000.s002.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"basis%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":664,"cached_input_tokens":0,"output_tokens":124,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002568,"input_uncached_usd":0.001328,"input_cached_usd":0.0,"output_usd":0.00124,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7361764159286395}
{"item_id":"semeval2013.d000.s002.t002","gold_sense_keys":["deal%1:10:00::"],"candidates":[{"index":1,"sense_key":"deal%1:04:02::","synset_id":"deal.n.01"},{"index":2,"sense_key":"deal%1:10:00::","synset_id":"bargain.n.01"},{"index":3,"sense_key":"deal%1:23:00::","synset_id":"batch.n.02"},{"index":4,"sense_key":"deal%1:27:00::","synset_id":"deal.n.04"},{"index":5,"sense_key":"deal%1:27:01::","synset_id":"softwood.n.01"},{"index":6,"sense_key":"deal%1:14:00::","synset_id":"hand.n.06"},{"index":7,"sense_key":"deal%1:11:00::","synset_id":"deal.n.07"},{"index":8,"sense_key":"deal%1:04:00::","synset_id":"deal.n.08"},{"index":9,"sense_key":"deal%1:04:01::","synset_id":"deal.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"deal%1:10:00::","call_ids":["semeval2013.d000.s002.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"deal%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1004,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0021279999999999997,"input_uncached_usd":0.002008,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.831225625006482}
{"item_id":"semeval2013.d000.s002.t004","gold_sense_keys":["obstacle%1:09:00::"],"candidates":[{"index":1,"sense_key":"obstacle%1:09:00::","synset_id":"obstacle.n.01"},{"index":2,"sense_key":"obstacle%1:06:00::","synset_id":"obstacle.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"obstacle%1:09:00::","call_ids":["semeval2013.d000.s002.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"obstacle%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":600,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00181,"input_uncached_usd":0.0012,"input_cached_usd":0.0,"output_usd":0.0006100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9086495420197025}
{"item_id":"semeval2013.d000.s002.t005","gold_sense_keys":["path%1:04:00::"],"candidates":[{"index":1,"sense_key":"path%1:04:00::","synset_id":"way.n.05"},{"index":2,"sense_key":"path%1:06:00::","synset_id":"path.n.02"},{"index":3,"sense_key":"path%1:15:00::","synset_id":"path.n.03"},{"index":4,"sense_key":"path%1:17:00::","synset_id":"path.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"path%1:04:00::","call_ids":["semeval2013.d000.s002.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"path%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":641,"cached_input_tokens":0,"output_tokens":116,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0024419999999999997,"input_uncached_usd":0.001282,"input_cached_usd":0.0,"output_usd":0.00116,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7441172909457237}
{"item_id":"semeval2013.d000.s003.t000","gold_sense_keys":["advance%1:11:01::"],"candidates":[{"index":1,"sense_key":"advance%1:11:00::","synset_id":"progress.n.03"},{"index":2,"sense_key":"advance%1:11:01::","synset_id":"improvement.n.01"},{"index":3,"sense_key":"advance%1:10:00::","synset_id":"overture.n.03"},{"index":4,"sense_key":"advance%1:04:00::","synset_id":"progress.n.02"},{"index":5,"sense_key":"advance%1:21:00::","synset_id":"advance.n.05"},{"index":6,"sense_key":"advance%1:04:01::","synset_id":"advance.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"advance%1:11:01::","call_ids":["semeval2013.d000.s003.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"advance%1:11:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":825,"cached_input_tokens":0,"output_tokens":120,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00285,"input_uncached_usd":0.00165,"input_cached_usd":0.0,"output_usd":0.0012000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9508831669809297}
{"item_id":"semeval2013.d000.s003.t001","gold_sense_keys":["negotiation%1:10:00::"],"candidates":[{"index":1,"sense_key":"negotiation%1:10:00::","synset_id":"negotiation.n.01"},{"index":2,"sense_key":"negotiation%1:04:00::","synset_id":"negotiation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"negotiation%1:10:00::","call_ids":["semeval2013.d000.s003.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"negotiation%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":629,"cached_input_tokens":0,"output_tokens":101,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002268,"input_uncached_usd":0.001258,"input_cached_usd":0.0,"output_usd":0.00101,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5276645830599591}
{"item_id":"semeval2013.d000.s003.t002","gold_sense_keys":["time%1:28:05::"],"candidates":[{"index":1,"sense_key":"time%1:11:00::","synset_id":"time.n.01"},{"index":2,"sense_key":"time%1:28:05::","synset_id":"time.n.02"},{"index":3,"sense_key":"time%1:28:00::","synset_id":"time.n.03"},{"index":4,"sense_key":"time%1:28:06::","synset_id":"time.n.04"},{"index":5,"sense_key":"time%1:03:00::","synset_id":"time.n.05"},{"index":6,"sense_key":"time%1:11:01::","synset_id":"time.n.06"},{"index":7,"sense_key":"time%1:28:03::","synset_id":"clock_time.n.01"},{"index":8,"sense_key":"time%1:28:04::","synset_id":"fourth_dimension.n.01"},{"index":9,"sense_key":"time%1:07:01::","synset_id":"meter.n.04"},{"index":10,"sense_key":"time%1:28:01::","synset_id":"prison_term.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"time%1:28:05::","call_ids":["semeval2013.d000.s003.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"time%1:28:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1068,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002786,"input_uncached_usd":0.002136,"input_cached_usd":0.0,"output_usd":0.0006500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8484650410246104}
{"item_id":"semeval2013.d000.s003.t004","gold_sense_keys":["leader%1:18:00::"],"candidates":[{"index":1,"sense_key":"leader%1:18:00::","synset_id":"leader.n.01"},{"index":2,"sense_key":"leader%1:06:00::","synset_id":"drawing_card.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"leader%1:18:00::","call_ids":["semeval2013.d000.s003.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"leader%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":594,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0013080000000000001,"input_uncached_usd":0.001188,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0863690419355407}
{"item_id":"semeval2013.d000.s003.t006","gold_sense_keys":["week%1:28:02::"],"candidates":[{"index":1,"sense_key":"week%1:28:00::","synset_id":"week.n.01"},{"index":2,"sense_key":"week%1:28:01::","synset_id":"workweek.n.01"},{"index":3,"sense_key":"week%1:28:02::","synset_id":"week.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"week%1:28:00::","call_ids":["semeval2013.d000.s003.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"week%1:28:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":653,"cached_input_tokens":0,"output_tokens":195,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003256,"input_uncached_usd":0.0013059999999999999,"input_cached_usd":0.0,"output_usd":0.0019500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.505556832998991}
{"item_id":"semeval2013.d000.s004.t001","gold_sense_keys":["text%1:10:00::","text%1:10:01::"],"candidates":[{"index":1,"sense_key":"text%1:10:00::","synset_id":"text.n.01"},{"index":2,"sense_key":"text%1:10:03::","synset_id":"text.n.02"},{"index":3,"sense_key":"text%1:10:02::","synset_id":"textbook.n.01"},{"index":4,"sense_key":"text%1:10:01::","synset_id":"text.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"text%1:10:00::","call_ids":["semeval2013.d000.s004.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"text%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":851,"cached_input_tokens":0,"output_tokens":100,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002702,"input_uncached_usd":0.001702,"input_cached_usd":0.0,"output_usd":0.001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7420586670050398}
{"item_id":"semeval2013.d000.s004.t002","gold_sense_keys":["money%1:21:00::"],"candidates":[{"index":1,"sense_key":"money%1:21:00::","synset_id":"money.n.01"},{"index":2,"sense_key":"money%1:21:02::","synset_id":"money.n.02"},{"index":3,"sense_key":"money%1:21:01::","synset_id":"money.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"money%1:21:00::","call_ids":["semeval2013.d000.s004.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"money%1:21:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":707,"cached_input_tokens":0,"output_tokens":122,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002634,"input_uncached_usd":0.001414,"input_cached_usd":0.0,"output_usd":0.0012200000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9730842909775674}
{"item_id":"semeval2013.d000.s004.t003","gold_sense_keys":["country%1:14:00::"],"candidates":[{"index":1,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":2,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":3,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":4,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":5,"sense_key":"country%1:15:02::","synset_id":"area.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"country%1:14:00::","call_ids":["semeval2013.d000.s004.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"country%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":915,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0026,"input_uncached_usd":0.0018299999999999998,"input_cached_usd":0.0,"output_usd":0.0007700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4991700409445912}
{"item_id":"semeval2013.d000.s004.t004","gold_sense_keys":["one%1:09:00::"],"candidates":[{"index":1,"sense_key":"one%1:23:00::","synset_id":"one.n.01"},{"index":2,"sense_key":"one%1:09:00::","synset_id":"one.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"one%1:09:00::","call_ids":["semeval2013.d000.s004.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"one%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":689,"cached_input_tokens":0,"output_tokens":81,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002188,"input_uncached_usd":0.0013779999999999999,"input_cached_usd":0.0,"output_usd":0.0008100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1821549580199644}
{"item_id":"semeval2013.d000.s004.t006","gold_sense_keys":["term%1:28:00::"],"candidates":[{"index":1,"sense_key":"term%1:10:00::","synset_id":"term.n.01"},{"index":2,"sense_key":"term%1:28:00::","synset_id":"term.n.02"},{"index":3,"sense_key":"term%1:10:02::","synset_id":"condition.n.07"},{"index":4,"sense_key":"term%1:09:00::","synset_id":"term.n.04"},{"index":5,"sense_key":"term%1:10:01::","synset_id":"term.n.05"},{"index":6,"sense_key":"term%1:28:01::","synset_id":"term.n.06"},{"index":7,"sense_key":"term%1:06:00::","synset_id":"terminus.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"term%1:28:00::","call_ids":["semeval2013.d000.s004.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"term%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":984,"cached_input_tokens":0,"output_tokens":49,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0024579999999999997,"input_uncached_usd":0.0019679999999999997,"input_cached_usd":0.0,"output_usd":0.0004900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.218214749940671}
{"item_id":"semeval2013.d000.s005.t000","gold_sense_keys":["range%1:14:00::"],"candidates":[{"index":1,"sense_key":"range%1:07:00::","synset_id":"scope.n.01"},{"index":2,"sense_key":"range%1:15:01::","synset_id":"range.n.02"},{"index":3,"sense_key":"range%1:15:02::","synset_id":"range.n.03"},{"index":4,"sense_key":"range%1:17:00::","synset_id":"range.n.04"},{"index":5,"sense_key":"range%1:06:00::","synset_id":"range.n.05"},{"index":6,"sense_key":"range%1:14:00::","synset_id":"range.n.06"},{"index":7,"sense_key":"range%1:14:01::","synset_id":"image.n.07"},{"index":8,"sense_key":"range%1:09:00::","synset_id":"compass.n.03"},{"index":9,"sense_key":"range%1:06:01::","synset_id":"stove.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"range%1:14:00::","call_ids":["semeval2013.d000.s005.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"range%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1275,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0026699999999999996,"input_uncached_usd":0.0025499999999999997,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2118355419952422}
{"item_id":"semeval2013.d000.s005.t001","gold_sense_keys":["option%1:09:00::"],"candidates":[{"index":1,"sense_key":"option%1:21:00::","synset_id":"option.n.01"},{"index":2,"sense_key":"option%1:09:00::","synset_id":"option.n.02"},{"index":3,"sense_key":"option%1:04:00::","synset_id":"choice.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"option%1:09:00::","call_ids":["semeval2013.d000.s005.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"option%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":797,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001714,"input_uncached_usd":0.001594,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9598875420633703}
{"item_id":"semeval2013.d000.s005.t002","gold_sense_keys":["question%1:10:00::","question%1:10:01::"],"candidates":[{"index":1,"sense_key":"question%1:10:00::","synset_id":"question.n.01"},{"index":2,"sense_key":"question%1:10:01::","synset_id":"question.n.02"},{"index":3,"sense_key":"question%1:10:03::","synset_id":"question.n.03"},{"index":4,"sense_key":"question%1:07:00::","synset_id":"doubt.n.02"},{"index":5,"sense_key":"question%1:10:02::","synset_id":"motion.n.05"},{"index":6,"sense_key":"question%1:10:04::","synset_id":"question.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"question%1:10:01::","call_ids":["semeval2013.d000.s005.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"question%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1037,"cached_input_tokens":0,"output_tokens":102,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003094,"input_uncached_usd":0.002074,"input_cached_usd":0.0,"output_usd":0.00102,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.636249707895331}
{"item_id":"semeval2013.d000.s005.t003","gold_sense_keys":["upper_limit%1:23:00::"],"candidates":[{"index":1,"sense_key":"upper_limit%1:23:00::","synset_id":"maximum.n.01"},{"index":2,"sense_key":"upper_limit%1:15:00::","synset_id":"upper_limit.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"upper_limit%1:23:00::","call_ids":["semeval2013.d000.s005.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"upper_limit%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":692,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0015040000000000001,"input_uncached_usd":0.001384,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7652615420520306}
{"item_id":"semeval2013.d000.s005.t004","gold_sense_keys":["temperature%1:07:00::"],"candidates":[{"index":1,"sense_key":"temperature%1:07:00::","synset_id":"temperature.n.01"},{"index":2,"sense_key":"temperature%1:09:00::","synset_id":"temperature.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"temperature%1:07:00::","call_ids":["semeval2013.d000.s005.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"temperature%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":694,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001508,"input_uncached_usd":0.001388,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0268914999905974}
{"item_id":"semeval2013.d000.s005.t005","gold_sense_keys":["rise%1:11:00::"],"candidates":[{"index":1,"sense_key":"rise%1:11:00::","synset_id":"rise.n.01"},{"index":2,"sense_key":"rise%1:04:00::","synset_id":"rise.n.02"},{"index":3,"sense_key":"rise%1:17:00::","synset_id":"ascent.n.01"},{"index":4,"sense_key":"rise%1:11:02::","synset_id":"rise.n.04"},{"index":5,"sense_key":"rise%1:07:02::","synset_id":"raise.n.01"},{"index":6,"sense_key":"rise%1:07:01::","synset_id":"upgrade.n.04"},{"index":7,"sense_key":"rise%1:11:01::","synset_id":"lift.n.04"},{"index":8,"sense_key":"rise%1:11:03::","synset_id":"emanation.n.03"},{"index":9,"sense_key":"rise%1:07:03::","synset_id":"rise.n.09"},{"index":10,"sense_key":"rise%1:04:01::","synset_id":"advance.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"rise%1:11:00::","call_ids":["semeval2013.d000.s005.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"rise%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1202,"cached_input_tokens":0,"output_tokens":151,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003914,"input_uncached_usd":0.002404,"input_cached_usd":0.0,"output_usd":0.00151,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.6546017499640584}
{"item_id":"semeval2013.d000.s006.t000","gold_sense_keys":["lot%1:23:00::"],"candidates":[{"index":1,"sense_key":"lot%1:23:00::","synset_id":"batch.n.02"},{"index":2,"sense_key":"lot%1:15:00::","synset_id":"lot.n.02"},{"index":3,"sense_key":"lot%1:14:00::","synset_id":"set.n.05"},{"index":4,"sense_key":"lot%1:26:00::","synset_id":"fortune.n.04"},{"index":5,"sense_key":"lot%1:06:00::","synset_id":"draw.n.04"},{"index":6,"sense_key":"lot%1:14:01::","synset_id":"bunch.n.03"},{"index":7,"sense_key":"lot%1:18:00::","synset_id":"lot.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"lot%1:23:00::","call_ids":["semeval2013.d000.s006.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"lot%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1156,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0024319999999999997,"input_uncached_usd":0.002312,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1425521669443697}
{"item_id":"semeval2013.d000.s006.t001","gold_sense_keys":["flexibility%1:07:01::"],"candidates":[{"index":1,"sense_key":"flexibility%1:07:02::","synset_id":"flexibility.n.01"},{"index":2,"sense_key":"flexibility%1:07:01::","synset_id":"flexibility.n.02"},{"index":3,"sense_key":"flexibility%1:07:03::","synset_id":"tractability.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"flexibility%1:07:01::","call_ids":["semeval2013.d000.s006.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"flexibility%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":830,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0017800000000000001,"input_uncached_usd":0.00166,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8534810419660062}
{"item_id":"semeval2013.d000.s006.t002","gold_sense_keys":["process%1:04:00::"],"candidates":[{"index":1,"sense_key":"process%1:04:00::","synset_id":"procedure.n.01"},{"index":2,"sense_key":"process%1:09:00::","synset_id":"process.n.02"},{"index":3,"sense_key":"process%1:10:00::","synset_id":"summons.n.03"},{"index":4,"sense_key":"process%1:09:01::","synset_id":"process.n.04"},{"index":5,"sense_key":"process%1:08:00::","synset_id":"process.n.05"},{"index":6,"sense_key":"process%1:03:00::","synset_id":"process.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"process%1:04:00::","call_ids":["semeval2013.d000.s006.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"process%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1100,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00273,"input_uncached_usd":0.0021999999999999997,"input_cached_usd":0.0,"output_usd":0.0005300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6179052500519902}
{"item_id":"semeval2013.d000.s006.t003","gold_sense_keys":["washington%1:15:01::"],"candidates":[{"index":1,"sense_key":"washington%1:15:01::","synset_id":"washington.n.01"},{"index":2,"sense_key":"washington%1:15:00::","synset_id":"washington.n.02"},{"index":3,"sense_key":"washington%1:14:00::","synset_id":"capital.n.06"},{"index":4,"sense_key":"washington%1:18:00::","synset_id":"washington.n.04"},{"index":5,"sense_key":"washington%1:18:01::","synset_id":"washington.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"washington%1:15:01::","call_ids":["semeval2013.d000.s006.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"washington%1:15:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1006,"cached_input_tokens":0,"output_tokens":110,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003112,"input_uncached_usd":0.002012,"input_cached_usd":0.0,"output_usd":0.0011,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6391192500013858}
{"item_id":"semeval2013.d000.s006.t004","gold_sense_keys":["representative%1:18:00::","representative%1:18:02::"],"candidates":[{"index":1,"sense_key":"representative%1:18:00::","synset_id":"representative.n.01"},{"index":2,"sense_key":"representative%1:18:02::","synset_id":"spokesperson.n.01"},{"index":3,"sense_key":"representative%1:18:01::","synset_id":"congressman.n.01"},{"index":4,"sense_key":"representative%1:09:00::","synset_id":"example.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"representative%1:18:02::","call_ids":["semeval2013.d000.s006.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"representative%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":894,"cached_input_tokens":0,"output_tokens":119,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0029779999999999997,"input_uncached_usd":0.0017879999999999999,"input_cached_usd":0.0,"output_usd":0.00119,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.5988779169274494}
{"item_id":"semeval2013.d000.s007.t000","gold_sense_keys":["document%1:10:00::"],"candidates":[{"index":1,"sense_key":"document%1:10:00::","synset_id":"document.n.01"},{"index":2,"sense_key":"document%1:06:00::","synset_id":"document.n.02"},{"index":3,"sense_key":"document%1:21:00::","synset_id":"document.n.03"},{"index":4,"sense_key":"document%1:10:01::","synset_id":"text_file.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"document%1:10:00::","call_ids":["semeval2013.d000.s007.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"document%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":829,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002198,"input_uncached_usd":0.001658,"input_cached_usd":0.0,"output_usd":0.00054,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2861233750591055}
{"item_id":"semeval2013.d000.s007.t001","gold_sense_keys":["negotiation%1:04:00::"],"candidates":[{"index":1,"sense_key":"negotiation%1:10:00::","synset_id":"negotiation.n.01"},{"index":2,"sense_key":"negotiation%1:04:00::","synset_id":"negotiation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"negotiation%1:10:00::","call_ids":["semeval2013.d000.s007.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"negotiation%1:10:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":770,"cached_input_tokens":0,"output_tokens":184,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00338,"input_uncached_usd":0.00154,"input_cached_usd":0.0,"output_usd":0.00184,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1956247079651803}
{"item_id":"semeval2013.d000.s007.t002","gold_sense_keys":["text%1:10:00::","text%1:10:01::"],"candidates":[{"index":1,"sense_key":"text%1:10:00::","synset_id":"text.n.01"},{"index":2,"sense_key":"text%1:10:03::","synset_id":"text.n.02"},{"index":3,"sense_key":"text%1:10:02::","synset_id":"textbook.n.01"},{"index":4,"sense_key":"text%1:10:01::","synset_id":"text.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"text%1:10:00::","call_ids":["semeval2013.d000.s007.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"text%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":927,"cached_input_tokens":0,"output_tokens":89,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002744,"input_uncached_usd":0.001854,"input_cached_usd":0.0,"output_usd":0.0008900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0601151670562103}
{"item_id":"semeval2013.d000.s007.t003","gold_sense_keys":["climate%1:26:00::"],"candidates":[{"index":1,"sense_key":"climate%1:26:00::","synset_id":"climate.n.01"},{"index":2,"sense_key":"climate%1:26:01::","synset_id":"climate.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"climate%1:26:00::","call_ids":["semeval2013.d000.s007.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"climate%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":779,"cached_input_tokens":0,"output_tokens":125,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0028079999999999997,"input_uncached_usd":0.001558,"input_cached_usd":0.0,"output_usd":0.00125,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.583278666017577}
{"item_id":"semeval2013.d000.s007.t004","gold_sense_keys":["official%1:18:01::"],"candidates":[{"index":1,"sense_key":"official%1:18:01::","synset_id":"official.n.01"},{"index":2,"sense_key":"official%1:18:00::","synset_id":"official.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"official%1:18:01::","call_ids":["semeval2013.d000.s007.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"official%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":737,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001594,"input_uncached_usd":0.0014739999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.7730804169550538}
{"item_id":"semeval2013.d000.s007.t005","gold_sense_keys":["picture%1:26:00::"],"candidates":[{"index":1,"sense_key":"picture%1:06:00::","synset_id":"picture.n.01"},{"index":2,"sense_key":"picture%1:06:01::","synset_id":"painting.n.01"},{"index":3,"sense_key":"picture%1:09:00::","synset_id":"mental_picture.n.01"},{"index":4,"sense_key":"picture%1:26:00::","synset_id":"picture.n.04"},{"index":5,"sense_key":"picture%1:10:00::","synset_id":"picture.n.05"},{"index":6,"sense_key":"picture%1:10:02::","synset_id":"movie.n.01"},{"index":7,"sense_key":"picture%1:10:01::","synset_id":"video.n.01"},{"index":8,"sense_key":"picture%1:10:03::","synset_id":"word_picture.n.01"},{"index":9,"sense_key":"picture%1:09:01::","synset_id":"picture.n.09"},{"index":10,"sense_key":"picture%1:06:02::","synset_id":"photograph.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"picture%1:26:00::","call_ids":["semeval2013.d000.s007.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"picture%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1432,"cached_input_tokens":0,"output_tokens":100,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003864,"input_uncached_usd":0.002864,"input_cached_usd":0.0,"output_usd":0.001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.310933667002246}
{"item_id":"semeval2013.d000.s008.t000","gold_sense_keys":["outline%1:09:00::"],"candidates":[{"index":1,"sense_key":"outline%1:15:00::","synset_id":"outline.n.01"},{"index":2,"sense_key":"outline%1:10:00::","synset_id":"outline.n.02"},{"index":3,"sense_key":"outline%1:09:00::","synset_id":"outline.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"outline%1:09:00::","call_ids":["semeval2013.d000.s008.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"outline%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":758,"cached_input_tokens":0,"output_tokens":132,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002836,"input_uncached_usd":0.001516,"input_cached_usd":0.0,"output_usd":0.0013200000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.304273249930702}
{"item_id":"semeval2013.d000.s008.t001","gold_sense_keys":["deal%1:10:00::"],"candidates":[{"index":1,"sense_key":"deal%1:04:02::","synset_id":"deal.n.01"},{"index":2,"sense_key":"deal%1:10:00::","synset_id":"bargain.n.01"},{"index":3,"sense_key":"deal%1:23:00::","synset_id":"batch.n.02"},{"index":4,"sense_key":"deal%1:27:00::","synset_id":"deal.n.04"},{"index":5,"sense_key":"deal%1:27:01::","synset_id":"softwood.n.01"},{"index":6,"sense_key":"deal%1:14:00::","synset_id":"hand.n.06"},{"index":7,"sense_key":"deal%1:11:00::","synset_id":"deal.n.07"},{"index":8,"sense_key":"deal%1:04:00::","synset_id":"deal.n.08"},{"index":9,"sense_key":"deal%1:04:01::","synset_id":"deal.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"deal%1:10:00::","call_ids":["semeval2013.d000.s008.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"deal%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1170,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00246,"input_uncached_usd":0.00234,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3725021660793573}
{"item_id":"semeval2013.d000.s008.t002","gold_sense_keys":["emission%1:04:00::","emission%1:27:00::"],"candidates":[{"index":1,"sense_key":"emission%1:04:00::","synset_id":"emission.n.01"},{"index":2,"sense_key":"emission%1:27:00::","synset_id":"discharge.n.03"},{"index":3,"sense_key":"emission%1:22:00::","synset_id":"emission.n.03"},{"index":4,"sense_key":"emission%1:22:01::","synset_id":"discharge.n.04"},{"index":5,"sense_key":"emission%1:11:00::","synset_id":"emission.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"emission%1:27:00::","call_ids":["semeval2013.d000.s008.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"emission%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":839,"cached_input_tokens":0,"output_tokens":130,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002978,"input_uncached_usd":0.001678,"input_cached_usd":0.0,"output_usd":0.0013000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2250162078998983}
{"item_id":"semeval2013.d000.s008.t004","gold_sense_keys":["level%1:07:00::"],"candidates":[{"index":1,"sense_key":"level%1:07:00::","synset_id":"degree.n.01"},{"index":2,"sense_key":"level%1:26:01::","synset_id":"grade.n.02"},{"index":3,"sense_key":"level%1:26:00::","synset_id":"degree.n.02"},{"index":4,"sense_key":"level%1:07:01::","synset_id":"level.n.04"},{"index":5,"sense_key":"level%1:06:00::","synset_id":"level.n.05"},{"index":6,"sense_key":"level%1:06:03::","synset_id":"horizontal_surface.n.01"},{"index":7,"sense_key":"level%1:09:00::","synset_id":"level.n.07"},{"index":8,"sense_key":"level%1:06:01::","synset_id":"floor.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"level%1:07:00::","call_ids":["semeval2013.d000.s008.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"level%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1136,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002822,"input_uncached_usd":0.0022719999999999997,"input_cached_usd":0.0,"output_usd":0.00055,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.691757041029632}
{"item_id":"semeval2013.d000.s008.t006","gold_sense_keys":["period%1:28:00::"],"candidates":[{"index":1,"sense_key":"period%1:28:00::","synset_id":"time_period.n.01"},{"index":2,"sense_key":"period%1:28:02::","synset_id":"period.n.02"},{"index":3,"sense_key":"period%1:28:07::","synset_id":"period.n.03"},{"index":4,"sense_key":"period%1:28:03::","synset_id":"period.n.04"},{"index":5,"sense_key":"period%1:28:04::","synset_id":"period.n.05"},{"index":6,"sense_key":"period%1:22:00::","synset_id":"menstruation.n.01"},{"index":7,"sense_key":"period%1:10:00::","synset_id":"period.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"period%1:28:00::","call_ids":["semeval2013.d000.s008.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"period%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1117,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0023539999999999998,"input_uncached_usd":0.002234,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9657658749492839}
{"item_id":"semeval2013.d000.s009.t000","gold_sense_keys":["country%1:14:00::"],"candidates":[{"index":1,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":2,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":3,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":4,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":5,"sense_key":"country%1:15:02::","synset_id":"area.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"country%1:14:00::","call_ids":["semeval2013.d000.s009.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"country%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":997,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002764,"input_uncached_usd":0.001994,"input_cached_usd":0.0,"output_usd":0.0007700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.642643541097641}
{"item_id":"semeval2013.d000.s009.t001","gold_sense_keys":["emission%1:04:00::","emission%1:27:00::"],"candidates":[{"index":1,"sense_key":"emission%1:04:00::","synset_id":"emission.n.01"},{"index":2,"sense_key":"emission%1:27:00::","synset_id":"discharge.n.03"},{"index":3,"sense_key":"emission%1:22:00::","synset_id":"emission.n.03"},{"index":4,"sense_key":"emission%1:22:01::","synset_id":"discharge.n.04"},{"index":5,"sense_key":"emission%1:11:00::","synset_id":"emission.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"emission%1:27:00::","call_ids":["semeval2013.d000.s009.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"emission%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":871,"cached_input_tokens":0,"output_tokens":116,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002902,"input_uncached_usd":0.0017419999999999998,"input_cached_usd":0.0,"output_usd":0.00116,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9399555829586461}
{"item_id":"semeval2013.d000.s010.t004","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["semeval2013.d000.s010.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":886,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001892,"input_uncached_usd":0.001772,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0616042909678072}
{"item_id":"semeval2013.d000.s010.t005","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["semeval2013.d000.s010.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":886,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001892,"input_uncached_usd":0.001772,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8820962500758469}
{"item_id":"semeval2013.d000.s010.t006","gold_sense_keys":["country%1:14:00::"],"candidates":[{"index":1,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":2,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":3,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":4,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":5,"sense_key":"country%1:15:02::","synset_id":"area.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"country%1:14:00::","call_ids":["semeval2013.d000.s010.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"country%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":979,"cached_input_tokens":0,"output_tokens":151,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003468,"input_uncached_usd":0.001958,"input_cached_usd":0.0,"output_usd":0.00151,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.924051458016038}
{"item_id":"semeval2013.d000.s010.t007","gold_sense_keys":["impact%1:19:00::"],"candidates":[{"index":1,"sense_key":"impact%1:11:00::","synset_id":"impact.n.01"},{"index":2,"sense_key":"impact%1:19:00::","synset_id":"impact.n.02"},{"index":3,"sense_key":"impact%1:04:00::","synset_id":"impingement.n.01"},{"index":4,"sense_key":"impact%1:04:01::","synset_id":"shock.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"impact%1:19:00::","call_ids":["semeval2013.d000.s010.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"impact%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":850,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00182,"input_uncached_usd":0.0017,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.208340207929723}
{"item_id":"semeval2013.d000.s010.t009","gold_sense_keys":["flood%1:19:00::"],"candidates":[{"index":1,"sense_key":"flood%1:19:00::","synset_id":"flood.n.01"},{"index":2,"sense_key":"flood%1:23:00::","synset_id":"flood.n.02"},{"index":3,"sense_key":"flood%1:06:00::","synset_id":"flood.n.03"},{"index":4,"sense_key":"flood%1:04:01::","synset_id":"flood.n.04"},{"index":5,"sense_key":"flood%1:04:00::","synset_id":"flood.n.05"},{"index":6,"sense_key":"flood%1:11:00::","synset_id":"flood_tide.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"flood%1:19:00::","call_ids":["semeval2013.d000.s010.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"flood%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1022,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0021639999999999997,"input_uncached_usd":0.002044,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8543277910212055}
{"item_id":"semeval2013.d000.s010.t010","gold_sense_keys":["drought%1:26:00::"],"candidates":[{"index":1,"sense_key":"drought%1:26:00::","synset_id":"drought.n.01"},{"index":2,"sense_key":"drought%1:28:00::","synset_id":"drought.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"drought%1:26:00::","call_ids":["semeval2013.d000.s010.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"drought%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":765,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00165,"input_uncached_usd":0.00153,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8469053750159219}
{"item_id":"semeval2013.d000.s011.t001","gold_sense_keys":["sort%1:09:00::"],"candidates":[{"index":1,"sense_key":"sort%1:09:00::","synset_id":"kind.n.01"},{"index":2,"sense_key":"sort%1:07:00::","synset_id":"sort.n.02"},{"index":3,"sense_key":"sort%1:18:00::","synset_id":"sort.n.03"},{"index":4,"sense_key":"sort%1:22:00::","synset_id":"sort.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"sort%1:09:00::","call_ids":["semeval2013.d000.s011.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"sort%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":901,"cached_input_tokens":0,"output_tokens":94,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002742,"input_uncached_usd":0.001802,"input_cached_usd":0.0,"output_usd":0.0009400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.594247999950312}
{"item_id":"semeval2013.d000.s011.t003","gold_sense_keys":["week%1:28:00::"],"candidates":[{"index":1,"sense_key":"week%1:28:00::","synset_id":"week.n.01"},{"index":2,"sense_key":"week%1:28:01::","synset_id":"workweek.n.01"},{"index":3,"sense_key":"week%1:28:02::","synset_id":"week.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"week%1:28:00::","call_ids":["semeval2013.d000.s011.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"week%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":792,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001704,"input_uncached_usd":0.001584,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4651377080008388}
{"item_id":"semeval2013.d000.s012.t001","gold_sense_keys":["climate%1:26:00::"],"candidates":[{"index":1,"sense_key":"climate%1:26:00::","synset_id":"climate.n.01"},{"index":2,"sense_key":"climate%1:26:01::","synset_id":"climate.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"climate%1:26:00::","call_ids":["semeval2013.d000.s012.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"climate%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":774,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002138,"input_uncached_usd":0.001548,"input_cached_usd":0.0,"output_usd":0.00059,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.510659624938853}
{"item_id":"semeval2013.d000.s012.t002","gold_sense_keys":["envoy%1:18:00::","envoy%1:18:01::"],"candidates":[{"index":1,"sense_key":"envoy%1:18:00::","synset_id":"envoy.n.01"},{"index":2,"sense_key":"envoy%1:18:01::","synset_id":"emissary.n.01"},{"index":3,"sense_key":"envoy%1:10:00::","synset_id":"envoy.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"envoy%1:18:01::","call_ids":["semeval2013.d000.s012.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"envoy%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":798,"cached_input_tokens":0,"output_tokens":113,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002726,"input_uncached_usd":0.001596,"input_cached_usd":0.0,"output_usd":0.0011300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0711651670280844}
{"item_id":"semeval2013.d000.s012.t004","gold_sense_keys":["cut%1:04:05::"],"candidates":[{"index":1,"sense_key":"cut%1:21:00::","synset_id":"cut.n.01"},{"index":2,"sense_key":"cut%1:10:00::","synset_id":"cut.n.02"},{"index":3,"sense_key":"cut%1:06:00::","synset_id":"cut.n.03"},{"index":4,"sense_key":"cut%1:26:01::","synset_id":"cut.n.04"},{"index":5,"sense_key":"cut%1:26:00::","synset_id":"cut.n.05"},{"index":6,"sense_key":"cut%1:13:00::","synset_id":"cut.n.06"},{"index":7,"sense_key":"cut%1:10:02::","synset_id":"stinger.n.02"},{"index":8,"sense_key":"cut%1:10:04::","synset_id":"cut.n.08"},{"index":9,"sense_key":"cut%1:10:01::","synset_id":"deletion.n.03"},{"index":10,"sense_key":"cut%1:09:00::","synset_id":"cut.n.10"},{"index":11,"sense_key":"cut%1:06:01::","synset_id":"cut.n.11"},{"index":12,"sense_key":"cut%1:04:03::","synset_id":"snub.n.02"},{"index":13,"sense_key":"cut%1:04:07::","synset_id":"baseball_swing.n.01"},{"index":14,"sense_key":"cut%1:04:08::","synset_id":"cut.n.14"},{"index":15,"sense_key":"cut%1:04:09::","synset_id":"cut.n.15"},{"index":16,"sense_key":"cut%1:04:02::","synset_id":"cut.n.16"},{"index":17,"sense_key":"cut%1:04:01::","synset_id":"cut.n.17"},{"index":18,"sense_key":"cut%1:04:00::","synset_id":"cut.n.18"},{"index":19,"sense_key":"cut%1:04:05::","synset_id":"cut.n.19"},{"index":20,"sense_key":"cut%1:04:06::","synset_id":"cut.n.20"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":19,"chosen_sense_key":"cut%1:04:05::","call_ids":["semeval2013.d000.s012.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":19,"predicted_sense_key":"cut%1:04:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1832,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0037839999999999996,"input_uncached_usd":0.0036639999999999997,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1931940410286188}
{"item_id":"semeval2013.d000.s012.t005","gold_sense_keys":["emission%1:04:00::","emission%1:27:00::"],"candidates":[{"index":1,"sense_key":"emission%1:04:00::","synset_id":"emission.n.01"},{"index":2,"sense_key":"emission%1:27:00::","synset_id":"discharge.n.03"},{"index":3,"sense_key":"emission%1:22:00::","synset_id":"emission.n.03"},{"index":4,"sense_key":"emission%1:22:01::","synset_id":"discharge.n.04"},{"index":5,"sense_key":"emission%1:11:00::","synset_id":"emission.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"emission%1:27:00::","call_ids":["semeval2013.d000.s012.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"emission%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":860,"cached_input_tokens":0,"output_tokens":149,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00321,"input_uncached_usd":0.00172,"input_cached_usd":0.0,"output_usd":0.0014900000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.5883682910352945}
{"item_id":"semeval2013.d000.s012.t006","gold_sense_keys":["one%1:09:00::"],"candidates":[{"index":1,"sense_key":"one%1:23:00::","synset_id":"one.n.01"},{"index":2,"sense_key":"one%1:09:00::","synset_id":"one.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"one%1:09:00::","call_ids":["semeval2013.d000.s012.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"one%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":760,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00216,"input_uncached_usd":0.0015199999999999999,"input_cached_usd":0.0,"output_usd":0.00064,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.444834041991271}
{"item_id":"semeval2013.d000.s013.t000","gold_sense_keys":["move%1:04:00::"],"candidates":[{"index":1,"sense_key":"move%1:04:00::","synset_id":"move.n.01"},{"index":2,"sense_key":"move%1:04:01::","synset_id":"move.n.02"},{"index":3,"sense_key":"move%1:04:03::","synset_id":"motion.n.03"},{"index":4,"sense_key":"move%1:04:04::","synset_id":"motion.n.06"},{"index":5,"sense_key":"move%1:04:02::","synset_id":"move.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"move%1:04:00::","call_ids":["semeval2013.d000.s013.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"move%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":934,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002498,"input_uncached_usd":0.0018679999999999999,"input_cached_usd":0.0,"output_usd":0.00063,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9238052089931443}
{"item_id":"semeval2013.d000.s013.t002","gold_sense_keys":["china%1:15:00::"],"candidates":[{"index":1,"sense_key":"china%1:15:00::","synset_id":"china.n.01"},{"index":2,"sense_key":"china%1:06:00::","synset_id":"china.n.02"},{"index":3,"sense_key":"china%1:15:01::","synset_id":"taiwan.n.01"},{"index":4,"sense_key":"china%1:06:01::","synset_id":"chinaware.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"china%1:15:00::","call_ids":["semeval2013.d000.s013.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"china%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":884,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001888,"input_uncached_usd":0.0017679999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1832492920802906}
{"item_id":"semeval2013.d000.s013.t003","gold_sense_keys":["bush_administration%1:14:01::"],"candidates":[{"index":1,"sense_key":"bush_administration%1:14:00::","synset_id":"bush_administration.n.01"},{"index":2,"sense_key":"bush_administration%1:14:01::","synset_id":"bush_administration.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bush_administration%1:14:01::","call_ids":["semeval2013.d000.s013.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bush_administration%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":706,"cached_input_tokens":0,"output_tokens":79,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002202,"input_uncached_usd":0.0014119999999999998,"input_cached_usd":0.0,"output_usd":0.00079,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0329327910440043}
{"item_id":"semeval2013.d000.s013.t004","gold_sense_keys":["official%1:18:01::"],"candidates":[{"index":1,"sense_key":"official%1:18:01::","synset_id":"official.n.01"},{"index":2,"sense_key":"official%1:18:00::","synset_id":"official.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"official%1:18:01::","call_ids":["semeval2013.d000.s013.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"official%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":724,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001568,"input_uncached_usd":0.0014479999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0959303750423715}
{"item_id":"semeval2013.d000.s013.t005","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["semeval2013.d000.s013.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":884,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001888,"input_uncached_usd":0.0017679999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5527057910803705}
{"item_id":"semeval2013.d000.s014.t000","gold_sense_keys":["united_states%1:14:00::"],"candidates":[{"index":1,"sense_key":"united_states%1:15:00::","synset_id":"united_states.n.01"},{"index":2,"sense_key":"united_states%1:14:00::","synset_id":"united_states_government.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"united_states%1:14:00::","call_ids":["semeval2013.d000.s014.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"united_states%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":803,"cached_input_tokens":0,"output_tokens":113,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002736,"input_uncached_usd":0.001606,"input_cached_usd":0.0,"output_usd":0.0011300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.37868804205209}
{"item_id":"semeval2013.d000.s014.t001","gold_sense_keys":["deal%1:10:00::"],"candidates":[{"index":1,"sense_key":"deal%1:04:02::","synset_id":"deal.n.01"},{"index":2,"sense_key":"deal%1:10:00::","synset_id":"bargain.n.01"},{"index":3,"sense_key":"deal%1:23:00::","synset_id":"batch.n.02"},{"index":4,"sense_key":"deal%1:27:00::","synset_id":"deal.n.04"},{"index":5,"sense_key":"deal%1:27:01::","synset_id":"softwood.n.01"},{"index":6,"sense_key":"deal%1:14:00::","synset_id":"hand.n.06"},{"index":7,"sense_key":"deal%1:11:00::","synset_id":"deal.n.07"},{"index":8,"sense_key":"deal%1:04:00::","synset_id":"deal.n.08"},{"index":9,"sense_key":"deal%1:04:01::","synset_id":"deal.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"deal%1:10:00::","call_ids":["semeval2013.d000.s014.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"deal%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1184,"cached_input_tokens":0,"output_tokens":75,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003118,"input_uncached_usd":0.002368,"input_cached_usd":0.0,"output_usd":0.00075,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4441165419993922}
{"item_id":"semeval2013.d000.s014.t004","gold_sense_keys":["text%1:10:00::","text%1:10:01::"],"candidates":[{"index":1,"sense_key":"text%1:10:00::","synset_id":"text.n.01"},{"index":2,"sense_key":"text%1:10:03::","synset_id":"text.n.02"},{"index":3,"sense_key":"text%1:10:02::","synset_id":"textbook.n.01"},{"index":4,"sense_key":"text%1:10:01::","synset_id":"text.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"text%1:10:00::","call_ids":["semeval2013.d000.s014.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"text%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":915,"cached_input_tokens":0,"output_tokens":98,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00281,"input_uncached_usd":0.0018299999999999998,"input_cached_usd":0.0,"output_usd":0.0009800000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.556944084004499}
{"item_id":"semeval2013.d000.s014.t005","gold_sense_keys":["cut%1:04:05::"],"candidates":[{"index":1,"sense_key":"cut%1:21:00::","synset_id":"cut.n.01"},{"index":2,"sense_key":"cut%1:10:00::","synset_id":"cut.n.02"},{"index":3,"sense_key":"cut%1:06:00::","synset_id":"cut.n.03"},{"index":4,"sense_key":"cut%1:26:01::","synset_id":"cut.n.04"},{"index":5,"sense_key":"cut%1:26:00::","synset_id":"cut.n.05"},{"index":6,"sense_key":"cut%1:13:00::","synset_id":"cut.n.06"},{"index":7,"sense_key":"cut%1:10:02::","synset_id":"stinger.n.02"},{"index":8,"sense_key":"cut%1:10:04::","synset_id":"cut.n.08"},{"index":9,"sense_key":"cut%1:10:01::","synset_id":"deletion.n.03"},{"index":10,"sense_key":"cut%1:09:00::","synset_id":"cut.n.10"},{"index":11,"sense_key":"cut%1:06:01::","synset_id":"cut.n.11"},{"index":12,"sense_key":"cut%1:04:03::","synset_id":"snub.n.02"},{"index":13,"sense_key":"cut%1:04:07::","synset_id":"baseball_swing.n.01"},{"index":14,"sense_key":"cut%1:04:08::","synset_id":"cut.n.14"},{"index":15,"sense_key":"cut%1:04:09::","synset_id":"cut.n.15"},{"index":16,"sense_key":"cut%1:04:02::","synset_id":"cut.n.16"},{"index":17,"sense_key":"cut%1:04:01::","synset_id":"cut.n.17"},{"index":18,"sense_key":"cut%1:04:00::","synset_id":"cut.n.18"},{"index":19,"sense_key":"cut%1:04:05::","synset_id":"cut.n.19"},{"index":20,"sense_key":"cut%1:04:06::","synset_id":"cut.n.20"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":19,"chosen_sense_key":"cut%1:04:05::","call_ids":["semeval2013.d000.s014.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":19,"predicted_sense_key":"cut%1:04:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1825,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00377,"input_uncached_usd":0.00365,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1549715839792043}
{"item_id":"semeval2013.d000.s014.t006","gold_sense_keys":["observer%1:18:01::"],"candidates":[{"index":1,"sense_key":"observer%1:18:00::","synset_id":"perceiver.n.01"},{"index":2,"sense_key":"observer%1:18:01::","synset_id":"observer.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"observer%1:18:00::","call_ids":["semeval2013.d000.s014.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"observer%1:18:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":728,"cached_input_tokens":0,"output_tokens":149,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002946,"input_uncached_usd":0.0014559999999999998,"input_cached_usd":0.0,"output_usd":0.0014900000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.122099708998576}
{"item_id":"semeval2013.d000.s015.t000","gold_sense_keys":["comment%1:10:00::"],"candidates":[{"index":1,"sense_key":"comment%1:10:00::","synset_id":"remark.n.01"},{"index":2,"sense_key":"comment%1:10:01::","synset_id":"comment.n.02"},{"index":3,"sense_key":"comment%1:10:02::","synset_id":"gossip.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"comment%1:10:00::","call_ids":["semeval2013.d000.s015.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"comment%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":863,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001846,"input_uncached_usd":0.0017259999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.7428923750994727}
{"item_id":"semeval2013.d000.s015.t001","gold_sense_keys":["hour%1:28:00::"],"candidates":[{"index":1,"sense_key":"hour%1:28:00::","synset_id":"hour.n.01"},{"index":2,"sense_key":"hour%1:28:01::","synset_id":"hour.n.02"},{"index":3,"sense_key":"hour%1:28:02::","synset_id":"hour.n.03"},{"index":4,"sense_key":"hour%1:07:00::","synset_id":"hour.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"hour%1:28:00::","call_ids":["semeval2013.d000.s015.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"hour%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":865,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00237,"input_uncached_usd":0.00173,"input_cached_usd":0.0,"output_usd":0.00064,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.805452375090681}
{"item_id":"semeval2013.d000.s015.t003","gold_sense_keys":["america%1:15:00::"],"candidates":[{"index":1,"sense_key":"america%1:15:00::","synset_id":"united_states.n.01"},{"index":2,"sense_key":"america%1:17:00::","synset_id":"america.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"america%1:15:00::","call_ids":["semeval2013.d000.s015.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"america%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":786,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001692,"input_uncached_usd":0.0015719999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1409138339804485}
{"item_id":"semeval2013.d000.s015.t004","gold_sense_keys":["climate%1:26:00::"],"candidates":[{"index":1,"sense_key":"climate%1:26:00::","synset_id":"climate.n.01"},{"index":2,"sense_key":"climate%1:26:01::","synset_id":"climate.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"climate%1:26:00::","call_ids":["semeval2013.d000.s015.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"climate%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":788,"cached_input_tokens":0,"output_tokens":92,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002496,"input_uncached_usd":0.001576,"input_cached_usd":0.0,"output_usd":0.00092,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6194403750123456}
{"item_id":"semeval2013.d000.s015.t007","gold_sense_keys":["week%1:28:02::"],"candidates":[{"index":1,"sense_key":"week%1:28:00::","synset_id":"week.n.01"},{"index":2,"sense_key":"week%1:28:01::","synset_id":"workweek.n.01"},{"index":3,"sense_key":"week%1:28:02::","synset_id":"week.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"week%1:28:02::","call_ids":["semeval2013.d000.s015.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"week%1:28:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":803,"cached_input_tokens":0,"output_tokens":104,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002646,"input_uncached_usd":0.001606,"input_cached_usd":0.0,"output_usd":0.0010400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8705747089115903}
{"item_id":"semeval2013.d000.s015.t008","gold_sense_keys":["united_states%1:14:00::"],"candidates":[{"index":1,"sense_key":"united_states%1:15:00::","synset_id":"united_states.n.01"},{"index":2,"sense_key":"united_states%1:14:00::","synset_id":"united_states_government.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"united_states%1:14:00::","call_ids":["semeval2013.d000.s015.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"united_states%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":824,"cached_input_tokens":0,"output_tokens":83,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002478,"input_uncached_usd":0.001648,"input_cached_usd":0.0,"output_usd":0.0008300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.251675167004578}
{"item_id":"semeval2013.d000.s015.t009","gold_sense_keys":["china%1:15:00::"],"candidates":[{"index":1,"sense_key":"china%1:15:00::","synset_id":"china.n.01"},{"index":2,"sense_key":"china%1:06:00::","synset_id":"china.n.02"},{"index":3,"sense_key":"china%1:15:01::","synset_id":"taiwan.n.01"},{"index":4,"sense_key":"china%1:06:01::","synset_id":"chinaware.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"china%1:15:00::","call_ids":["semeval2013.d000.s015.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"china%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":907,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001934,"input_uncached_usd":0.0018139999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.231367125059478}
{"item_id":"semeval2013.d000.s016.t001","gold_sense_keys":["world%1:17:00::"],"candidates":[{"index":1,"sense_key":"world%1:17:01::","synset_id":"universe.n.01"},{"index":2,"sense_key":"world%1:14:01::","synset_id":"world.n.02"},{"index":3,"sense_key":"world%1:09:00::","synset_id":"world.n.03"},{"index":4,"sense_key":"world%1:17:00::","synset_id":"earth.n.01"},{"index":5,"sense_key":"world%1:14:02::","synset_id":"populace.n.01"},{"index":6,"sense_key":"world%1:17:02::","synset_id":"world.n.06"},{"index":7,"sense_key":"world%1:09:01::","synset_id":"worldly_concern.n.01"},{"index":8,"sense_key":"world%1:05:00::","synset_id":"world.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"world%1:14:01::","call_ids":["semeval2013.d000.s016.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"world%1:14:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1200,"cached_input_tokens":0,"output_tokens":217,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004569999999999999,"input_uncached_usd":0.0024,"input_cached_usd":0.0,"output_usd":0.00217,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":4.0608271670062095}
{"item_id":"semeval2013.d000.s017.t000","gold_sense_keys":["battle%1:04:02::"],"candidates":[{"index":1,"sense_key":"battle%1:04:00::","synset_id":"battle.n.01"},{"index":2,"sense_key":"battle%1:04:01::","synset_id":"struggle.n.01"},{"index":3,"sense_key":"battle%1:04:02::","synset_id":"conflict.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"battle%1:04:02::","call_ids":["semeval2013.d000.s017.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"battle%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":987,"cached_input_tokens":0,"output_tokens":123,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0032040000000000003,"input_uncached_usd":0.001974,"input_cached_usd":0.0,"output_usd":0.0012300000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9127544170478359}
{"item_id":"semeval2013.d000.s017.t001","gold_sense_keys":["economy%1:14:00::"],"candidates":[{"index":1,"sense_key":"economy%1:14:00::","synset_id":"economy.n.01"},{"index":2,"sense_key":"economy%1:09:01::","synset_id":"economy.n.02"},{"index":3,"sense_key":"economy%1:07:00::","synset_id":"economy.n.03"},{"index":4,"sense_key":"economy%1:04:00::","synset_id":"economy.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"economy%1:14:00::","call_ids":["semeval2013.d000.s017.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"economy%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":923,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001966,"input_uncached_usd":0.001846,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.211056916974485}
{"item_id":"semeval2013.d000.s017.t003","gold_sense_keys":["china%1:15:00::"],"candidates":[{"index":1,"sense_key":"china%1:15:00::","synset_id":"china.n.01"},{"index":2,"sense_key":"china%1:06:00::","synset_id":"china.n.02"},{"index":3,"sense_key":"china%1:15:01::","synset_id":"taiwan.n.01"},{"index":4,"sense_key":"china%1:06:01::","synset_id":"chinaware.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"china%1:15:00::","call_ids":["semeval2013.d000.s017.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"china%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":965,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0020499999999999997,"input_uncached_usd":0.0019299999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4235670829657465}
{"item_id":"semeval2013.d000.s017.t004","gold_sense_keys":["united_states%1:14:00::"],"candidates":[{"index":1,"sense_key":"united_states%1:15:00::","synset_id":"united_states.n.01"},{"index":2,"sense_key":"united_states%1:14:00::","synset_id":"united_states_government.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"united_states%1:15:00::","call_ids":["semeval2013.d000.s017.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"united_states%1:15:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":882,"cached_input_tokens":0,"output_tokens":117,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002934,"input_uncached_usd":0.001764,"input_cached_usd":0.0,"output_usd":0.00117,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7911031669937074}
{"item_id":"semeval2013.d000.s017.t005","gold_sense_keys":["obligation%1:04:00::"],"candidates":[{"index":1,"sense_key":"obligation%1:04:00::","synset_id":"duty.n.01"},{"index":2,"sense_key":"obligation%1:26:00::","synset_id":"obligation.n.02"},{"index":3,"sense_key":"obligation%1:24:00::","synset_id":"obligation.n.03"},{"index":4,"sense_key":"obligation%1:21:00::","synset_id":"debt_instrument.n.01"},{"index":5,"sense_key":"obligation%1:10:00::","synset_id":"obligation.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"obligation%1:04:00::","call_ids":["semeval2013.d000.s017.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"obligation%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1032,"cached_input_tokens":0,"output_tokens":157,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003634,"input_uncached_usd":0.002064,"input_cached_usd":0.0,"output_usd":0.0015700000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.5669483749661595}
{"item_id":"semeval2013.d000.s017.t006","gold_sense_keys":["nation%1:14:00::"],"candidates":[{"index":1,"sense_key":"nation%1:14:00::","synset_id":"state.n.04"},{"index":2,"sense_key":"nation%1:14:01::","synset_id":"nation.n.02"},{"index":3,"sense_key":"nation%1:18:00::","synset_id":"nation.n.03"},{"index":4,"sense_key":"nation%1:14:02::","synset_id":"nation.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"nation%1:14:00::","call_ids":["semeval2013.d000.s017.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"nation%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":988,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0020959999999999998,"input_uncached_usd":0.001976,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8874268749495968}
{"item_id":"semeval2013.d000.s017.t007","gold_sense_keys":["world%1:17:00::"],"candidates":[{"index":1,"sense_key":"world%1:17:01::","synset_id":"universe.n.01"},{"index":2,"sense_key":"world%1:14:01::","synset_id":"world.n.02"},{"index":3,"sense_key":"world%1:09:00::","synset_id":"world.n.03"},{"index":4,"sense_key":"world%1:17:00::","synset_id":"earth.n.01"},{"index":5,"sense_key":"world%1:14:02::","synset_id":"populace.n.01"},{"index":6,"sense_key":"world%1:17:02::","synset_id":"world.n.06"},{"index":7,"sense_key":"world%1:09:01::","synset_id":"worldly_concern.n.01"},{"index":8,"sense_key":"world%1:05:00::","synset_id":"world.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"world%1:05:00::","call_ids":["semeval2013.d000.s017.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"world%1:05:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1260,"cached_input_tokens":0,"output_tokens":239,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0049099999999999994,"input_uncached_usd":0.0025199999999999997,"input_cached_usd":0.0,"output_usd":0.00239,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.338555374997668}
{"item_id":"semeval2013.d000.s017.t009","gold_sense_keys":["country%1:14:00::"],"candidates":[{"index":1,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":2,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":3,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":4,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":5,"sense_key":"country%1:15:02::","synset_id":"area.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"country%1:14:00::","call_ids":["semeval2013.d000.s017.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"country%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1058,"cached_input_tokens":0,"output_tokens":109,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0032059999999999996,"input_uncached_usd":0.002116,"input_cached_usd":0.0,"output_usd":0.00109,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7729017920792103}
{"item_id":"semeval2013.d000.s017.t010","gold_sense_keys":["imperative%1:04:00::"],"candidates":[{"index":1,"sense_key":"imperative%1:24:00::","synset_id":"imperative_mood.n.01"},{"index":2,"sense_key":"imperative%1:04:00::","synset_id":"imperative.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"imperative%1:04:00::","call_ids":["semeval2013.d000.s017.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"imperative%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":808,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0017360000000000001,"input_uncached_usd":0.001616,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5747775840573013}
{"item_id":"semeval2013.d000.s018.t002","gold_sense_keys":["cost%1:21:00::"],"candidates":[{"index":1,"sense_key":"cost%1:21:00::","synset_id":"cost.n.01"},{"index":2,"sense_key":"cost%1:07:00::","synset_id":"monetary_value.n.01"},{"index":3,"sense_key":"cost%1:07:01::","synset_id":"price.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cost%1:21:00::","call_ids":["semeval2013.d000.s018.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cost%1:21:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":890,"cached_input_tokens":0,"output_tokens":128,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0030600000000000002,"input_uncached_usd":0.00178,"input_cached_usd":0.0,"output_usd":0.00128,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8794782500481233}
{"item_id":"semeval2013.d000.s018.t004","gold_sense_keys":["country%1:14:00::"],"candidates":[{"index":1,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":2,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":3,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":4,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":5,"sense_key":"country%1:15:02::","synset_id":"area.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"country%1:14:00::","call_ids":["semeval2013.d000.s018.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"country%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1039,"cached_input_tokens":0,"output_tokens":99,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003068,"input_uncached_usd":0.002078,"input_cached_usd":0.0,"output_usd":0.00099,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1008947920054197}
{"item_id":"semeval2013.d000.s018.t005","gold_sense_keys":["action%1:04:02::"],"candidates":[{"index":1,"sense_key":"action%1:04:02::","synset_id":"action.n.01"},{"index":2,"sense_key":"action%1:26:00::","synset_id":"action.n.02"},{"index":3,"sense_key":"action%1:04:03::","synset_id":"military_action.n.01"},{"index":4,"sense_key":"action%1:22:00::","synset_id":"natural_process.n.01"},{"index":5,"sense_key":"action%1:10:00::","synset_id":"action.n.05"},{"index":6,"sense_key":"action%1:07:00::","synset_id":"action.n.06"},{"index":7,"sense_key":"action%1:06:00::","synset_id":"action.n.07"},{"index":8,"sense_key":"action%1:04:01::","synset_id":"legal_action.n.01"},{"index":9,"sense_key":"action%1:04:04::","synset_id":"action.n.09"},{"index":10,"sense_key":"action%1:04:00::","synset_id":"action.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"action%1:04:02::","call_ids":["semeval2013.d000.s018.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"action%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1311,"cached_input_tokens":0,"output_tokens":110,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0037219999999999996,"input_uncached_usd":0.0026219999999999998,"input_cached_usd":0.0,"output_usd":0.0011,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1968570000026375}
{"item_id":"semeval2013.d000.s019.t003","gold_sense_keys":["central_america%1:14:00::"],"candidates":[{"index":1,"sense_key":"central_america%1:15:00::","synset_id":"central_america.n.01"},{"index":2,"sense_key":"central_america%1:14:00::","synset_id":"central_america.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"central_america%1:14:00::","call_ids":["semeval2013.d000.s019.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"central_america%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":812,"cached_input_tokens":0,"output_tokens":79,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002414,"input_uncached_usd":0.001624,"input_cached_usd":0.0,"output_usd":0.00079,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0229536660481244}
{"item_id":"semeval2013.d000.s020.t000","gold_sense_keys":["country%1:14:00::"],"candidates":[{"index":1,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":2,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":3,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":4,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":5,"sense_key":"country%1:15:02::","synset_id":"area.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"country%1:14:00::","call_ids":["semeval2013.d000.s020.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"country%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1049,"cached_input_tokens":0,"output_tokens":112,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0032180000000000004,"input_uncached_usd":0.002098,"input_cached_usd":0.0,"output_usd":0.0011200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9899182080989704}
{"item_id":"semeval2013.d000.s020.t001","gold_sense_keys":["impact%1:19:00::"],"candidates":[{"index":1,"sense_key":"impact%1:11:00::","synset_id":"impact.n.01"},{"index":2,"sense_key":"impact%1:19:00::","synset_id":"impact.n.02"},{"index":3,"sense_key":"impact%1:04:00::","synset_id":"impingement.n.01"},{"index":4,"sense_key":"impact%1:04:01::","synset_id":"shock.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"impact%1:19:00::","call_ids":["semeval2013.d000.s020.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"impact%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":920,"cached_input_tokens":0,"output_tokens":41,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00225,"input_uncached_usd":0.0018399999999999998,"input_cached_usd":0.0,"output_usd":0.00041000000000000005,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2996349160093814}
{"item_id":"semeval2013.d000.s020.t005","gold_sense_keys":["participant%1:18:00::"],"candidates":[{"index":1,"sense_key":"participant%1:18:00::","synset_id":"participant.n.01"},{"index":2,"sense_key":"participant%1:18:01::","synset_id":"player.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"participant%1:18:00::","call_ids":["semeval2013.d000.s020.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"participant%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":773,"cached_input_tokens":0,"output_tokens":90,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002446,"input_uncached_usd":0.0015459999999999998,"input_cached_usd":0.0,"output_usd":0.0009000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7414149169344455}
{"item_id":"semeval2013.d000.s020.t007","gold_sense_keys":["deal%1:10:00::"],"candidates":[{"index":1,"sense_key":"deal%1:04:02::","synset_id":"deal.n.01"},{"index":2,"sense_key":"deal%1:10:00::","synset_id":"bargain.n.01"},{"index":3,"sense_key":"deal%1:23:00::","synset_id":"batch.n.02"},{"index":4,"sense_key":"deal%1:27:00::","synset_id":"deal.n.04"},{"index":5,"sense_key":"deal%1:27:01::","synset_id":"softwood.n.01"},{"index":6,"sense_key":"deal%1:14:00::","synset_id":"hand.n.06"},{"index":7,"sense_key":"deal%1:11:00::","synset_id":"deal.n.07"},{"index":8,"sense_key":"deal%1:04:00::","synset_id":"deal.n.08"},{"index":9,"sense_key":"deal%1:04:01::","synset_id":"deal.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"deal%1:10:00::","call_ids":["semeval2013.d000.s020.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"deal%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1254,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0026279999999999997,"input_uncached_usd":0.002508,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5628562499769032}
{"item_id":"semeval2013.d000.s021.t000","gold_sense_keys":["member%1:14:00::"],"candidates":[{"index":1,"sense_key":"member%1:18:00::","synset_id":"member.n.01"},{"index":2,"sense_key":"member%1:24:00::","synset_id":"member.n.02"},{"index":3,"sense_key":"member%1:08:00::","synset_id":"extremity.n.01"},{"index":4,"sense_key":"member%1:14:00::","synset_id":"member.n.04"},{"index":5,"sense_key":"member%1:08:01::","synset_id":"penis.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"member%1:14:00::","call_ids":["semeval2013.d000.s021.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"member%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1020,"cached_input_tokens":0,"output_tokens":87,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00291,"input_uncached_usd":0.0020399999999999997,"input_cached_usd":0.0,"output_usd":0.0008700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9636058340547606}
{"item_id":"semeval2013.d000.s021.t001","gold_sense_keys":["draft%1:10:00::"],"candidates":[{"index":1,"sense_key":"draft%1:21:00::","synset_id":"draft.n.01"},{"index":2,"sense_key":"draft%1:19:00::","synset_id":"draft.n.02"},{"index":3,"sense_key":"draft%1:06:01::","synset_id":"draft.n.03"},{"index":4,"sense_key":"draft%1:13:00::","synset_id":"draft.n.04"},{"index":5,"sense_key":"draft%1:10:00::","synset_id":"draft.n.05"},{"index":6,"sense_key":"draft%1:07:00::","synset_id":"draft.n.06"},{"index":7,"sense_key":"draft%1:06:02::","synset_id":"draft.n.07"},{"index":8,"sense_key":"draft%1:06:00::","synset_id":"draft.n.08"},{"index":9,"sense_key":"draft%1:04:03::","synset_id":"conscription.n.01"},{"index":10,"sense_key":"draft%1:04:02::","synset_id":"gulp.n.01"},{"index":11,"sense_key":"draft%1:04:00::","synset_id":"draft.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"draft%1:10:00::","call_ids":["semeval2013.d000.s021.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"draft%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1288,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0026959999999999996,"input_uncached_usd":0.0025759999999999997,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1145842499099672}
{"item_id":"semeval2013.d000.s021.t003","gold_sense_keys":["proposal%1:10:00::"],"candidates":[{"index":1,"sense_key":"proposal%1:10:00::","synset_id":"proposal.n.01"},{"index":2,"sense_key":"proposal%1:10:02::","synset_id":"marriage_proposal.n.01"},{"index":3,"sense_key":"proposal%1:10:01::","synset_id":"proposal.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"proposal%1:10:00::","call_ids":["semeval2013.d000.s021.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"proposal%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":840,"cached_input_tokens":0,"output_tokens":94,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00262,"input_uncached_usd":0.0016799999999999999,"input_cached_usd":0.0,"output_usd":0.0009400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5755602919962257}
{"item_id":"semeval2013.d000.s021.t005","gold_sense_keys":["morning%1:28:00::"],"candidates":[{"index":1,"sense_key":"morning%1:28:00::","synset_id":"morning.n.01"},{"index":2,"sense_key":"morning%1:10:00::","synset_id":"good_morning.n.01"},{"index":3,"sense_key":"morning%1:28:01::","synset_id":"dawn.n.01"},{"index":4,"sense_key":"morning%1:11:00::","synset_id":"dawn.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"morning%1:28:00::","call_ids":["semeval2013.d000.s021.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"morning%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":928,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0024460000000000003,"input_uncached_usd":0.001856,"input_cached_usd":0.0,"output_usd":0.00059,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.276257749996148}
{"item_id":"semeval2013.d000.s022.t000","gold_sense_keys":["climate%1:26:00::"],"candidates":[{"index":1,"sense_key":"climate%1:26:00::","synset_id":"climate.n.01"},{"index":2,"sense_key":"climate%1:26:01::","synset_id":"climate.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"climate%1:26:00::","call_ids":["semeval2013.d000.s022.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"climate%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":833,"cached_input_tokens":0,"output_tokens":85,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002516,"input_uncached_usd":0.001666,"input_cached_usd":0.0,"output_usd":0.0008500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8613977500936016}
{"item_id":"semeval2013.d000.s022.t001","gold_sense_keys":["negotiation%1:10:00::"],"candidates":[{"index":1,"sense_key":"negotiation%1:10:00::","synset_id":"negotiation.n.01"},{"index":2,"sense_key":"negotiation%1:04:00::","synset_id":"negotiation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"negotiation%1:10:00::","call_ids":["semeval2013.d000.s022.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"negotiation%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":824,"cached_input_tokens":0,"output_tokens":157,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0032180000000000004,"input_uncached_usd":0.001648,"input_cached_usd":0.0,"output_usd":0.0015700000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.7277075829915702}
{"item_id":"semeval2013.d000.s022.t002","gold_sense_keys":["behalf%1:04:00::"],"candidates":[{"index":1,"sense_key":"behalf%1:04:00::","synset_id":"behalf.n.01"},{"index":2,"sense_key":"behalf%1:07:00::","synset_id":"behalf.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"behalf%1:04:00::","call_ids":["semeval2013.d000.s022.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"behalf%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":839,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0017980000000000001,"input_uncached_usd":0.001678,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0994406249374151}
{"item_id":"semeval2013.d000.s022.t004","gold_sense_keys":["pressure%1:07:00::"],"candidates":[{"index":1,"sense_key":"pressure%1:19:00::","synset_id":"pressure.n.01"},{"index":2,"sense_key":"pressure%1:07:00::","synset_id":"pressure.n.02"},{"index":3,"sense_key":"pressure%1:04:00::","synset_id":"press.n.09"},{"index":4,"sense_key":"pressure%1:26:00::","synset_id":"imperativeness.n.01"},{"index":5,"sense_key":"pressure%1:09:00::","synset_id":"pressure.n.05"},{"index":6,"sense_key":"pressure%1:26:01::","synset_id":"pressure.n.06"},{"index":7,"sense_key":"pressure%1:19:01::","synset_id":"atmospheric_pressure.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"pressure%1:07:00::","call_ids":["semeval2013.d000.s022.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"pressure%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1163,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0029360000000000002,"input_uncached_usd":0.002326,"input_cached_usd":0.0,"output_usd":0.0006100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2662546670762822}
{"item_id":"semeval2013.d000.s022.t005","gold_sense_keys":["process%1:04:00::"],"candidates":[{"index":1,"sense_key":"process%1:04:00::","synset_id":"procedure.n.01"},{"index":2,"sense_key":"process%1:09:00::","synset_id":"process.n.02"},{"index":3,"sense_key":"process%1:10:00::","synset_id":"summons.n.03"},{"index":4,"sense_key":"process%1:09:01::","synset_id":"process.n.04"},{"index":5,"sense_key":"process%1:08:00::","synset_id":"process.n.05"},{"index":6,"sense_key":"process%1:03:00::","synset_id":"process.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"process%1:04:00::","call_ids":["semeval2013.d000.s022.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"process%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1137,"cached_input_tokens":0,"output_tokens":118,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003454,"input_uncached_usd":0.002274,"input_cached_usd":0.0,"output_usd":0.00118,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.9684583339840174}
{"item_id":"semeval2013.d000.s022.t006","gold_sense_keys":["action%1:04:04::"],"candidates":[{"index":1,"sense_key":"action%1:04:02::","synset_id":"action.n.01"},{"index":2,"sense_key":"action%1:26:00::","synset_id":"action.n.02"},{"index":3,"sense_key":"action%1:04:03::","synset_id":"military_action.n.01"},{"index":4,"sense_key":"action%1:22:00::","synset_id":"natural_process.n.01"},{"index":5,"sense_key":"action%1:10:00::","synset_id":"action.n.05"},{"index":6,"sense_key":"action%1:07:00::","synset_id":"action.n.06"},{"index":7,"sense_key":"action%1:06:00::","synset_id":"action.n.07"},{"index":8,"sense_key":"action%1:04:01::","synset_id":"legal_action.n.01"},{"index":9,"sense_key":"action%1:04:04::","synset_id":"action.n.09"},{"index":10,"sense_key":"action%1:04:00::","synset_id":"action.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"action%1:04:04::","call_ids":["semeval2013.d000.s022.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"action%1:04:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1316,"cached_input_tokens":0,"output_tokens":145,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004082,"input_uncached_usd":0.002632,"input_cached_usd":0.0,"output_usd":0.0014500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.3645678339526057}
{"item_id":"semeval2013.d000.s022.t007","gold_sense_keys":["china%1:15:00::"],"candidates":[{"index":1,"sense_key":"china%1:15:00::","synset_id":"china.n.01"},{"index":2,"sense_key":"china%1:06:00::","synset_id":"china.n.02"},{"index":3,"sense_key":"china%1:15:01::","synset_id":"taiwan.n.01"},{"index":4,"sense_key":"china%1:06:01::","synset_id":"chinaware.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"china%1:15:00::","call_ids":["semeval2013.d000.s022.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"china%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":952,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0020239999999999998,"input_uncached_usd":0.0019039999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0027982500614598}
{"item_id":"semeval2013.d000.s023.t001","gold_sense_keys":["urgency%1:07:00::","urgency%1:26:00::"],"candidates":[{"index":1,"sense_key":"urgency%1:26:00::","synset_id":"urgency.n.01"},{"index":2,"sense_key":"urgency%1:07:00::","synset_id":"urgency.n.02"},{"index":3,"sense_key":"urgency%1:26:01::","synset_id":"urgency.n.03"},{"index":4,"sense_key":"urgency%1:10:00::","synset_id":"importunity.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"urgency%1:07:00::","call_ids":["semeval2013.d000.s023.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"urgency%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":933,"cached_input_tokens":0,"output_tokens":171,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003576,"input_uncached_usd":0.0018659999999999998,"input_cached_usd":0.0,"output_usd":0.0017100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.391253291978501}
{"item_id":"semeval2013.d000.s023.t003","gold_sense_keys":["fact%1:09:01::"],"candidates":[{"index":1,"sense_key":"fact%1:09:01::","synset_id":"fact.n.01"},{"index":2,"sense_key":"fact%1:10:01::","synset_id":"fact.n.02"},{"index":3,"sense_key":"fact%1:26:01::","synset_id":"fact.n.03"},{"index":4,"sense_key":"fact%1:09:02::","synset_id":"fact.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"fact%1:09:01::","call_ids":["semeval2013.d000.s023.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"fact%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":944,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002448,"input_uncached_usd":0.001888,"input_cached_usd":0.0,"output_usd":0.0005600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.416297624935396}
{"item_id":"semeval2013.d000.s023.t004","gold_sense_keys":["issue%1:09:01::"],"candidates":[{"index":1,"sense_key":"issue%1:09:01::","synset_id":"issue.n.01"},{"index":2,"sense_key":"issue%1:10:00::","synset_id":"issue.n.02"},{"index":3,"sense_key":"issue%1:09:00::","synset_id":"topic.n.02"},{"index":4,"sense_key":"issue%1:04:01::","synset_id":"issue.n.04"},{"index":5,"sense_key":"issue%1:21:01::","synset_id":"issue.n.05"},{"index":6,"sense_key":"issue%1:21:00::","synset_id":"return.n.06"},{"index":7,"sense_key":"issue%1:19:00::","synset_id":"consequence.n.01"},{"index":8,"sense_key":"issue%1:18:00::","synset_id":"offspring.n.01"},{"index":9,"sense_key":"issue%1:11:00::","synset_id":"emergence.n.02"},{"index":10,"sense_key":"issue%1:06:00::","synset_id":"exit.n.01"},{"index":11,"sense_key":"issue%1:04:00::","synset_id":"issue.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"issue%1:09:01::","call_ids":["semeval2013.d000.s023.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"issue%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1481,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0036119999999999998,"input_uncached_usd":0.002962,"input_cached_usd":0.0,"output_usd":0.0006500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1191834999481216}
{"item_id":"semeval2013.d000.s023.t005","gold_sense_keys":["deal%1:10:00::"],"candidates":[{"index":1,"sense_key":"deal%1:04:02::","synset_id":"deal.n.01"},{"index":2,"sense_key":"deal%1:10:00::","synset_id":"bargain.n.01"},{"index":3,"sense_key":"deal%1:23:00::","synset_id":"batch.n.02"},{"index":4,"sense_key":"deal%1:27:00::","synset_id":"deal.n.04"},{"index":5,"sense_key":"deal%1:27:01::","synset_id":"softwood.n.01"},{"index":6,"sense_key":"deal%1:14:00::","synset_id":"hand.n.06"},{"index":7,"sense_key":"deal%1:11:00::","synset_id":"deal.n.07"},{"index":8,"sense_key":"deal%1:04:00::","synset_id":"deal.n.08"},{"index":9,"sense_key":"deal%1:04:01::","synset_id":"deal.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"deal%1:10:00::","call_ids":["semeval2013.d000.s023.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"deal%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1260,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0026399999999999996,"input_uncached_usd":0.0025199999999999997,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1159398750169203}
{"item_id":"semeval2013.d000.s024.t000","gold_sense_keys":["official%1:18:01::"],"candidates":[{"index":1,"sense_key":"official%1:18:01::","synset_id":"official.n.01"},{"index":2,"sense_key":"official%1:18:00::","synset_id":"official.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"official%1:18:01::","call_ids":["semeval2013.d000.s024.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"official%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":807,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001734,"input_uncached_usd":0.001614,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0944871250540018}
{"item_id":"semeval2013.d000.s024.t001","gold_sense_keys":["minister%1:18:02::"],"candidates":[{"index":1,"sense_key":"minister%1:18:00::","synset_id":"curate.n.01"},{"index":2,"sense_key":"minister%1:18:02::","synset_id":"minister.n.02"},{"index":3,"sense_key":"minister%1:18:01::","synset_id":"minister.n.03"},{"index":4,"sense_key":"minister%1:04:00::","synset_id":"minister.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"minister%1:18:02::","call_ids":["semeval2013.d000.s024.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"minister%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":930,"cached_input_tokens":0,"output_tokens":109,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00295,"input_uncached_usd":0.0018599999999999999,"input_cached_usd":0.0,"output_usd":0.00109,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7938878330169246}
{"item_id":"semeval2013.d000.s024.t002","gold_sense_keys":["plane%1:06:01::"],"candidates":[{"index":1,"sense_key":"plane%1:06:01::","synset_id":"airplane.n.01"},{"index":2,"sense_key":"plane%1:25:00::","synset_id":"plane.n.02"},{"index":3,"sense_key":"plane%1:26:00::","synset_id":"plane.n.03"},{"index":4,"sense_key":"plane%1:06:02::","synset_id":"plane.n.04"},{"index":5,"sense_key":"plane%1:06:00::","synset_id":"plane.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"plane%1:06:01::","call_ids":["semeval2013.d000.s024.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"plane%1:06:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1070,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00226,"input_uncached_usd":0.00214,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2357314999680966}
{"item_id":"semeval2013.d000.s024.t003","gold_sense_keys":["hall%1:06:02::","hall%1:06:03::","hall%1:06:05::"],"candidates":[{"index":1,"sense_key":"hall%1:06:03::","synset_id":"hallway.n.01"},{"index":2,"sense_key":"hall%1:06:05::","synset_id":"anteroom.n.01"},{"index":3,"sense_key":"hall%1:06:02::","synset_id":"hall.n.03"},{"index":4,"sense_key":"hall%1:06:04::","synset_id":"dormitory.n.01"},{"index":5,"sense_key":"hall%1:06:06::","synset_id":"manor_hall.n.01"},{"index":6,"sense_key":"hall%1:18:04::","synset_id":"hall.n.06"},{"index":7,"sense_key":"hall%1:18:03::","synset_id":"hall.n.07"},{"index":8,"sense_key":"hall%1:18:02::","synset_id":"hall.n.08"},{"index":9,"sense_key":"hall%1:18:01::","synset_id":"hall.n.09"},{"index":10,"sense_key":"hall%1:18:00::","synset_id":"hall.n.10"},{"index":11,"sense_key":"hall%1:06:07::","synset_id":"mansion.n.02"},{"index":12,"sense_key":"hall%1:06:08::","synset_id":"hall.n.12"},{"index":13,"sense_key":"hall%1:06:01::","synset_id":"hall.n.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"hall%1:06:03::","call_ids":["semeval2013.d000.s024.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"hall%1:06:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1486,"cached_input_tokens":0,"output_tokens":71,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003682,"input_uncached_usd":0.002972,"input_cached_usd":0.0,"output_usd":0.00071,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3740344170946628}
{"item_id":"semeval2013.d000.s024.t004","gold_sense_keys":["meeting%1:14:00::"],"candidates":[{"index":1,"sense_key":"meeting%1:14:00::","synset_id":"meeting.n.01"},{"index":2,"sense_key":"meeting%1:14:01::","synset_id":"meeting.n.02"},{"index":3,"sense_key":"meeting%1:11:00::","synset_id":"meeting.n.03"},{"index":4,"sense_key":"meeting%1:04:00::","synset_id":"meeting.n.04"},{"index":5,"sense_key":"meeting%1:04:02::","synset_id":"merging.n.01"},{"index":6,"sense_key":"meeting%1:15:00::","synset_id":"confluence.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"meeting%1:14:00::","call_ids":["semeval2013.d000.s024.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"meeting%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1131,"cached_input_tokens":0,"output_tokens":84,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0031019999999999997,"input_uncached_usd":0.0022619999999999997,"input_cached_usd":0.0,"output_usd":0.00084,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0641417501028627}
{"item_id":"semeval2013.d000.s024.t006","gold_sense_keys":["claim%1:04:00::","claim%1:10:00::"],"candidates":[{"index":1,"sense_key":"claim%1:10:00::","synset_id":"claim.n.01"},{"index":2,"sense_key":"claim%1:10:02::","synset_id":"claim.n.02"},{"index":3,"sense_key":"claim%1:04:00::","synset_id":"claim.n.03"},{"index":4,"sense_key":"claim%1:07:02::","synset_id":"claim.n.04"},{"index":5,"sense_key":"claim%1:07:01::","synset_id":"title.n.07"},{"index":6,"sense_key":"claim%1:10:01::","synset_id":"call.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"claim%1:10:00::","call_ids":["semeval2013.d000.s024.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"claim%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1036,"cached_input_tokens":0,"output_tokens":230,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004372000000000001,"input_uncached_usd":0.002072,"input_cached_usd":0.0,"output_usd":0.0023000000000000004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.7202875829534605}
{"item_id":"semeval2013.d000.s024.t007","gold_sense_keys":["week%1:28:00::"],"candidates":[{"index":1,"sense_key":"week%1:28:00::","synset_id":"week.n.01"},{"index":2,"sense_key":"week%1:28:01::","synset_id":"workweek.n.01"},{"index":3,"sense_key":"week%1:28:02::","synset_id":"week.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"week%1:28:00::","call_ids":["semeval2013.d000.s024.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"week%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":863,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002306,"input_uncached_usd":0.0017259999999999999,"input_cached_usd":0.0,"output_usd":0.00058,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.42957195895724}
{"item_id":"semeval2013.d000.s025.t000","gold_sense_keys":["gathering%1:14:00::"],"candidates":[{"index":1,"sense_key":"gathering%1:14:00::","synset_id":"gathering.n.01"},{"index":2,"sense_key":"gathering%1:04:00::","synset_id":"assembly.n.06"},{"index":3,"sense_key":"gathering%1:04:01::","synset_id":"gather.n.02"},{"index":4,"sense_key":"gathering%1:06:00::","synset_id":"gather.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"gathering%1:14:00::","call_ids":["semeval2013.d000.s025.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"gathering%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":903,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001926,"input_uncached_usd":0.0018059999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2613175410078838}
{"item_id":"semeval2013.d000.s025.t001","gold_sense_keys":["people%1:14:00::"],"candidates":[{"index":1,"sense_key":"people%1:14:00::","synset_id":"people.n.01"},{"index":2,"sense_key":"people%1:14:01::","synset_id":"citizenry.n.01"},{"index":3,"sense_key":"people%1:14:02::","synset_id":"people.n.03"},{"index":4,"sense_key":"people%1:14:03::","synset_id":"multitude.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"people%1:14:00::","call_ids":["semeval2013.d000.s025.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"people%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":933,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001986,"input_uncached_usd":0.0018659999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.99660729104653}
{"item_id":"semeval2013.d000.s025.t002","gold_sense_keys":["day%1:28:00::"],"candidates":[{"index":1,"sense_key":"day%1:28:00::","synset_id":"day.n.01"},{"index":2,"sense_key":"day%1:28:03::","synset_id":"day.n.02"},{"index":3,"sense_key":"day%1:28:01::","synset_id":"day.n.03"},{"index":4,"sense_key":"day%1:28:02::","synset_id":"day.n.04"},{"index":5,"sense_key":"day%1:28:04::","synset_id":"day.n.05"},{"index":6,"sense_key":"day%1:28:05::","synset_id":"day.n.06"},{"index":7,"sense_key":"day%1:28:07::","synset_id":"day.n.07"},{"index":8,"sense_key":"day%1:28:06::","synset_id":"sidereal_day.n.01"},{"index":9,"sense_key":"day%1:26:00::","synset_id":"day.n.09"},{"index":10,"sense_key":"day%1:18:00::","synset_id":"day.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"day%1:28:00::","call_ids":["semeval2013.d000.s025.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"day%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1353,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003376,"input_uncached_usd":0.002706,"input_cached_usd":0.0,"output_usd":0.00067,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.927063624956645}
{"item_id":"semeval2013.d000.s025.t004","gold_sense_keys":["show%1:10:00::"],"candidates":[{"index":1,"sense_key":"show%1:04:00::","synset_id":"show.n.01"},{"index":2,"sense_key":"show%1:10:03::","synset_id":"display.n.01"},{"index":3,"sense_key":"show%1:10:00::","synset_id":"show.n.03"},{"index":4,"sense_key":"show%1:04:01::","synset_id":"appearance.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"show%1:10:00::","call_ids":["semeval2013.d000.s025.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"show%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":931,"cached_input_tokens":0,"output_tokens":69,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002552,"input_uncached_usd":0.001862,"input_cached_usd":0.0,"output_usd":0.0006900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7972307079471648}
{"item_id":"semeval2013.d000.s025.t005","gold_sense_keys":["country%1:14:00::"],"candidates":[{"index":1,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":2,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":3,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":4,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":5,"sense_key":"country%1:15:02::","synset_id":"area.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"country%1:14:00::","call_ids":["semeval2013.d000.s025.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"country%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1052,"cached_input_tokens":0,"output_tokens":87,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002974,"input_uncached_usd":0.002104,"input_cached_usd":0.0,"output_usd":0.0008700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7040773750049993}
{"item_id":"semeval2013.d000.s025.t006","gold_sense_keys":["draft%1:10:00::"],"candidates":[{"index":1,"sense_key":"draft%1:21:00::","synset_id":"draft.n.01"},{"index":2,"sense_key":"draft%1:19:00::","synset_id":"draft.n.02"},{"index":3,"sense_key":"draft%1:06:01::","synset_id":"draft.n.03"},{"index":4,"sense_key":"draft%1:13:00::","synset_id":"draft.n.04"},{"index":5,"sense_key":"draft%1:10:00::","synset_id":"draft.n.05"},{"index":6,"sense_key":"draft%1:07:00::","synset_id":"draft.n.06"},{"index":7,"sense_key":"draft%1:06:02::","synset_id":"draft.n.07"},{"index":8,"sense_key":"draft%1:06:00::","synset_id":"draft.n.08"},{"index":9,"sense_key":"draft%1:04:03::","synset_id":"conscription.n.01"},{"index":10,"sense_key":"draft%1:04:02::","synset_id":"gulp.n.01"},{"index":11,"sense_key":"draft%1:04:00::","synset_id":"draft.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"draft%1:10:00::","call_ids":["semeval2013.d000.s025.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"draft%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1297,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002714,"input_uncached_usd":0.002594,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3187205829890445}
{"item_id":"semeval2013.d000.s025.t007","gold_sense_keys":["proposal%1:10:00::"],"candidates":[{"index":1,"sense_key":"proposal%1:10:00::","synset_id":"proposal.n.01"},{"index":2,"sense_key":"proposal%1:10:02::","synset_id":"marriage_proposal.n.01"},{"index":3,"sense_key":"proposal%1:10:01::","synset_id":"proposal.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"proposal%1:10:00::","call_ids":["semeval2013.d000.s025.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"proposal%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":849,"cached_input_tokens":0,"output_tokens":81,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002508,"input_uncached_usd":0.0016979999999999999,"input_cached_usd":0.0,"output_usd":0.0008100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7429159579332918}
{"item_id":"semeval2013.d000.s025.t008","gold_sense_keys":["paper%1:10:00::"],"candidates":[{"index":1,"sense_key":"paper%1:27:00::","synset_id":"paper.n.01"},{"index":2,"sense_key":"paper%1:10:01::","synset_id":"composition.n.08"},{"index":3,"sense_key":"paper%1:10:03::","synset_id":"newspaper.n.01"},{"index":4,"sense_key":"paper%1:10:00::","synset_id":"paper.n.04"},{"index":5,"sense_key":"paper%1:10:02::","synset_id":"paper.n.05"},{"index":6,"sense_key":"paper%1:14:00::","synset_id":"newspaper.n.02"},{"index":7,"sense_key":"paper%1:06:00::","synset_id":"newspaper.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"paper%1:10:00::","call_ids":["semeval2013.d000.s025.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"paper%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1128,"cached_input_tokens":0,"output_tokens":89,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003146,"input_uncached_usd":0.0022559999999999998,"input_cached_usd":0.0,"output_usd":0.0008900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3076769999461249}
{"item_id":"semeval2013.d000.s025.t010","gold_sense_keys":["challenge%1:26:00::"],"candidates":[{"index":1,"sense_key":"challenge%1:26:00::","synset_id":"challenge.n.01"},{"index":2,"sense_key":"challenge%1:10:00::","synset_id":"challenge.n.02"},{"index":3,"sense_key":"challenge%1:10:01::","synset_id":"challenge.n.03"},{"index":4,"sense_key":"challenge%1:10:02::","synset_id":"challenge.n.04"},{"index":5,"sense_key":"challenge%1:10:03::","synset_id":"challenge.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"challenge%1:26:00::","call_ids":["semeval2013.d000.s025.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"challenge%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":932,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001984,"input_uncached_usd":0.001864,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0952642500633374}
{"item_id":"semeval2013.d000.s026.t000","gold_sense_keys":["intensity%1:07:00::"],"candidates":[{"index":1,"sense_key":"intensity%1:07:03::","synset_id":"intensity.n.01"},{"index":2,"sense_key":"intensity%1:07:00::","synset_id":"intensity.n.02"},{"index":3,"sense_key":"intensity%1:07:02::","synset_id":"volume.n.06"},{"index":4,"sense_key":"intensity%1:07:01::","synset_id":"saturation.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"intensity%1:07:00::","call_ids":["semeval2013.d000.s026.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"intensity%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":947,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0020139999999999997,"input_uncached_usd":0.0018939999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.069698000093922}
{"item_id":"semeval2013.d000.s026.t001","gold_sense_keys":["minister%1:18:02::"],"candidates":[{"index":1,"sense_key":"minister%1:18:00::","synset_id":"curate.n.01"},{"index":2,"sense_key":"minister%1:18:02::","synset_id":"minister.n.02"},{"index":3,"sense_key":"minister%1:18:01::","synset_id":"minister.n.03"},{"index":4,"sense_key":"minister%1:04:00::","synset_id":"minister.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"minister%1:18:02::","call_ids":["semeval2013.d000.s026.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"minister%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":897,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0019140000000000001,"input_uncached_usd":0.001794,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.258088667062111}
{"item_id":"semeval2013.d000.s027.t002","gold_sense_keys":["policy%1:09:00::"],"candidates":[{"index":1,"sense_key":"policy%1:09:00::","synset_id":"policy.n.01"},{"index":2,"sense_key":"policy%1:10:00::","synset_id":"policy.n.02"},{"index":3,"sense_key":"policy%1:10:02::","synset_id":"policy.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"policy%1:09:00::","call_ids":["semeval2013.d000.s027.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"policy%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":831,"cached_input_tokens":0,"output_tokens":96,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002622,"input_uncached_usd":0.0016619999999999998,"input_cached_usd":0.0,"output_usd":0.0009600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5501988750183955}
{"item_id":"semeval2013.d000.s028.t000","gold_sense_keys":["heavyweight%1:18:00::"],"candidates":[{"index":1,"sense_key":"heavyweight%1:18:04::","synset_id":"heavyweight.n.01"},{"index":2,"sense_key":"heavyweight%1:18:02::","synset_id":"heavyweight.n.02"},{"index":3,"sense_key":"heavyweight%1:18:01::","synset_id":"heavyweight.n.03"},{"index":4,"sense_key":"heavyweight%1:18:03::","synset_id":"giant.n.04"},{"index":5,"sense_key":"heavyweight%1:18:00::","synset_id":"colossus.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"heavyweight%1:18:00::","call_ids":["semeval2013.d000.s028.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"heavyweight%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":812,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001744,"input_uncached_usd":0.001624,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9157445420278236}
{"item_id":"semeval2013.d001.s000.t001","gold_sense_keys":["israel%1:15:00::"],"candidates":[{"index":1,"sense_key":"israel%1:15:00::","synset_id":"israel.n.01"},{"index":2,"sense_key":"israel%1:15:01::","synset_id":"israel.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"israel%1:15:00::","call_ids":["semeval2013.d001.s000.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"israel%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":525,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00117,"input_uncached_usd":0.00105,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.972364709014073}
{"item_id":"semeval2013.d001.s001.t002","gold_sense_keys":["team%1:14:00::"],"candidates":[{"index":1,"sense_key":"team%1:14:00::","synset_id":"team.n.01"},{"index":2,"sense_key":"team%1:14:01::","synset_id":"team.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"team%1:14:00::","call_ids":["semeval2013.d001.s001.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"team%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":498,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001116,"input_uncached_usd":0.000996,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0713119159918278}
{"item_id":"semeval2013.d001.s001.t003","gold_sense_keys":["loss%1:04:00::"],"candidates":[{"index":1,"sense_key":"loss%1:21:01::","synset_id":"loss.n.01"},{"index":2,"sense_key":"loss%1:22:00::","synset_id":"loss.n.02"},{"index":3,"sense_key":"loss%1:04:00::","synset_id":"loss.n.03"},{"index":4,"sense_key":"loss%1:07:00::","synset_id":"loss.n.04"},{"index":5,"sense_key":"loss%1:11:01::","synset_id":"loss.n.05"},{"index":6,"sense_key":"loss%1:21:00::","synset_id":"loss.n.06"},{"index":7,"sense_key":"loss%1:11:00::","synset_id":"personnel_casualty.n.01"},{"index":8,"sense_key":"loss%1:11:02::","synset_id":"passing.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"loss%1:21:01::","call_ids":["semeval2013.d001.s001.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"loss%1:21:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":887,"cached_input_tokens":0,"output_tokens":172,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003494,"input_uncached_usd":0.001774,"input_cached_usd":0.0,"output_usd":0.0017200000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.086075749946758}
{"item_id":"semeval2013.d001.s001.t004","gold_sense_keys":["performance%1:04:00::","performance%1:04:03::"],"candidates":[{"index":1,"sense_key":"performance%1:10:00::","synset_id":"performance.n.01"},{"index":2,"sense_key":"performance%1:04:01::","synset_id":"performance.n.02"},{"index":3,"sense_key":"performance%1:04:00::","synset_id":"performance.n.03"},{"index":4,"sense_key":"performance%1:04:03::","synset_id":"performance.n.04"},{"index":5,"sense_key":"performance%1:22:00::","synset_id":"operation.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"performance%1:04:03::","call_ids":["semeval2013.d001.s001.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"performance%1:04:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":806,"cached_input_tokens":0,"output_tokens":191,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003522,"input_uncached_usd":0.001612,"input_cached_usd":0.0,"output_usd":0.0019100000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.4398731250548735}
{"item_id":"semeval2013.d001.s001.t005","gold_sense_keys":["team%1:14:00::"],"candidates":[{"index":1,"sense_key":"team%1:14:00::","synset_id":"team.n.01"},{"index":2,"sense_key":"team%1:14:01::","synset_id":"team.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"team%1:14:00::","call_ids":["semeval2013.d001.s001.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"team%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":497,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001114,"input_uncached_usd":0.0009939999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9065338750369847}
{"item_id":"semeval2013.d001.s001.t006","gold_sense_keys":["point%1:23:01::"],"candidates":[{"index":1,"sense_key":"point%1:09:00::","synset_id":"point.n.01"},{"index":2,"sense_key":"point%1:15:00::","synset_id":"point.n.02"},{"index":3,"sense_key":"point%1:10:01::","synset_id":"point.n.03"},{"index":4,"sense_key":"point%1:09:01::","synset_id":"detail.n.01"},{"index":5,"sense_key":"point%1:26:00::","synset_id":"degree.n.02"},{"index":6,"sense_key":"point%1:28:00::","synset_id":"point.n.06"},{"index":7,"sense_key":"point%1:09:02::","synset_id":"point.n.07"},{"index":8,"sense_key":"point%1:25:00::","synset_id":"point.n.08"},{"index":9,"sense_key":"point%1:25:02::","synset_id":"point.n.09"},{"index":10,"sense_key":"point%1:23:01::","synset_id":"point.n.10"},{"index":11,"sense_key":"point%1:17:02::","synset_id":"point.n.11"},{"index":12,"sense_key":"point%1:10:03::","synset_id":"item.n.01"},{"index":13,"sense_key":"point%1:24:04::","synset_id":"point.n.13"},{"index":14,"sense_key":"point%1:07:01::","synset_id":"point.n.14"},{"index":15,"sense_key":"point%1:06:00::","synset_id":"point.n.15"},{"index":16,"sense_key":"point%1:24:00::","synset_id":"compass_point.n.01"},{"index":17,"sense_key":"point%1:23:00::","synset_id":"point.n.17"},{"index":18,"sense_key":"point%1:21:00::","synset_id":"point.n.18"},{"index":19,"sense_key":"point%1:10:00::","synset_id":"period.n.07"},{"index":20,"sense_key":"point%1:10:02::","synset_id":"point.n.20"},{"index":21,"sense_key":"point%1:10:04::","synset_id":"decimal_point.n.01"},{"index":22,"sense_key":"point%1:07:00::","synset_id":"point.n.22"},{"index":23,"sense_key":"point%1:07:02::","synset_id":"point.n.23"},{"index":24,"sense_key":"point%1:06:02::","synset_id":"point.n.24"},{"index":25,"sense_key":"point%1:06:01::","synset_id":"point.n.25"},{"index":26,"sense_key":"point%1:06:03::","synset_id":"distributor_point.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"point%1:23:01::","call_ids":["semeval2013.d001.s001.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"point%1:23:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1898,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003916,"input_uncached_usd":0.003796,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0719829170266166}
{"item_id":"semeval2013.d001.s001.t007","gold_sense_keys":["top%1:26:00::"],"candidates":[{"index":1,"sense_key":"top%1:15:01::","synset_id":"top.n.01"},{"index":2,"sense_key":"top%1:15:00::","synset_id":"top.n.02"},{"index":3,"sense_key":"top%1:15:02::","synset_id":"peak.n.04"},{"index":4,"sense_key":"top%1:28:00::","synset_id":"top.n.04"},{"index":5,"sense_key":"top%1:26:00::","synset_id":"acme.n.01"},{"index":6,"sense_key":"top%1:07:02::","synset_id":"top.n.06"},{"index":7,"sense_key":"top%1:06:04::","synset_id":"top.n.07"},{"index":8,"sense_key":"top%1:06:03::","synset_id":"top.n.08"},{"index":9,"sense_key":"top%1:06:01::","synset_id":"top.n.09"},{"index":10,"sense_key":"top%1:06:00::","synset_id":"top.n.10"},{"index":11,"sense_key":"top%1:06:02::","synset_id":"circus_tent.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"top%1:26:00::","call_ids":["semeval2013.d001.s001.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"top%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1198,"cached_input_tokens":0,"output_tokens":113,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0035259999999999996,"input_uncached_usd":0.0023959999999999997,"input_cached_usd":0.0,"output_usd":0.0011300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7387012500548735}
{"item_id":"semeval2013.d001.s001.t008","gold_sense_keys":["group%1:03:00::"],"candidates":[{"index":1,"sense_key":"group%1:03:00::","synset_id":"group.n.01"},{"index":2,"sense_key":"group%1:27:00::","synset_id":"group.n.02"},{"index":3,"sense_key":"group%1:09:00::","synset_id":"group.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"group%1:03:00::","call_ids":["semeval2013.d001.s001.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"group%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":581,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0012820000000000002,"input_uncached_usd":0.001162,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2138449589256197}
{"item_id":"semeval2013.d001.s002.t000","gold_sense_keys":["team%1:14:00::"],"candidates":[{"index":1,"sense_key":"team%1:14:00::","synset_id":"team.n.01"},{"index":2,"sense_key":"team%1:14:01::","synset_id":"team.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"team%1:14:00::","call_ids":["semeval2013.d001.s002.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"team%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":537,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001194,"input_uncached_usd":0.0010739999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3754095840267837}
{"item_id":"semeval2013.d001.s002.t001","gold_sense_keys":["absence%1:04:00::","absence%1:26:00::"],"candidates":[{"index":1,"sense_key":"absence%1:26:00::","synset_id":"absence.n.01"},{"index":2,"sense_key":"absence%1:04:00::","synset_id":"absence.n.02"},{"index":3,"sense_key":"absence%1:28:00::","synset_id":"absence.n.03"},{"index":4,"sense_key":"absence%1:26:01::","synset_id":"absence.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"absence%1:04:00::","call_ids":["semeval2013.d001.s002.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"absence%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":684,"cached_input_tokens":0,"output_tokens":141,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002778,"input_uncached_usd":0.0013679999999999999,"input_cached_usd":0.0,"output_usd":0.00141,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.3538196249864995}
{"item_id":"semeval2013.d001.s002.t002","gold_sense_keys":["involvement%1:04:00::"],"candidates":[{"index":1,"sense_key":"involvement%1:04:00::","synset_id":"engagement.n.07"},{"index":2,"sense_key":"involvement%1:24:00::","synset_id":"involvement.n.02"},{"index":3,"sense_key":"involvement%1:09:00::","synset_id":"interest.n.01"},{"index":4,"sense_key":"involvement%1:26:00::","synset_id":"affair.n.02"},{"index":5,"sense_key":"involvement%1:26:01::","synset_id":"participation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"involvement%1:04:00::","call_ids":["semeval2013.d001.s002.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"involvement%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":776,"cached_input_tokens":0,"output_tokens":111,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0026620000000000003,"input_uncached_usd":0.001552,"input_cached_usd":0.0,"output_usd":0.00111,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9237921671010554}
{"item_id":"semeval2013.d001.s002.t004","gold_sense_keys":["end%1:09:00::","end%1:11:00::","end%1:28:00::"],"candidates":[{"index":1,"sense_key":"end%1:15:00::","synset_id":"end.n.01"},{"index":2,"sense_key":"end%1:28:00::","synset_id":"end.n.02"},{"index":3,"sense_key":"end%1:11:00::","synset_id":"end.n.03"},{"index":4,"sense_key":"end%1:09:02::","synset_id":"goal.n.01"},{"index":5,"sense_key":"end%1:09:00::","synset_id":"end.n.05"},{"index":6,"sense_key":"end%1:26:00::","synset_id":"end.n.06"},{"index":7,"sense_key":"end%1:15:02::","synset_id":"end.n.07"},{"index":8,"sense_key":"end%1:18:00::","synset_id":"end.n.08"},{"index":9,"sense_key":"end%1:15:01::","synset_id":"end.n.09"},{"index":10,"sense_key":"end%1:15:03::","synset_id":"end.n.10"},{"index":11,"sense_key":"end%1:04:00::","synset_id":"end.n.11"},{"index":12,"sense_key":"end%1:10:00::","synset_id":"conclusion.n.08"},{"index":13,"sense_key":"end%1:06:02::","synset_id":"end.n.13"},{"index":14,"sense_key":"end%1:04:02::","synset_id":"end.n.14"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"end%1:11:00::","call_ids":["semeval2013.d001.s002.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"end%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1268,"cached_input_tokens":0,"output_tokens":115,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003686,"input_uncached_usd":0.002536,"input_cached_usd":0.0,"output_usd":0.0011500000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0409773340215907}
{"item_id":"semeval2013.d001.s003.t001","gold_sense_keys":["sideline%1:15:00::"],"candidates":[{"index":1,"sense_key":"sideline%1:15:00::","synset_id":"sideline.n.01"},{"index":2,"sense_key":"sideline%1:06:00::","synset_id":"sideline.n.02"},{"index":3,"sense_key":"sideline%1:04:00::","synset_id":"avocation.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"sideline%1:15:00::","call_ids":["semeval2013.d001.s003.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"sideline%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":643,"cached_input_tokens":0,"output_tokens":135,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0026360000000000003,"input_uncached_usd":0.001286,"input_cached_usd":0.0,"output_usd":0.00135,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.35894125001505}
{"item_id":"semeval2013.d001.s003.t002","gold_sense_keys":["lot%1:23:00::"],"candidates":[{"index":1,"sense_key":"lot%1:23:00::","synset_id":"batch.n.02"},{"index":2,"sense_key":"lot%1:15:00::","synset_id":"lot.n.02"},{"index":3,"sense_key":"lot%1:14:00::","synset_id":"set.n.05"},{"index":4,"sense_key":"lot%1:26:00::","synset_id":"fortune.n.04"},{"index":5,"sense_key":"lot%1:06:00::","synset_id":"draw.n.04"},{"index":6,"sense_key":"lot%1:14:01::","synset_id":"bunch.n.03"},{"index":7,"sense_key":"lot%1:18:00::","synset_id":"lot.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"lot%1:23:00::","call_ids":["semeval2013.d001.s003.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"lot%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":995,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00211,"input_uncached_usd":0.00199,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1889077079249546}
{"item_id":"semeval2013.d001.s003.t003","gold_sense_keys":["triumph%1:11:00::"],"candidates":[{"index":1,"sense_key":"triumph%1:11:00::","synset_id":"victory.n.01"},{"index":2,"sense_key":"triumph%1:12:00::","synset_id":"triumph.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"triumph%1:11:00::","call_ids":["semeval2013.d001.s003.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"triumph%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":596,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001312,"input_uncached_usd":0.001192,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2916568339569494}
{"item_id":"semeval2013.d001.s004.t000","gold_sense_keys":["game%1:04:03::"],"candidates":[{"index":1,"sense_key":"game%1:04:00::","synset_id":"game.n.01"},{"index":2,"sense_key":"game%1:04:03::","synset_id":"game.n.02"},{"index":3,"sense_key":"game%1:04:01::","synset_id":"game.n.03"},{"index":4,"sense_key":"game%1:05:00::","synset_id":"game.n.04"},{"index":5,"sense_key":"game%1:28:00::","synset_id":"game.n.05"},{"index":6,"sense_key":"game%1:23:00::","synset_id":"game.n.06"},{"index":7,"sense_key":"game%1:13:00::","synset_id":"game.n.07"},{"index":8,"sense_key":"game%1:09:00::","synset_id":"plot.n.01"},{"index":9,"sense_key":"game%1:06:00::","synset_id":"game.n.09"},{"index":10,"sense_key":"game%1:04:02::","synset_id":"game.n.10"},{"index":11,"sense_key":"game%1:04:04::","synset_id":"game.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"game%1:04:03::","call_ids":["semeval2013.d001.s004.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"game%1:04:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1093,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0027660000000000002,"input_uncached_usd":0.002186,"input_cached_usd":0.0,"output_usd":0.00058,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9948139160405844}
{"item_id":"semeval2013.d001.s004.t001","gold_sense_keys":["favor%1:07:00::"],"candidates":[{"index":1,"sense_key":"favor%1:04:00::","synset_id":"favor.n.01"},{"index":2,"sense_key":"favor%1:07:00::","synset_id":"favor.n.02"},{"index":3,"sense_key":"favor%1:09:00::","synset_id":"favor.n.03"},{"index":4,"sense_key":"favor%1:12:00::","synset_id":"favor.n.04"},{"index":5,"sense_key":"favor%1:06:00::","synset_id":"party_favor.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"favor%1:07:00::","call_ids":["semeval2013.d001.s004.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"favor%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":784,"cached_input_tokens":0,"output_tokens":80,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002368,"input_uncached_usd":0.001568,"input_cached_usd":0.0,"output_usd":0.0008,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.349500291980803}
{"item_id":"semeval2013.d001.s005.t000","gold_sense_keys":["basket%1:04:00::"],"candidates":[{"index":1,"sense_key":"basket%1:06:00::","synset_id":"basket.n.01"},{"index":2,"sense_key":"basket%1:23:00::","synset_id":"basket.n.02"},{"index":3,"sense_key":"basket%1:06:01::","synset_id":"basket.n.03"},{"index":4,"sense_key":"basket%1:04:00::","synset_id":"basket.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"basket%1:04:00::","call_ids":["semeval2013.d001.s005.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"basket%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":770,"cached_input_tokens":0,"output_tokens":50,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00204,"input_uncached_usd":0.00154,"input_cached_usd":0.0,"output_usd":0.0005,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7867830420145765}
{"item_id":"semeval2013.d001.s005.t001","gold_sense_keys":["perimeter%1:25:00::"],"candidates":[{"index":1,"sense_key":"perimeter%1:25:00::","synset_id":"margin.n.01"},{"index":2,"sense_key":"perimeter%1:25:01::","synset_id":"perimeter.n.02"},{"index":3,"sense_key":"perimeter%1:07:00::","synset_id":"circumference.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"perimeter%1:25:00::","call_ids":["semeval2013.d001.s005.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"perimeter%1:25:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":704,"cached_input_tokens":0,"output_tokens":88,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002288,"input_uncached_usd":0.001408,"input_cached_usd":0.0,"output_usd":0.00088,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.009718250017613}
{"item_id":"semeval2013.d001.s005.t002","gold_sense_keys":["speculation%1:09:00::"],"candidates":[{"index":1,"sense_key":"speculation%1:10:03::","synset_id":"guess.n.01"},{"index":2,"sense_key":"speculation%1:09:00::","synset_id":"speculation.n.02"},{"index":3,"sense_key":"speculation%1:21:00::","synset_id":"speculation.n.03"},{"index":4,"sense_key":"speculation%1:09:01::","synset_id":"meditation.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"speculation%1:09:00::","call_ids":["semeval2013.d001.s005.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"speculation%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":892,"cached_input_tokens":0,"output_tokens":143,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003214,"input_uncached_usd":0.001784,"input_cached_usd":0.0,"output_usd":0.00143,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2092458750121295}
{"item_id":"semeval2013.d001.s006.t000","gold_sense_keys":["scorer%1:18:00::"],"candidates":[{"index":1,"sense_key":"scorer%1:18:00::","synset_id":"scorer.n.01"},{"index":2,"sense_key":"scorer%1:18:01::","synset_id":"scorer.n.02"},{"index":3,"sense_key":"scorer%1:18:02::","synset_id":"scorekeeper.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"scorer%1:18:00::","call_ids":["semeval2013.d001.s006.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"scorer%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":750,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0016200000000000001,"input_uncached_usd":0.0015,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0763127909740433}
{"item_id":"semeval2013.d001.s006.t001","gold_sense_keys":["end%1:09:00::","end%1:11:00::","end%1:28:00::"],"candidates":[{"index":1,"sense_key":"end%1:15:00::","synset_id":"end.n.01"},{"index":2,"sense_key":"end%1:28:00::","synset_id":"end.n.02"},{"index":3,"sense_key":"end%1:11:00::","synset_id":"end.n.03"},{"index":4,"sense_key":"end%1:09:02::","synset_id":"goal.n.01"},{"index":5,"sense_key":"end%1:09:00::","synset_id":"end.n.05"},{"index":6,"sense_key":"end%1:26:00::","synset_id":"end.n.06"},{"index":7,"sense_key":"end%1:15:02::","synset_id":"end.n.07"},{"index":8,"sense_key":"end%1:18:00::","synset_id":"end.n.08"},{"index":9,"sense_key":"end%1:15:01::","synset_id":"end.n.09"},{"index":10,"sense_key":"end%1:15:03::","synset_id":"end.n.10"},{"index":11,"sense_key":"end%1:04:00::","synset_id":"end.n.11"},{"index":12,"sense_key":"end%1:10:00::","synset_id":"conclusion.n.08"},{"index":13,"sense_key":"end%1:06:02::","synset_id":"end.n.13"},{"index":14,"sense_key":"end%1:04:02::","synset_id":"end.n.14"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"end%1:28:00::","call_ids":["semeval2013.d001.s006.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"end%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1436,"cached_input_tokens":0,"output_tokens":125,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004122,"input_uncached_usd":0.002872,"input_cached_usd":0.0,"output_usd":0.00125,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0370969159994274}
{"item_id":"semeval2013.d001.s007.t000","gold_sense_keys":["quarter%1:28:01::"],"candidates":[{"index":1,"sense_key":"quarter%1:23:01::","synset_id":"one-fourth.n.01"},{"index":2,"sense_key":"quarter%1:15:00::","synset_id":"quarter.n.02"},{"index":3,"sense_key":"quarter%1:28:01::","synset_id":"quarter.n.03"},{"index":4,"sense_key":"quarter%1:28:03::","synset_id":"quarter.n.04"},{"index":5,"sense_key":"quarter%1:28:00::","synset_id":"quarter.n.05"},{"index":6,"sense_key":"quarter%1:28:02::","synset_id":"quarter.n.06"},{"index":7,"sense_key":"quarter%1:24:00::","synset_id":"quarter.n.07"},{"index":8,"sense_key":"quarter%1:23:02::","synset_id":"quarter.n.08"},{"index":9,"sense_key":"quarter%1:23:04::","synset_id":"quarter.n.09"},{"index":10,"sense_key":"quarter%1:21:00::","synset_id":"quarter.n.10"},{"index":11,"sense_key":"quarter%1:18:00::","synset_id":"quarter.n.11"},{"index":12,"sense_key":"quarter%1:06:00::","synset_id":"stern.n.01"},{"index":13,"sense_key":"quarter%1:06:01::","synset_id":"quarter.n.13"},{"index":14,"sense_key":"quarter%1:04:00::","synset_id":"quarter.n.14"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"quarter%1:28:01::","call_ids":["semeval2013.d001.s007.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"quarter%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1346,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002812,"input_uncached_usd":0.002692,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2628323340322822}
{"item_id":"semeval2013.d001.s007.t001","gold_sense_keys":["team%1:14:00::"],"candidates":[{"index":1,"sense_key":"team%1:14:00::","synset_id":"team.n.01"},{"index":2,"sense_key":"team%1:14:01::","synset_id":"team.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"team%1:14:00::","call_ids":["semeval2013.d001.s007.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"team%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":663,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001446,"input_uncached_usd":0.001326,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0173667080234736}
{"item_id":"semeval2013.d001.s007.t003","gold_sense_keys":["work%1:04:00::"],"candidates":[{"index":1,"sense_key":"work%1:04:00::","synset_id":"work.n.01"},{"index":2,"sense_key":"work%1:06:00::","synset_id":"work.n.02"},{"index":3,"sense_key":"work%1:04:01::","synset_id":"employment.n.02"},{"index":4,"sense_key":"work%1:09:00::","synset_id":"study.n.02"},{"index":5,"sense_key":"work%1:19:00::","synset_id":"work.n.05"},{"index":6,"sense_key":"work%1:06:01::","synset_id":"workplace.n.01"},{"index":7,"sense_key":"work%1:06:02::","synset_id":"oeuvre.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"work%1:04:00::","call_ids":["semeval2013.d001.s007.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"work%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1087,"cached_input_tokens":0,"output_tokens":70,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002874,"input_uncached_usd":0.0021739999999999997,"input_cached_usd":0.0,"output_usd":0.0007000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6738927500555292}
{"item_id":"semeval2013.d001.s007.t004","gold_sense_keys":["whistle%1:10:00::"],"candidates":[{"index":1,"sense_key":"whistle%1:11:00::","synset_id":"whistle.n.01"},{"index":2,"sense_key":"whistle%1:10:00::","synset_id":"whistle.n.02"},{"index":3,"sense_key":"whistle%1:06:02::","synset_id":"whistle.n.03"},{"index":4,"sense_key":"whistle%1:06:01::","synset_id":"whistle.n.04"},{"index":5,"sense_key":"whistle%1:06:00::","synset_id":"pennywhistle.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"whistle%1:11:00::","call_ids":["semeval2013.d001.s007.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"whistle%1:11:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":875,"cached_input_tokens":0,"output_tokens":123,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00298,"input_uncached_usd":0.0017499999999999998,"input_cached_usd":0.0,"output_usd":0.0012300000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5680068749934435}
{"item_id":"semeval2013.d001.s007.t006","gold_sense_keys":["return%1:04:00::","return%1:04:01::"],"candidates":[{"index":1,"sense_key":"return%1:10:01::","synset_id":"tax_return.n.01"},{"index":2,"sense_key":"return%1:04:01::","synset_id":"return.n.02"},{"index":3,"sense_key":"return%1:11:02::","synset_id":"return.n.03"},{"index":4,"sense_key":"return%1:04:02::","synset_id":"restitution.n.03"},{"index":5,"sense_key":"return%1:04:00::","synset_id":"return.n.05"},{"index":6,"sense_key":"return%1:21:00::","synset_id":"return.n.06"},{"index":7,"sense_key":"return%1:11:01::","synset_id":"recurrence.n.01"},{"index":8,"sense_key":"return%1:10:02::","synset_id":"rejoinder.n.01"},{"index":9,"sense_key":"return%1:06:00::","synset_id":"return_key.n.01"},{"index":10,"sense_key":"return%1:04:06::","synset_id":"return.n.10"},{"index":11,"sense_key":"return%1:04:04::","synset_id":"return.n.11"},{"index":12,"sense_key":"return%1:04:05::","synset_id":"return.n.12"},{"index":13,"sense_key":"return%1:04:07::","synset_id":"reappearance.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"return%1:04:01::","call_ids":["semeval2013.d001.s007.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"return%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1393,"cached_input_tokens":0,"output_tokens":83,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003616,"input_uncached_usd":0.002786,"input_cached_usd":0.0,"output_usd":0.0008300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3390662500169128}
{"item_id":"semeval2013.d001.s007.t007","gold_sense_keys":["home%1:15:02::"],"candidates":[{"index":1,"sense_key":"home%1:15:04::","synset_id":"home.n.01"},{"index":2,"sense_key":"home%1:06:00::","synset_id":"dwelling.n.01"},{"index":3,"sense_key":"home%1:15:02::","synset_id":"home.n.03"},{"index":4,"sense_key":"home%1:06:02::","synset_id":"home_plate.n.01"},{"index":5,"sense_key":"home%1:15:05::","synset_id":"base.n.14"},{"index":6,"sense_key":"home%1:15:03::","synset_id":"home.n.06"},{"index":7,"sense_key":"home%1:26:00::","synset_id":"home.n.07"},{"index":8,"sense_key":"home%1:14:00::","synset_id":"family.n.01"},{"index":9,"sense_key":"home%1:06:01::","synset_id":"home.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"home%1:15:02::","call_ids":["semeval2013.d001.s007.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"home%1:15:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1151,"cached_input_tokens":0,"output_tokens":115,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003452,"input_uncached_usd":0.002302,"input_cached_usd":0.0,"output_usd":0.0011500000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.720981791964732}
{"item_id":"semeval2013.d001.s007.t008","gold_sense_keys":["lead%1:23:00::"],"candidates":[{"index":1,"sense_key":"lead%1:07:02::","synset_id":"lead.n.01"},{"index":2,"sense_key":"lead%1:27:00::","synset_id":"lead.n.02"},{"index":3,"sense_key":"lead%1:09:00::","synset_id":"lead.n.03"},{"index":4,"sense_key":"lead%1:04:01::","synset_id":"lead.n.04"},{"index":5,"sense_key":"lead%1:25:00::","synset_id":"lead.n.05"},{"index":6,"sense_key":"lead%1:10:01::","synset_id":"lead.n.06"},{"index":7,"sense_key":"lead%1:23:00::","synset_id":"lead.n.07"},{"index":8,"sense_key":"lead%1:18:00::","synset_id":"star.n.04"},{"index":9,"sense_key":"lead%1:15:00::","synset_id":"lead.n.09"},{"index":10,"sense_key":"lead%1:10:02::","synset_id":"tip.n.03"},{"index":11,"sense_key":"lead%1:10:00::","synset_id":"lead.n.11"},{"index":12,"sense_key":"lead%1:07:01::","synset_id":"spark_advance.n.01"},{"index":13,"sense_key":"lead%1:06:03::","synset_id":"leash.n.01"},{"index":14,"sense_key":"lead%1:06:02::","synset_id":"lead.n.14"},{"index":15,"sense_key":"lead%1:06:00::","synset_id":"lead.n.15"},{"index":16,"sense_key":"lead%1:06:01::","synset_id":"jumper_cable.n.01"},{"index":17,"sense_key":"lead%1:04:00::","synset_id":"lead.n.17"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"lead%1:23:00::","call_ids":["semeval2013.d001.s007.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"lead%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1602,"cached_input_tokens":0,"output_tokens":69,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003894,"input_uncached_usd":0.003204,"input_cached_usd":0.0,"output_usd":0.0006900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.322177332942374}
{"item_id":"semeval2013.d001.s008.t000","gold_sense_keys":["team%1:14:00::"],"candidates":[{"index":1,"sense_key":"team%1:14:00::","synset_id":"team.n.01"},{"index":2,"sense_key":"team%1:14:01::","synset_id":"team.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"team%1:14:00::","call_ids":["semeval2013.d001.s008.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"team%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":674,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0014680000000000001,"input_uncached_usd":0.001348,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.087972541921772}
{"item_id":"semeval2013.d001.s008.t001","gold_sense_keys":["point%1:23:01::"],"candidates":[{"index":1,"sense_key":"point%1:09:00::","synset_id":"point.n.01"},{"index":2,"sense_key":"point%1:15:00::","synset_id":"point.n.02"},{"index":3,"sense_key":"point%1:10:01::","synset_id":"point.n.03"},{"index":4,"sense_key":"point%1:09:01::","synset_id":"detail.n.01"},{"index":5,"sense_key":"point%1:26:00::","synset_id":"degree.n.02"},{"index":6,"sense_key":"point%1:28:00::","synset_id":"point.n.06"},{"index":7,"sense_key":"point%1:09:02::","synset_id":"point.n.07"},{"index":8,"sense_key":"point%1:25:00::","synset_id":"point.n.08"},{"index":9,"sense_key":"point%1:25:02::","synset_id":"point.n.09"},{"index":10,"sense_key":"point%1:23:01::","synset_id":"point.n.10"},{"index":11,"sense_key":"point%1:17:02::","synset_id":"point.n.11"},{"index":12,"sense_key":"point%1:10:03::","synset_id":"item.n.01"},{"index":13,"sense_key":"point%1:24:04::","synset_id":"point.n.13"},{"index":14,"sense_key":"point%1:07:01::","synset_id":"point.n.14"},{"index":15,"sense_key":"point%1:06:00::","synset_id":"point.n.15"},{"index":16,"sense_key":"point%1:24:00::","synset_id":"compass_point.n.01"},{"index":17,"sense_key":"point%1:23:00::","synset_id":"point.n.17"},{"index":18,"sense_key":"point%1:21:00::","synset_id":"point.n.18"},{"index":19,"sense_key":"point%1:10:00::","synset_id":"period.n.07"},{"index":20,"sense_key":"point%1:10:02::","synset_id":"point.n.20"},{"index":21,"sense_key":"point%1:10:04::","synset_id":"decimal_point.n.01"},{"index":22,"sense_key":"point%1:07:00::","synset_id":"point.n.22"},{"index":23,"sense_key":"point%1:07:02::","synset_id":"point.n.23"},{"index":24,"sense_key":"point%1:06:02::","synset_id":"point.n.24"},{"index":25,"sense_key":"point%1:06:01::","synset_id":"point.n.25"},{"index":26,"sense_key":"point%1:06:03::","synset_id":"distributor_point.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"point%1:23:01::","call_ids":["semeval2013.d001.s008.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"point%1:23:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2075,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00427,"input_uncached_usd":0.00415,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4516348340548575}
{"item_id":"semeval2013.d001.s008.t002","gold_sense_keys":["point%1:23:01::"],"candidates":[{"index":1,"sense_key":"point%1:09:00::","synset_id":"point.n.01"},{"index":2,"sense_key":"point%1:15:00::","synset_id":"point.n.02"},{"index":3,"sense_key":"point%1:10:01::","synset_id":"point.n.03"},{"index":4,"sense_key":"point%1:09:01::","synset_id":"detail.n.01"},{"index":5,"sense_key":"point%1:26:00::","synset_id":"degree.n.02"},{"index":6,"sense_key":"point%1:28:00::","synset_id":"point.n.06"},{"index":7,"sense_key":"point%1:09:02::","synset_id":"point.n.07"},{"index":8,"sense_key":"point%1:25:00::","synset_id":"point.n.08"},{"index":9,"sense_key":"point%1:25:02::","synset_id":"point.n.09"},{"index":10,"sense_key":"point%1:23:01::","synset_id":"point.n.10"},{"index":11,"sense_key":"point%1:17:02::","synset_id":"point.n.11"},{"index":12,"sense_key":"point%1:10:03::","synset_id":"item.n.01"},{"index":13,"sense_key":"point%1:24:04::","synset_id":"point.n.13"},{"index":14,"sense_key":"point%1:07:01::","synset_id":"point.n.14"},{"index":15,"sense_key":"point%1:06:00::","synset_id":"point.n.15"},{"index":16,"sense_key":"point%1:24:00::","synset_id":"compass_point.n.01"},{"index":17,"sense_key":"point%1:23:00::","synset_id":"point.n.17"},{"index":18,"sense_key":"point%1:21:00::","synset_id":"point.n.18"},{"index":19,"sense_key":"point%1:10:00::","synset_id":"period.n.07"},{"index":20,"sense_key":"point%1:10:02::","synset_id":"point.n.20"},{"index":21,"sense_key":"point%1:10:04::","synset_id":"decimal_point.n.01"},{"index":22,"sense_key":"point%1:07:00::","synset_id":"point.n.22"},{"index":23,"sense_key":"point%1:07:02::","synset_id":"point.n.23"},{"index":24,"sense_key":"point%1:06:02::","synset_id":"point.n.24"},{"index":25,"sense_key":"point%1:06:01::","synset_id":"point.n.25"},{"index":26,"sense_key":"point%1:06:03::","synset_id":"distributor_point.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"point%1:23:01::","call_ids":["semeval2013.d001.s008.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"point%1:23:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2075,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00427,"input_uncached_usd":0.00415,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1225697919726372}
{"item_id":"semeval2013.d001.s008.t003","gold_sense_keys":["night%1:28:03::"],"candidates":[{"index":1,"sense_key":"night%1:28:00::","synset_id":"night.n.01"},{"index":2,"sense_key":"night%1:28:01::","synset_id":"night.n.02"},{"index":3,"sense_key":"night%1:28:04::","synset_id":"night.n.03"},{"index":4,"sense_key":"night%1:28:02::","synset_id":"night.n.04"},{"index":5,"sense_key":"night%1:26:00::","synset_id":"night.n.05"},{"index":6,"sense_key":"night%1:28:05::","synset_id":"night.n.06"},{"index":7,"sense_key":"night%1:28:03::","synset_id":"night.n.07"},{"index":8,"sense_key":"night%1:18:00::","synset_id":"nox.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"night%1:28:02::","call_ids":["semeval2013.d001.s008.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"night%1:28:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":956,"cached_input_tokens":0,"output_tokens":138,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003292,"input_uncached_usd":0.0019119999999999999,"input_cached_usd":0.0,"output_usd":0.0013800000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8431484999600798}
{"item_id":"semeval2013.d001.s009.t000","gold_sense_keys":["game%1:04:03::"],"candidates":[{"index":1,"sense_key":"game%1:04:00::","synset_id":"game.n.01"},{"index":2,"sense_key":"game%1:04:03::","synset_id":"game.n.02"},{"index":3,"sense_key":"game%1:04:01::","synset_id":"game.n.03"},{"index":4,"sense_key":"game%1:05:00::","synset_id":"game.n.04"},{"index":5,"sense_key":"game%1:28:00::","synset_id":"game.n.05"},{"index":6,"sense_key":"game%1:23:00::","synset_id":"game.n.06"},{"index":7,"sense_key":"game%1:13:00::","synset_id":"game.n.07"},{"index":8,"sense_key":"game%1:09:00::","synset_id":"plot.n.01"},{"index":9,"sense_key":"game%1:06:00::","synset_id":"game.n.09"},{"index":10,"sense_key":"game%1:04:02::","synset_id":"game.n.10"},{"index":11,"sense_key":"game%1:04:04::","synset_id":"game.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"game%1:04:03::","call_ids":["semeval2013.d001.s009.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"game%1:04:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1138,"cached_input_tokens":0,"output_tokens":79,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0030659999999999997,"input_uncached_usd":0.002276,"input_cached_usd":0.0,"output_usd":0.00079,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.6652372089447454}
{"item_id":"semeval2013.d001.s009.t003","gold_sense_keys":["pressure%1:07:00::"],"candidates":[{"index":1,"sense_key":"pressure%1:19:00::","synset_id":"pressure.n.01"},{"index":2,"sense_key":"pressure%1:07:00::","synset_id":"pressure.n.02"},{"index":3,"sense_key":"pressure%1:04:00::","synset_id":"press.n.09"},{"index":4,"sense_key":"pressure%1:26:00::","synset_id":"imperativeness.n.01"},{"index":5,"sense_key":"pressure%1:09:00::","synset_id":"pressure.n.05"},{"index":6,"sense_key":"pressure%1:26:01::","synset_id":"pressure.n.06"},{"index":7,"sense_key":"pressure%1:19:01::","synset_id":"atmospheric_pressure.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"pressure%1:07:00::","call_ids":["semeval2013.d001.s009.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"pressure%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1055,"cached_input_tokens":0,"output_tokens":115,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00326,"input_uncached_usd":0.00211,"input_cached_usd":0.0,"output_usd":0.0011500000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8773098749807104}
{"item_id":"semeval2013.d001.s009.t004","gold_sense_keys":["game%1:04:03::"],"candidates":[{"index":1,"sense_key":"game%1:04:00::","synset_id":"game.n.01"},{"index":2,"sense_key":"game%1:04:03::","synset_id":"game.n.02"},{"index":3,"sense_key":"game%1:04:01::","synset_id":"game.n.03"},{"index":4,"sense_key":"game%1:05:00::","synset_id":"game.n.04"},{"index":5,"sense_key":"game%1:28:00::","synset_id":"game.n.05"},{"index":6,"sense_key":"game%1:23:00::","synset_id":"game.n.06"},{"index":7,"sense_key":"game%1:13:00::","synset_id":"game.n.07"},{"index":8,"sense_key":"game%1:09:00::","synset_id":"plot.n.01"},{"index":9,"sense_key":"game%1:06:00::","synset_id":"game.n.09"},{"index":10,"sense_key":"game%1:04:02::","synset_id":"game.n.10"},{"index":11,"sense_key":"game%1:04:04::","synset_id":"game.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"game%1:04:03::","call_ids":["semeval2013.d001.s009.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"game%1:04:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1138,"cached_input_tokens":0,"output_tokens":85,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003126,"input_uncached_usd":0.002276,"input_cached_usd":0.0,"output_usd":0.0008500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3174674999900162}
{"item_id":"semeval2013.d001.s010.t000","gold_sense_keys":["role%1:04:00::"],"candidates":[{"index":1,"sense_key":"role%1:04:00::","synset_id":"function.n.03"},{"index":2,"sense_key":"role%1:09:00::","synset_id":"character.n.04"},{"index":3,"sense_key":"role%1:07:00::","synset_id":"function.n.02"},{"index":4,"sense_key":"role%1:04:01::","synset_id":"role.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"role%1:04:00::","call_ids":["semeval2013.d001.s010.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"role%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":812,"cached_input_tokens":0,"output_tokens":191,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0035340000000000002,"input_uncached_usd":0.001624,"input_cached_usd":0.0,"output_usd":0.0019100000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.5911952080205083}
{"item_id":"semeval2013.d001.s012.t000","gold_sense_keys":["player%1:18:01::"],"candidates":[{"index":1,"sense_key":"player%1:18:01::","synset_id":"player.n.01"},{"index":2,"sense_key":"player%1:18:02::","synset_id":"musician.n.01"},{"index":3,"sense_key":"player%1:18:00::","synset_id":"actor.n.01"},{"index":4,"sense_key":"player%1:18:04::","synset_id":"player.n.04"},{"index":5,"sense_key":"player%1:18:03::","synset_id":"player.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"player%1:18:01::","call_ids":["semeval2013.d001.s012.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"player%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":914,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001948,"input_uncached_usd":0.001828,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1286550000077114}
{"item_id":"semeval2013.d001.s012.t001","gold_sense_keys":["couple%1:23:01::"],"candidates":[{"index":1,"sense_key":"couple%1:14:00::","synset_id":"couple.n.01"},{"index":2,"sense_key":"couple%1:14:01::","synset_id":"couple.n.02"},{"index":3,"sense_key":"couple%1:23:01::","synset_id":"couple.n.03"},{"index":4,"sense_key":"couple%1:23:00::","synset_id":"couple.n.04"},{"index":5,"sense_key":"couple%1:17:00::","synset_id":"couple.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"couple%1:23:01::","call_ids":["semeval2013.d001.s012.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"couple%1:23:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":941,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0024619999999999998,"input_uncached_usd":0.001882,"input_cached_usd":0.0,"output_usd":0.00058,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3199429999804124}
{"item_id":"semeval2013.d001.s012.t002","gold_sense_keys":["play%1:04:05::"],"candidates":[{"index":1,"sense_key":"play%1:10:01::","synset_id":"play.n.01"},{"index":2,"sense_key":"play%1:10:03::","synset_id":"play.n.02"},{"index":3,"sense_key":"play%1:04:05::","synset_id":"play.n.03"},{"index":4,"sense_key":"play%1:04:04::","synset_id":"maneuver.n.03"},{"index":5,"sense_key":"play%1:26:00::","synset_id":"play.n.05"},{"index":6,"sense_key":"play%1:04:08::","synset_id":"play.n.06"},{"index":7,"sense_key":"play%1:04:03::","synset_id":"bid.n.02"},{"index":8,"sense_key":"play%1:04:06::","synset_id":"play.n.08"},{"index":9,"sense_key":"play%1:28:00::","synset_id":"playing_period.n.01"},{"index":10,"sense_key":"play%1:26:01::","synset_id":"free_rein.n.01"},{"index":11,"sense_key":"play%1:11:00::","synset_id":"shimmer.n.01"},{"index":12,"sense_key":"play%1:10:02::","synset_id":"fun.n.02"},{"index":13,"sense_key":"play%1:07:00::","synset_id":"looseness.n.05"},{"index":14,"sense_key":"play%1:04:00::","synset_id":"play.n.14"},{"index":15,"sense_key":"play%1:04:01::","synset_id":"turn.n.03"},{"index":16,"sense_key":"play%1:04:02::","synset_id":"gambling.n.01"},{"index":17,"sense_key":"play%1:04:07::","synset_id":"play.n.17"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"play%1:04:05::","call_ids":["semeval2013.d001.s012.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"play%1:04:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1786,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004131999999999999,"input_uncached_usd":0.0035719999999999997,"input_cached_usd":0.0,"output_usd":0.0005600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3050002920208499}
{"item_id":"semeval2013.d001.s012.t003","gold_sense_keys":["help%1:04:00::"],"candidates":[{"index":1,"sense_key":"help%1:04:00::","synset_id":"aid.n.02"},{"index":2,"sense_key":"help%1:18:00::","synset_id":"assistant.n.01"},{"index":3,"sense_key":"help%1:07:00::","synset_id":"aid.n.01"},{"index":4,"sense_key":"help%1:07:01::","synset_id":"avail.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"help%1:04:00::","call_ids":["semeval2013.d001.s012.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"help%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":957,"cached_input_tokens":0,"output_tokens":108,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002994,"input_uncached_usd":0.001914,"input_cached_usd":0.0,"output_usd":0.00108,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6471235830103979}
{"item_id":"semeval2013.d001.s012.t005","gold_sense_keys":["lead%1:23:00::"],"candidates":[{"index":1,"sense_key":"lead%1:07:02::","synset_id":"lead.n.01"},{"index":2,"sense_key":"lead%1:27:00::","synset_id":"lead.n.02"},{"index":3,"sense_key":"lead%1:09:00::","synset_id":"lead.n.03"},{"index":4,"sense_key":"lead%1:04:01::","synset_id":"lead.n.04"},{"index":5,"sense_key":"lead%1:25:00::","synset_id":"lead.n.05"},{"index":6,"sense_key":"lead%1:10:01::","synset_id":"lead.n.06"},{"index":7,"sense_key":"lead%1:23:00::","synset_id":"lead.n.07"},{"index":8,"sense_key":"lead%1:18:00::","synset_id":"star.n.04"},{"index":9,"sense_key":"lead%1:15:00::","synset_id":"lead.n.09"},{"index":10,"sense_key":"lead%1:10:02::","synset_id":"tip.n.03"},{"index":11,"sense_key":"lead%1:10:00::","synset_id":"lead.n.11"},{"index":12,"sense_key":"lead%1:07:01::","synset_id":"spark_advance.n.01"},{"index":13,"sense_key":"lead%1:06:03::","synset_id":"leash.n.01"},{"index":14,"sense_key":"lead%1:06:02::","synset_id":"lead.n.14"},{"index":15,"sense_key":"lead%1:06:00::","synset_id":"lead.n.15"},{"index":16,"sense_key":"lead%1:06:01::","synset_id":"jumper_cable.n.01"},{"index":17,"sense_key":"lead%1:04:00::","synset_id":"lead.n.17"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"lead%1:23:00::","call_ids":["semeval2013.d001.s012.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"lead%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1695,"cached_input_tokens":0,"output_tokens":103,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00442,"input_uncached_usd":0.00339,"input_cached_usd":0.0,"output_usd":0.00103,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5398397909011692}
{"item_id":"semeval2013.d001.s012.t006","gold_sense_keys":["team%1:14:00::"],"candidates":[{"index":1,"sense_key":"team%1:14:00::","synset_id":"team.n.01"},{"index":2,"sense_key":"team%1:14:01::","synset_id":"team.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"team%1:14:00::","call_ids":["semeval2013.d001.s012.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"team%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":756,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001632,"input_uncached_usd":0.0015119999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1452686249976978}
{"item_id":"semeval2013.d001.s012.t007","gold_sense_keys":["end%1:09:00::","end%1:11:00::","end%1:28:00::"],"candidates":[{"index":1,"sense_key":"end%1:15:00::","synset_id":"end.n.01"},{"index":2,"sense_key":"end%1:28:00::","synset_id":"end.n.02"},{"index":3,"sense_key":"end%1:11:00::","synset_id":"end.n.03"},{"index":4,"sense_key":"end%1:09:02::","synset_id":"goal.n.01"},{"index":5,"sense_key":"end%1:09:00::","synset_id":"end.n.05"},{"index":6,"sense_key":"end%1:26:00::","synset_id":"end.n.06"},{"index":7,"sense_key":"end%1:15:02::","synset_id":"end.n.07"},{"index":8,"sense_key":"end%1:18:00::","synset_id":"end.n.08"},{"index":9,"sense_key":"end%1:15:01::","synset_id":"end.n.09"},{"index":10,"sense_key":"end%1:15:03::","synset_id":"end.n.10"},{"index":11,"sense_key":"end%1:04:00::","synset_id":"end.n.11"},{"index":12,"sense_key":"end%1:10:00::","synset_id":"conclusion.n.08"},{"index":13,"sense_key":"end%1:06:02::","synset_id":"end.n.13"},{"index":14,"sense_key":"end%1:04:02::","synset_id":"end.n.14"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"end%1:28:00::","call_ids":["semeval2013.d001.s012.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"end%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1487,"cached_input_tokens":0,"output_tokens":130,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004274,"input_uncached_usd":0.002974,"input_cached_usd":0.0,"output_usd":0.0013000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7328460419084877}
{"item_id":"semeval2013.d001.s012.t008","gold_sense_keys":["quarter%1:28:01::"],"candidates":[{"index":1,"sense_key":"quarter%1:23:01::","synset_id":"one-fourth.n.01"},{"index":2,"sense_key":"quarter%1:15:00::","synset_id":"quarter.n.02"},{"index":3,"sense_key":"quarter%1:28:01::","synset_id":"quarter.n.03"},{"index":4,"sense_key":"quarter%1:28:03::","synset_id":"quarter.n.04"},{"index":5,"sense_key":"quarter%1:28:00::","synset_id":"quarter.n.05"},{"index":6,"sense_key":"quarter%1:28:02::","synset_id":"quarter.n.06"},{"index":7,"sense_key":"quarter%1:24:00::","synset_id":"quarter.n.07"},{"index":8,"sense_key":"quarter%1:23:02::","synset_id":"quarter.n.08"},{"index":9,"sense_key":"quarter%1:23:04::","synset_id":"quarter.n.09"},{"index":10,"sense_key":"quarter%1:21:00::","synset_id":"quarter.n.10"},{"index":11,"sense_key":"quarter%1:18:00::","synset_id":"quarter.n.11"},{"index":12,"sense_key":"quarter%1:06:00::","synset_id":"stern.n.01"},{"index":13,"sense_key":"quarter%1:06:01::","synset_id":"quarter.n.13"},{"index":14,"sense_key":"quarter%1:04:00::","synset_id":"quarter.n.14"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"quarter%1:28:01::","call_ids":["semeval2013.d001.s012.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"quarter%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1439,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002998,"input_uncached_usd":0.002878,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.315097791957669}
{"item_id":"semeval2013.d001.s013.t001","gold_sense_keys":["minute%1:28:00::"],"candidates":[{"index":1,"sense_key":"minute%1:28:00::","synset_id":"minute.n.01"},{"index":2,"sense_key":"minute%1:28:01::","synset_id":"moment.n.02"},{"index":3,"sense_key":"minute%1:28:02::","synset_id":"moment.n.01"},{"index":4,"sense_key":"minute%1:23:00::","synset_id":"minute.n.04"},{"index":5,"sense_key":"minute%1:10:00::","synset_id":"minute.n.05"},{"index":6,"sense_key":"minute%1:07:00::","synset_id":"hour.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"minute%1:28:00::","call_ids":["semeval2013.d001.s013.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"minute%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1001,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0021219999999999998,"input_uncached_usd":0.002002,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9310114999534562}
{"item_id":"semeval2013.d001.s014.t000","gold_sense_keys":["team%1:14:00::"],"candidates":[{"index":1,"sense_key":"team%1:14:00::","synset_id":"team.n.01"},{"index":2,"sense_key":"team%1:14:01::","synset_id":"team.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"team%1:14:00::","call_ids":["semeval2013.d001.s014.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"team%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":738,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001596,"input_uncached_usd":0.0014759999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.7944540829630569}
{"item_id":"semeval2013.d001.s014.t001","gold_sense_keys":["team%1:14:00::"],"candidates":[{"index":1,"sense_key":"team%1:14:00::","synset_id":"team.n.01"},{"index":2,"sense_key":"team%1:14:01::","synset_id":"team.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"team%1:14:00::","call_ids":["semeval2013.d001.s014.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"team%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":738,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001596,"input_uncached_usd":0.0014759999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1894537919433787}
{"item_id":"semeval2013.d001.s014.t002","gold_sense_keys":["line%1:15:01::"],"candidates":[{"index":1,"sense_key":"line%1:14:03::","synset_id":"line.n.01"},{"index":2,"sense_key":"line%1:10:01::","synset_id":"line.n.02"},{"index":3,"sense_key":"line%1:14:01::","synset_id":"line.n.03"},{"index":4,"sense_key":"line%1:25:00::","synset_id":"line.n.04"},{"index":5,"sense_key":"line%1:10:02::","synset_id":"line.n.05"},{"index":6,"sense_key":"line%1:19:00::","synset_id":"line.n.06"},{"index":7,"sense_key":"line%1:15:02::","synset_id":"line.n.07"},{"index":8,"sense_key":"line%1:09:01::","synset_id":"argumentation.n.02"},{"index":9,"sense_key":"line%1:06:01::","synset_id":"cable.n.02"},{"index":10,"sense_key":"line%1:14:02::","synset_id":"course.n.02"},{"index":11,"sense_key":"line%1:15:00::","synset_id":"line.n.11"},{"index":12,"sense_key":"line%1:25:01::","synset_id":"wrinkle.n.01"},{"index":13,"sense_key":"line%1:06:06::","synset_id":"pipeline.n.02"},{"index":14,"sense_key":"line%1:06:03::","synset_id":"line.n.14"},{"index":15,"sense_key":"line%1:06:07::","synset_id":"telephone_line.n.02"},{"index":16,"sense_key":"line%1:04:01::","synset_id":"line.n.16"},{"index":17,"sense_key":"line%1:14:00::","synset_id":"lineage.n.01"},{"index":18,"sense_key":"line%1:06:00::","synset_id":"line.n.18"},{"index":19,"sense_key":"line%1:04:00::","synset_id":"occupation.n.01"},{"index":20,"sense_key":"line%1:15:01::","synset_id":"line.n.20"},{"index":21,"sense_key":"line%1:10:05::","synset_id":"channel.n.05"},{"index":22,"sense_key":"line%1:06:08::","synset_id":"line.n.22"},{"index":23,"sense_key":"line%1:06:05::","synset_id":"line.n.23"},{"index":24,"sense_key":"line%1:23:00::","synset_id":"agate_line.n.01"},{"index":25,"sense_key":"line%1:21:00::","synset_id":"credit_line.n.01"},{"index":26,"sense_key":"line%1:10:03::","synset_id":"tune.n.01"},{"index":27,"sense_key":"line%1:10:06::","synset_id":"line.n.27"},{"index":28,"sense_key":"line%1:10:04::","synset_id":"note.n.02"},{"index":29,"sense_key":"line%1:09:00::","synset_id":"line.n.29"},{"index":30,"sense_key":"line%1:06:09::","synset_id":"production_line.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":20,"chosen_sense_key":"line%1:15:01::","call_ids":["semeval2013.d001.s014.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":20,"predicted_sense_key":"line%1:15:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2494,"cached_input_tokens":0,"output_tokens":70,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0056879999999999995,"input_uncached_usd":0.004987999999999999,"input_cached_usd":0.0,"output_usd":0.0007000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7644927500514314}
{"item_id":"semeval2013.d001.s015.t001","gold_sense_keys":["team%1:14:00::"],"candidates":[{"index":1,"sense_key":"team%1:14:00::","synset_id":"team.n.01"},{"index":2,"sense_key":"team%1:14:01::","synset_id":"team.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"team%1:14:00::","call_ids":["semeval2013.d001.s015.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"team%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":657,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001434,"input_uncached_usd":0.0013139999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3388826670125127}
{"item_id":"semeval2013.d001.s015.t003","gold_sense_keys":["game%1:04:03::"],"candidates":[{"index":1,"sense_key":"game%1:04:00::","synset_id":"game.n.01"},{"index":2,"sense_key":"game%1:04:03::","synset_id":"game.n.02"},{"index":3,"sense_key":"game%1:04:01::","synset_id":"game.n.03"},{"index":4,"sense_key":"game%1:05:00::","synset_id":"game.n.04"},{"index":5,"sense_key":"game%1:28:00::","synset_id":"game.n.05"},{"index":6,"sense_key":"game%1:23:00::","synset_id":"game.n.06"},{"index":7,"sense_key":"game%1:13:00::","synset_id":"game.n.07"},{"index":8,"sense_key":"game%1:09:00::","synset_id":"plot.n.01"},{"index":9,"sense_key":"game%1:06:00::","synset_id":"game.n.09"},{"index":10,"sense_key":"game%1:04:02::","synset_id":"game.n.10"},{"index":11,"sense_key":"game%1:04:04::","synset_id":"game.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"game%1:04:03::","call_ids":["semeval2013.d001.s015.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"game%1:04:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1133,"cached_input_tokens":0,"output_tokens":74,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003006,"input_uncached_usd":0.002266,"input_cached_usd":0.0,"output_usd":0.0007400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3944972079480067}
{"item_id":"semeval2013.d001.s015.t005","gold_sense_keys":["time%1:11:00::"],"candidates":[{"index":1,"sense_key":"time%1:11:00::","synset_id":"time.n.01"},{"index":2,"sense_key":"time%1:28:05::","synset_id":"time.n.02"},{"index":3,"sense_key":"time%1:28:00::","synset_id":"time.n.03"},{"index":4,"sense_key":"time%1:28:06::","synset_id":"time.n.04"},{"index":5,"sense_key":"time%1:03:00::","synset_id":"time.n.05"},{"index":6,"sense_key":"time%1:11:01::","synset_id":"time.n.06"},{"index":7,"sense_key":"time%1:28:03::","synset_id":"clock_time.n.01"},{"index":8,"sense_key":"time%1:28:04::","synset_id":"fourth_dimension.n.01"},{"index":9,"sense_key":"time%1:07:01::","synset_id":"meter.n.04"},{"index":10,"sense_key":"time%1:28:01::","synset_id":"prison_term.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"time%1:11:00::","call_ids":["semeval2013.d001.s015.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"time%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1150,"cached_input_tokens":0,"output_tokens":46,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00276,"input_uncached_usd":0.0023,"input_cached_usd":0.0,"output_usd":0.00046,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.526272707968019}
{"item_id":"semeval2013.d001.s015.t006","gold_sense_keys":["season%1:28:02::"],"candidates":[{"index":1,"sense_key":"season%1:28:02::","synset_id":"season.n.01"},{"index":2,"sense_key":"season%1:28:00::","synset_id":"season.n.02"},{"index":3,"sense_key":"season%1:28:01::","synset_id":"season.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"season%1:28:02::","call_ids":["semeval2013.d001.s015.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"season%1:28:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":774,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002188,"input_uncached_usd":0.001548,"input_cached_usd":0.0,"output_usd":0.00064,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6415622079512104}
{"item_id":"semeval2013.d002.s000.t000","gold_sense_keys":["wall_street%1:14:00::"],"candidates":[{"index":1,"sense_key":"wall_street%1:15:00::","synset_id":"wall_street.n.01"},{"index":2,"sense_key":"wall_street%1:14:00::","synset_id":"wall_street.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"wall_street%1:14:00::","call_ids":["semeval2013.d002.s000.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"wall_street%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":454,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001498,"input_uncached_usd":0.000908,"input_cached_usd":0.0,"output_usd":0.00059,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5155977919930592}
{"item_id":"semeval2013.d002.s000.t002","gold_sense_keys":["dollar%1:23:00::"],"candidates":[{"index":1,"sense_key":"dollar%1:23:00::","synset_id":"dollar.n.01"},{"index":2,"sense_key":"dollar%1:21:00::","synset_id":"dollar.n.02"},{"index":3,"sense_key":"dollar%1:21:01::","synset_id":"dollar.n.03"},{"index":4,"sense_key":"dollar%1:10:00::","synset_id":"dollar.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"dollar%1:23:00::","call_ids":["semeval2013.d002.s000.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"dollar%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":572,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0012640000000000001,"input_uncached_usd":0.001144,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6027277919929475}
{"item_id":"semeval2013.d002.s001.t001","gold_sense_keys":["heading%1:15:00::"],"candidates":[{"index":1,"sense_key":"heading%1:10:00::","synset_id":"heading.n.01"},{"index":2,"sense_key":"heading%1:15:00::","synset_id":"bearing.n.02"},{"index":3,"sense_key":"heading%1:06:00::","synset_id":"drift.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"heading%1:15:00::","call_ids":["semeval2013.d002.s001.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"heading%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":605,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00179,"input_uncached_usd":0.00121,"input_cached_usd":0.0,"output_usd":0.00058,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2858607090311125}
{"item_id":"semeval2013.d002.s001.t003","gold_sense_keys":["market%1:14:01::"],"candidates":[{"index":1,"sense_key":"market%1:04:00::","synset_id":"market.n.01"},{"index":2,"sense_key":"market%1:14:00::","synset_id":"market.n.02"},{"index":3,"sense_key":"market%1:06:00::","synset_id":"grocery_store.n.01"},{"index":4,"sense_key":"market%1:14:01::","synset_id":"market.n.04"},{"index":5,"sense_key":"market%1:06:01::","synset_id":"marketplace.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"market%1:14:01::","call_ids":["semeval2013.d002.s001.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"market%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":712,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001544,"input_uncached_usd":0.001424,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1842960839858279}
{"item_id":"semeval2013.d002.s001.t004","gold_sense_keys":["indicator%1:10:00::"],"candidates":[{"index":1,"sense_key":"indicator%1:10:00::","synset_id":"index.n.02"},{"index":2,"sense_key":"indicator%1:10:01::","synset_id":"indicator.n.02"},{"index":3,"sense_key":"indicator%1:06:00::","synset_id":"indicator.n.03"},{"index":4,"sense_key":"indicator%1:27:00::","synset_id":"indicator.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"indicator%1:10:00::","call_ids":["semeval2013.d002.s001.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"indicator%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":649,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001418,"input_uncached_usd":0.001298,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5897318749921396}
{"item_id":"semeval2013.d002.s001.t005","gold_sense_keys":["united_states%1:15:00::"],"candidates":[{"index":1,"sense_key":"united_states%1:15:00::","synset_id":"united_states.n.01"},{"index":2,"sense_key":"united_states%1:14:00::","synset_id":"united_states_government.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"united_states%1:15:00::","call_ids":["semeval2013.d002.s001.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"united_states%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":590,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00184,"input_uncached_usd":0.0011799999999999998,"input_cached_usd":0.0,"output_usd":0.0006600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3129518330097198}
{"item_id":"semeval2013.d002.s001.t006","gold_sense_keys":["dollar%1:23:00::"],"candidates":[{"index":1,"sense_key":"dollar%1:23:00::","synset_id":"dollar.n.01"},{"index":2,"sense_key":"dollar%1:21:00::","synset_id":"dollar.n.02"},{"index":3,"sense_key":"dollar%1:21:01::","synset_id":"dollar.n.03"},{"index":4,"sense_key":"dollar%1:10:00::","synset_id":"dollar.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"dollar%1:23:00::","call_ids":["semeval2013.d002.s001.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"dollar%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":651,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001422,"input_uncached_usd":0.001302,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9264856670051813}
{"item_id":"semeval2013.d002.s002.t001","gold_sense_keys":["figure%1:07:01::"],"candidates":[{"index":1,"sense_key":"figure%1:10:00::","synset_id":"figure.n.01"},{"index":2,"sense_key":"figure%1:08:00::","synset_id":"human_body.n.01"},{"index":3,"sense_key":"figure%1:23:00::","synset_id":"digit.n.01"},{"index":4,"sense_key":"figure%1:06:00::","synset_id":"figure.n.04"},{"index":5,"sense_key":"figure%1:18:00::","synset_id":"name.n.04"},{"index":6,"sense_key":"figure%1:25:00::","synset_id":"figure.n.06"},{"index":7,"sense_key":"figure%1:21:00::","synset_id":"figure.n.07"},{"index":8,"sense_key":"figure%1:07:00::","synset_id":"figure.n.08"},{"index":9,"sense_key":"figure%1:07:01::","synset_id":"number.n.01"},{"index":10,"sense_key":"figure%1:10:01::","synset_id":"trope.n.01"},{"index":11,"sense_key":"figure%1:09:00::","synset_id":"figure.n.11"},{"index":12,"sense_key":"figure%1:06:01::","synset_id":"design.n.04"},{"index":13,"sense_key":"figure%1:04:00::","synset_id":"figure.n.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"figure%1:21:00::","call_ids":["semeval2013.d002.s002.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"figure%1:21:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1211,"cached_input_tokens":0,"output_tokens":146,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003882,"input_uncached_usd":0.0024219999999999997,"input_cached_usd":0.0,"output_usd":0.0014600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.003737166058272}
{"item_id":"semeval2013.d002.s002.t006","gold_sense_keys":["predominance%1:26:00::"],"candidates":[{"index":1,"sense_key":"predominance%1:26:00::","synset_id":"predominance.n.01"},{"index":2,"sense_key":"predominance%1:07:00::","synset_id":"predomination.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"predominance%1:07:00::","call_ids":["semeval2013.d002.s002.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"predominance%1:07:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":577,"cached_input_tokens":0,"output_tokens":104,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0021939999999999998,"input_uncached_usd":0.0011539999999999999,"input_cached_usd":0.0,"output_usd":0.0010400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.427907667006366}
{"item_id":"semeval2013.d002.s004.t000","gold_sense_keys":["market%1:14:01::"],"candidates":[{"index":1,"sense_key":"market%1:04:00::","synset_id":"market.n.01"},{"index":2,"sense_key":"market%1:14:00::","synset_id":"market.n.02"},{"index":3,"sense_key":"market%1:06:00::","synset_id":"grocery_store.n.01"},{"index":4,"sense_key":"market%1:14:01::","synset_id":"market.n.04"},{"index":5,"sense_key":"market%1:06:01::","synset_id":"marketplace.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"market%1:14:01::","call_ids":["semeval2013.d002.s004.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"market%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":832,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001784,"input_uncached_usd":0.001664,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.45424595894292}
{"item_id":"semeval2013.d002.s005.t002","gold_sense_keys":["volume%1:07:03::"],"candidates":[{"index":1,"sense_key":"volume%1:23:00::","synset_id":"volume.n.01"},{"index":2,"sense_key":"volume%1:07:03::","synset_id":"bulk.n.02"},{"index":3,"sense_key":"volume%1:06:00::","synset_id":"book.n.02"},{"index":4,"sense_key":"volume%1:10:00::","synset_id":"volume.n.04"},{"index":5,"sense_key":"volume%1:23:01::","synset_id":"volume.n.05"},{"index":6,"sense_key":"volume%1:07:02::","synset_id":"volume.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"volume%1:07:03::","call_ids":["semeval2013.d002.s005.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"volume%1:07:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":975,"cached_input_tokens":0,"output_tokens":157,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00352,"input_uncached_usd":0.00195,"input_cached_usd":0.0,"output_usd":0.0015700000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0659788749180734}
{"item_id":"semeval2013.d002.s005.t003","gold_sense_keys":["case%1:09:00::"],"candidates":[{"index":1,"sense_key":"case%1:11:00::","synset_id":"case.n.01"},{"index":2,"sense_key":"case%1:26:00::","synset_id":"event.n.02"},{"index":3,"sense_key":"case%1:04:00::","synset_id":"lawsuit.n.01"},{"index":4,"sense_key":"case%1:09:00::","synset_id":"case.n.04"},{"index":5,"sense_key":"case%1:06:00::","synset_id":"case.n.05"},{"index":6,"sense_key":"case%1:18:03::","synset_id":"case.n.06"},{"index":7,"sense_key":"case%1:18:02::","synset_id":"subject.n.06"},{"index":8,"sense_key":"case%1:10:02::","synset_id":"case.n.08"},{"index":9,"sense_key":"case%1:10:01::","synset_id":"case.n.09"},{"index":10,"sense_key":"case%1:23:00::","synset_id":"case.n.10"},{"index":11,"sense_key":"case%1:10:00::","synset_id":"case.n.11"},{"index":12,"sense_key":"case%1:26:02::","synset_id":"case.n.12"},{"index":13,"sense_key":"case%1:18:04::","synset_id":"character.n.05"},{"index":14,"sense_key":"case%1:10:03::","synset_id":"font.n.01"},{"index":15,"sense_key":"case%1:08:00::","synset_id":"sheath.n.02"},{"index":16,"sense_key":"case%1:06:06::","synset_id":"shell.n.08"},{"index":17,"sense_key":"case%1:06:07::","synset_id":"casing.n.03"},{"index":18,"sense_key":"case%1:06:03::","synset_id":"case.n.18"},{"index":19,"sense_key":"case%1:06:02::","synset_id":"case.n.19"},{"index":20,"sense_key":"case%1:06:01::","synset_id":"case.n.20"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"case%1:09:00::","call_ids":["semeval2013.d002.s005.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"case%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1852,"cached_input_tokens":0,"output_tokens":39,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0040939999999999995,"input_uncached_usd":0.003704,"input_cached_usd":0.0,"output_usd":0.00039000000000000005,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.201030166936107}
{"item_id":"semeval2013.d002.s005.t004","gold_sense_keys":["end%1:09:00::","end%1:11:00::","end%1:28:00::"],"candidates":[{"index":1,"sense_key":"end%1:15:00::","synset_id":"end.n.01"},{"index":2,"sense_key":"end%1:28:00::","synset_id":"end.n.02"},{"index":3,"sense_key":"end%1:11:00::","synset_id":"end.n.03"},{"index":4,"sense_key":"end%1:09:02::","synset_id":"goal.n.01"},{"index":5,"sense_key":"end%1:09:00::","synset_id":"end.n.05"},{"index":6,"sense_key":"end%1:26:00::","synset_id":"end.n.06"},{"index":7,"sense_key":"end%1:15:02::","synset_id":"end.n.07"},{"index":8,"sense_key":"end%1:18:00::","synset_id":"end.n.08"},{"index":9,"sense_key":"end%1:15:01::","synset_id":"end.n.09"},{"index":10,"sense_key":"end%1:15:03::","synset_id":"end.n.10"},{"index":11,"sense_key":"end%1:04:00::","synset_id":"end.n.11"},{"index":12,"sense_key":"end%1:10:00::","synset_id":"conclusion.n.08"},{"index":13,"sense_key":"end%1:06:02::","synset_id":"end.n.13"},{"index":14,"sense_key":"end%1:04:02::","synset_id":"end.n.14"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"end%1:28:00::","call_ids":["semeval2013.d002.s005.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"end%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1407,"cached_input_tokens":0,"output_tokens":52,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0033339999999999997,"input_uncached_usd":0.0028139999999999997,"input_cached_usd":0.0,"output_usd":0.0005200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6152963750064373}
{"item_id":"semeval2013.d002.s005.t005","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["semeval2013.d002.s005.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":858,"cached_input_tokens":0,"output_tokens":81,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002526,"input_uncached_usd":0.0017159999999999999,"input_cached_usd":0.0,"output_usd":0.0008100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9943668339401484}
{"item_id":"semeval2013.d002.s006.t000","gold_sense_keys":["trend%1:15:02::"],"candidates":[{"index":1,"sense_key":"trend%1:15:02::","synset_id":"tendency.n.04"},{"index":2,"sense_key":"trend%1:15:01::","synset_id":"course.n.03"},{"index":3,"sense_key":"trend%1:09:00::","synset_id":"drift.n.05"},{"index":4,"sense_key":"trend%1:09:02::","synset_id":"vogue.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"trend%1:15:02::","call_ids":["semeval2013.d002.s006.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"trend%1:15:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":880,"cached_input_tokens":0,"output_tokens":48,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00224,"input_uncached_usd":0.0017599999999999998,"input_cached_usd":0.0,"output_usd":0.00048000000000000007,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2650817079702392}
{"item_id":"semeval2013.d002.s006.t001","gold_sense_keys":["publication%1:04:02::"],"candidates":[{"index":1,"sense_key":"publication%1:10:00::","synset_id":"publication.n.01"},{"index":2,"sense_key":"publication%1:04:01::","synset_id":"issue.n.11"},{"index":3,"sense_key":"publication%1:04:02::","synset_id":"publication.n.03"},{"index":4,"sense_key":"publication%1:04:00::","synset_id":"publication.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"publication%1:04:02::","call_ids":["semeval2013.d002.s006.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"publication%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":773,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001666,"input_uncached_usd":0.0015459999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2431477919453755}
{"item_id":"semeval2013.d002.s006.t002","gold_sense_keys":["indicator%1:10:00::"],"candidates":[{"index":1,"sense_key":"indicator%1:10:00::","synset_id":"index.n.02"},{"index":2,"sense_key":"indicator%1:10:01::","synset_id":"indicator.n.02"},{"index":3,"sense_key":"indicator%1:06:00::","synset_id":"indicator.n.03"},{"index":4,"sense_key":"indicator%1:27:00::","synset_id":"indicator.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"indicator%1:10:00::","call_ids":["semeval2013.d002.s006.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"indicator%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":847,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001814,"input_uncached_usd":0.001694,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.486112041049637}
{"item_id":"semeval2013.d002.s006.t003","gold_sense_keys":["market%1:14:01::"],"candidates":[{"index":1,"sense_key":"market%1:04:00::","synset_id":"market.n.01"},{"index":2,"sense_key":"market%1:14:00::","synset_id":"market.n.02"},{"index":3,"sense_key":"market%1:06:00::","synset_id":"grocery_store.n.01"},{"index":4,"sense_key":"market%1:14:01::","synset_id":"market.n.04"},{"index":5,"sense_key":"market%1:06:01::","synset_id":"marketplace.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"market%1:14:01::","call_ids":["semeval2013.d002.s006.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"market%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":910,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0024200000000000003,"input_uncached_usd":0.00182,"input_cached_usd":0.0,"output_usd":0.0006000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9785014169756323}
{"item_id":"semeval2013.d002.s006.t005","gold_sense_keys":["relation%1:03:00::"],"candidates":[{"index":1,"sense_key":"relation%1:03:00::","synset_id":"relation.n.01"},{"index":2,"sense_key":"relation%1:04:01::","synset_id":"sexual_intercourse.n.01"},{"index":3,"sense_key":"relation%1:18:00::","synset_id":"relative.n.01"},{"index":4,"sense_key":"relation%1:10:01::","synset_id":"relation.n.04"},{"index":5,"sense_key":"relation%1:09:00::","synset_id":"relation_back.n.01"},{"index":6,"sense_key":"relation%1:04:00::","synset_id":"relation.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"relation%1:03:00::","call_ids":["semeval2013.d002.s006.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"relation%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1052,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002654,"input_uncached_usd":0.002104,"input_cached_usd":0.0,"output_usd":0.00055,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6072714590700343}
{"item_id":"semeval2013.d002.s006.t007","gold_sense_keys":["recovery%1:11:00::"],"candidates":[{"index":1,"sense_key":"recovery%1:11:00::","synset_id":"recovery.n.01"},{"index":2,"sense_key":"recovery%1:22:00::","synset_id":"convalescence.n.01"},{"index":3,"sense_key":"recovery%1:04:00::","synset_id":"recovery.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"recovery%1:11:00::","call_ids":["semeval2013.d002.s006.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"recovery%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":780,"cached_input_tokens":0,"output_tokens":89,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00245,"input_uncached_usd":0.00156,"input_cached_usd":0.0,"output_usd":0.0008900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5047558749793097}
{"item_id":"semeval2013.d002.s006.t008","gold_sense_keys":["analyst%1:18:00::","analyst%1:18:02::"],"candidates":[{"index":1,"sense_key":"analyst%1:18:00::","synset_id":"analyst.n.01"},{"index":2,"sense_key":"analyst%1:18:02::","synset_id":"analyst.n.02"},{"index":3,"sense_key":"analyst%1:18:01::","synset_id":"analyst.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"analyst%1:18:02::","call_ids":["semeval2013.d002.s006.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"analyst%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":766,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001652,"input_uncached_usd":0.001532,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0898055420257151}
{"item_id":"semeval2013.d002.s007.t000","gold_sense_keys":["united_states%1:15:00::"],"candidates":[{"index":1,"sense_key":"united_states%1:15:00::","synset_id":"united_states.n.01"},{"index":2,"sense_key":"united_states%1:14:00::","synset_id":"united_states_government.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"united_states%1:15:00::","call_ids":["semeval2013.d002.s007.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"united_states%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":767,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002144,"input_uncached_usd":0.001534,"input_cached_usd":0.0,"output_usd":0.0006100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8218460000352934}
{"item_id":"semeval2013.d002.s007.t003","gold_sense_keys":["month%1:28:01::"],"candidates":[{"index":1,"sense_key":"month%1:28:01::","synset_id":"calendar_month.n.01"},{"index":2,"sense_key":"month%1:28:00::","synset_id":"month.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"month%1:28:01::","call_ids":["semeval2013.d002.s007.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"month%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":696,"cached_input_tokens":0,"output_tokens":122,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002612,"input_uncached_usd":0.001392,"input_cached_usd":0.0,"output_usd":0.0012200000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8669553330400959}
{"item_id":"semeval2013.d002.s007.t004","gold_sense_keys":["row%1:07:00::"],"candidates":[{"index":1,"sense_key":"row%1:14:00::","synset_id":"row.n.01"},{"index":2,"sense_key":"row%1:10:00::","synset_id":"quarrel.n.01"},{"index":3,"sense_key":"row%1:17:00::","synset_id":"row.n.03"},{"index":4,"sense_key":"row%1:06:00::","synset_id":"course.n.08"},{"index":5,"sense_key":"row%1:14:01::","synset_id":"row.n.05"},{"index":6,"sense_key":"row%1:07:00::","synset_id":"row.n.06"},{"index":7,"sense_key":"row%1:04:00::","synset_id":"rowing.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"row%1:07:00::","call_ids":["semeval2013.d002.s007.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"row%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":954,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002028,"input_uncached_usd":0.001908,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1549614999676123}
{"item_id":"semeval2013.d002.s008.t000","gold_sense_keys":["confidence%1:12:00::","confidence%1:26:02::"],"candidates":[{"index":1,"sense_key":"confidence%1:09:00::","synset_id":"assurance.n.01"},{"index":2,"sense_key":"confidence%1:12:00::","synset_id":"confidence.n.02"},{"index":3,"sense_key":"confidence%1:26:02::","synset_id":"confidence.n.03"},{"index":4,"sense_key":"confidence%1:26:03::","synset_id":"confidence.n.04"},{"index":5,"sense_key":"confidence%1:10:00::","synset_id":"confidence.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"confidence%1:26:02::","call_ids":["semeval2013.d002.s008.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"confidence%1:26:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":943,"cached_input_tokens":0,"output_tokens":76,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002646,"input_uncached_usd":0.0018859999999999999,"input_cached_usd":0.0,"output_usd":0.00076,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9161564169917256}
{"item_id":"semeval2013.d002.s008.t003","gold_sense_keys":["month%1:28:01::"],"candidates":[{"index":1,"sense_key":"month%1:28:01::","synset_id":"calendar_month.n.01"},{"index":2,"sense_key":"month%1:28:00::","synset_id":"month.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"month%1:28:01::","call_ids":["semeval2013.d002.s008.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"month%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":704,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002058,"input_uncached_usd":0.001408,"input_cached_usd":0.0,"output_usd":0.0006500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.775286625023}
{"item_id":"semeval2013.d002.s008.t004","gold_sense_keys":["estimate%1:09:00::"],"candidates":[{"index":1,"sense_key":"estimate%1:09:00::","synset_id":"estimate.n.01"},{"index":2,"sense_key":"estimate%1:04:01::","synset_id":"estimate.n.02"},{"index":3,"sense_key":"estimate%1:10:00::","synset_id":"appraisal.n.02"},{"index":4,"sense_key":"estimate%1:10:01::","synset_id":"estimate.n.04"},{"index":5,"sense_key":"estimate%1:09:01::","synset_id":"estimate.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"estimate%1:09:00::","call_ids":["semeval2013.d002.s008.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"estimate%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":928,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001976,"input_uncached_usd":0.001856,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2342654169769958}
{"item_id":"semeval2013.d002.s008.t005","gold_sense_keys":["index%1:10:01::"],"candidates":[{"index":1,"sense_key":"index%1:24:00::","synset_id":"index.n.01"},{"index":2,"sense_key":"index%1:10:01::","synset_id":"index.n.02"},{"index":3,"sense_key":"index%1:10:02::","synset_id":"exponent.n.03"},{"index":4,"sense_key":"index%1:10:00::","synset_id":"index.n.04"},{"index":5,"sense_key":"index%1:08:00::","synset_id":"index.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"index%1:10:01::","call_ids":["semeval2013.d002.s008.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"index%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":867,"cached_input_tokens":0,"output_tokens":120,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002934,"input_uncached_usd":0.0017339999999999999,"input_cached_usd":0.0,"output_usd":0.0012000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9843516249675304}
{"item_id":"semeval2013.d002.s009.t000","gold_sense_keys":["index%1:10:01::"],"candidates":[{"index":1,"sense_key":"index%1:24:00::","synset_id":"index.n.01"},{"index":2,"sense_key":"index%1:10:01::","synset_id":"index.n.02"},{"index":3,"sense_key":"index%1:10:02::","synset_id":"exponent.n.03"},{"index":4,"sense_key":"index%1:10:00::","synset_id":"index.n.04"},{"index":5,"sense_key":"index%1:08:00::","synset_id":"index.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"index%1:10:01::","call_ids":["semeval2013.d002.s009.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"index%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":837,"cached_input_tokens":0,"output_tokens":94,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002614,"input_uncached_usd":0.001674,"input_cached_usd":0.0,"output_usd":0.0009400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.082812917069532}
{"item_id":"semeval2013.d002.s009.t001","gold_sense_keys":["rise%1:04:01::","rise%1:11:00::"],"candidates":[{"index":1,"sense_key":"rise%1:11:00::","synset_id":"rise.n.01"},{"index":2,"sense_key":"rise%1:04:00::","synset_id":"rise.n.02"},{"index":3,"sense_key":"rise%1:17:00::","synset_id":"ascent.n.01"},{"index":4,"sense_key":"rise%1:11:02::","synset_id":"rise.n.04"},{"index":5,"sense_key":"rise%1:07:02::","synset_id":"raise.n.01"},{"index":6,"sense_key":"rise%1:07:01::","synset_id":"upgrade.n.04"},{"index":7,"sense_key":"rise%1:11:01::","synset_id":"lift.n.04"},{"index":8,"sense_key":"rise%1:11:03::","synset_id":"emanation.n.03"},{"index":9,"sense_key":"rise%1:07:03::","synset_id":"rise.n.09"},{"index":10,"sense_key":"rise%1:04:01::","synset_id":"advance.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"rise%1:04:01::","call_ids":["semeval2013.d002.s009.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"rise%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1168,"cached_input_tokens":0,"output_tokens":146,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003796,"input_uncached_usd":0.002336,"input_cached_usd":0.0,"output_usd":0.0014600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1193433749722317}
{"item_id":"semeval2013.d002.s009.t002","gold_sense_keys":["dollar%1:23:00::"],"candidates":[{"index":1,"sense_key":"dollar%1:23:00::","synset_id":"dollar.n.01"},{"index":2,"sense_key":"dollar%1:21:00::","synset_id":"dollar.n.02"},{"index":3,"sense_key":"dollar%1:21:01::","synset_id":"dollar.n.03"},{"index":4,"sense_key":"dollar%1:10:00::","synset_id":"dollar.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"dollar%1:23:00::","call_ids":["semeval2013.d002.s009.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"dollar%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":805,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00173,"input_uncached_usd":0.0016099999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2403084579855204}
{"item_id":"semeval2013.d002.s009.t003","gold_sense_keys":["high%1:07:00::"],"candidates":[{"index":1,"sense_key":"high%1:07:00::","synset_id":"high.n.01"},{"index":2,"sense_key":"high%1:26:00::","synset_id":"high.n.02"},{"index":3,"sense_key":"high%1:26:02::","synset_id":"high.n.03"},{"index":4,"sense_key":"high%1:26:01::","synset_id":"high.n.04"},{"index":5,"sense_key":"high%1:15:00::","synset_id":"high.n.05"},{"index":6,"sense_key":"high%1:14:00::","synset_id":"senior_high_school.n.01"},{"index":7,"sense_key":"high%1:06:00::","synset_id":"high_gear.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"high%1:07:00::","call_ids":["semeval2013.d002.s009.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"high%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":962,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002044,"input_uncached_usd":0.001924,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.173588000005111}
{"item_id":"semeval2013.d002.s009.t005","gold_sense_keys":["pressure%1:07:00::"],"candidates":[{"index":1,"sense_key":"pressure%1:19:00::","synset_id":"pressure.n.01"},{"index":2,"sense_key":"pressure%1:07:00::","synset_id":"pressure.n.02"},{"index":3,"sense_key":"pressure%1:04:00::","synset_id":"press.n.09"},{"index":4,"sense_key":"pressure%1:26:00::","synset_id":"imperativeness.n.01"},{"index":5,"sense_key":"pressure%1:09:00::","synset_id":"pressure.n.05"},{"index":6,"sense_key":"pressure%1:26:01::","synset_id":"pressure.n.06"},{"index":7,"sense_key":"pressure%1:19:01::","synset_id":"atmospheric_pressure.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"pressure%1:07:00::","call_ids":["semeval2013.d002.s009.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"pressure%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1038,"cached_input_tokens":0,"output_tokens":126,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0033359999999999996,"input_uncached_usd":0.0020759999999999997,"input_cached_usd":0.0,"output_usd":0.00126,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8088457499397919}
{"item_id":"semeval2013.d002.s009.t006","gold_sense_keys":["sector%1:14:00::"],"candidates":[{"index":1,"sense_key":"sector%1:25:00::","synset_id":"sector.n.01"},{"index":2,"sense_key":"sector%1:14:00::","synset_id":"sector.n.02"},{"index":3,"sense_key":"sector%1:09:00::","synset_id":"sector.n.03"},{"index":4,"sense_key":"sector%1:23:00::","synset_id":"sector.n.04"},{"index":5,"sense_key":"sector%1:15:00::","synset_id":"sector.n.05"},{"index":6,"sense_key":"sector%1:06:00::","synset_id":"sector.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"sector%1:14:00::","call_ids":["semeval2013.d002.s009.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"sector%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":849,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002238,"input_uncached_usd":0.0016979999999999999,"input_cached_usd":0.0,"output_usd":0.00054,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1172990839695558}
{"item_id":"semeval2013.d002.s009.t007","gold_sense_keys":["situation%1:26:00::"],"candidates":[{"index":1,"sense_key":"situation%1:26:00::","synset_id":"situation.n.01"},{"index":2,"sense_key":"situation%1:26:01::","synset_id":"situation.n.02"},{"index":3,"sense_key":"situation%1:26:02::","synset_id":"situation.n.03"},{"index":4,"sense_key":"situation%1:15:00::","synset_id":"site.n.02"},{"index":5,"sense_key":"situation%1:04:00::","synset_id":"position.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"situation%1:26:00::","call_ids":["semeval2013.d002.s009.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"situation%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":935,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00199,"input_uncached_usd":0.00187,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.289950666949153}
{"item_id":"semeval2013.d002.s010.t001","gold_sense_keys":["security%1:21:04::"],"candidates":[{"index":1,"sense_key":"security%1:26:00::","synset_id":"security.n.01"},{"index":2,"sense_key":"security%1:21:01::","synset_id":"security.n.02"},{"index":3,"sense_key":"security%1:12:00::","synset_id":"security.n.03"},{"index":4,"sense_key":"security%1:21:04::","synset_id":"security.n.04"},{"index":5,"sense_key":"security%1:21:00::","synset_id":"security.n.05"},{"index":6,"sense_key":"security%1:14:00::","synset_id":"security.n.06"},{"index":7,"sense_key":"security%1:10:00::","synset_id":"security.n.07"},{"index":8,"sense_key":"security%1:06:00::","synset_id":"security_system.n.02"},{"index":9,"sense_key":"security%1:04:00::","synset_id":"security.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"security%1:21:04::","call_ids":["semeval2013.d002.s010.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"security%1:21:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1088,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002296,"input_uncached_usd":0.002176,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2990728330332786}
{"item_id":"semeval2013.d002.s010.t002","gold_sense_keys":["ground%1:15:00::"],"candidates":[{"index":1,"sense_key":"ground%1:17:00::","synset_id":"land.n.04"},{"index":2,"sense_key":"ground%1:16:00::","synset_id":"reason.n.01"},{"index":3,"sense_key":"ground%1:27:00::","synset_id":"earth.n.02"},{"index":4,"sense_key":"ground%1:24:00::","synset_id":"footing.n.02"},{"index":5,"sense_key":"ground%1:15:00::","synset_id":"ground.n.05"},{"index":6,"sense_key":"ground%1:09:00::","synset_id":"background.n.02"},{"index":7,"sense_key":"ground%1:17:01::","synset_id":"land.n.02"},{"index":8,"sense_key":"ground%1:09:01::","synset_id":"ground.n.08"},{"index":9,"sense_key":"ground%1:06:02::","synset_id":"ground.n.09"},{"index":10,"sense_key":"ground%1:06:01::","synset_id":"ground.n.10"},{"index":11,"sense_key":"ground%1:06:00::","synset_id":"flat_coat.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"ground%1:15:00::","call_ids":["semeval2013.d002.s010.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"ground%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1281,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003192,"input_uncached_usd":0.002562,"input_cached_usd":0.0,"output_usd":0.00063,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.479420040966943}
{"item_id":"semeval2013.d002.s012.t001","gold_sense_keys":["bond%1:21:02::"],"candidates":[{"index":1,"sense_key":"bond%1:19:00::","synset_id":"chemical_bond.n.01"},{"index":2,"sense_key":"bond%1:21:02::","synset_id":"bond.n.02"},{"index":3,"sense_key":"bond%1:24:00::","synset_id":"alliance.n.02"},{"index":4,"sense_key":"bond%1:21:03::","synset_id":"bail.n.01"},{"index":5,"sense_key":"bond%1:06:00::","synset_id":"shackle.n.01"},{"index":6,"sense_key":"bond%1:06:01::","synset_id":"attachment.n.04"},{"index":7,"sense_key":"bond%1:27:00::","synset_id":"bond.n.07"},{"index":8,"sense_key":"bond%1:18:00::","synset_id":"bond.n.08"},{"index":9,"sense_key":"bond%1:18:01::","synset_id":"bond.n.09"},{"index":10,"sense_key":"bond%1:07:00::","synset_id":"adhesiveness.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bond%1:21:02::","call_ids":["semeval2013.d002.s012.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bond%1:21:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1244,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0026079999999999996,"input_uncached_usd":0.0024879999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9922547499882057}
{"item_id":"semeval2013.d002.s012.t002","gold_sense_keys":["yield%1:21:00::"],"candidates":[{"index":1,"sense_key":"yield%1:04:00::","synset_id":"output.n.02"},{"index":2,"sense_key":"yield%1:21:00::","synset_id":"return.n.06"},{"index":3,"sense_key":"yield%1:06:00::","synset_id":"yield.n.03"},{"index":4,"sense_key":"yield%1:23:00::","synset_id":"output.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"yield%1:21:00::","call_ids":["semeval2013.d002.s012.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"yield%1:21:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":725,"cached_input_tokens":0,"output_tokens":36,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00181,"input_uncached_usd":0.00145,"input_cached_usd":0.0,"output_usd":0.00036,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.311838083085604}
{"item_id":"semeval2013.d002.s012.t004","gold_sense_keys":["evening%1:28:00::"],"candidates":[{"index":1,"sense_key":"evening%1:28:00::","synset_id":"evening.n.01"},{"index":2,"sense_key":"evening%1:28:01::","synset_id":"evening.n.02"},{"index":3,"sense_key":"evening%1:28:02::","synset_id":"evening.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"evening%1:28:00::","call_ids":["semeval2013.d002.s012.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"evening%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":688,"cached_input_tokens":0,"output_tokens":100,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0023759999999999996,"input_uncached_usd":0.0013759999999999998,"input_cached_usd":0.0,"output_usd":0.001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7174852910684422}
{"item_id":"semeval2013.d002.s012.t005","gold_sense_keys":["bond%1:21:02::"],"candidates":[{"index":1,"sense_key":"bond%1:19:00::","synset_id":"chemical_bond.n.01"},{"index":2,"sense_key":"bond%1:21:02::","synset_id":"bond.n.02"},{"index":3,"sense_key":"bond%1:24:00::","synset_id":"alliance.n.02"},{"index":4,"sense_key":"bond%1:21:03::","synset_id":"bail.n.01"},{"index":5,"sense_key":"bond%1:06:00::","synset_id":"shackle.n.01"},{"index":6,"sense_key":"bond%1:06:01::","synset_id":"attachment.n.04"},{"index":7,"sense_key":"bond%1:27:00::","synset_id":"bond.n.07"},{"index":8,"sense_key":"bond%1:18:00::","synset_id":"bond.n.08"},{"index":9,"sense_key":"bond%1:18:01::","synset_id":"bond.n.09"},{"index":10,"sense_key":"bond%1:07:00::","synset_id":"adhesiveness.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bond%1:21:02::","call_ids":["semeval2013.d002.s012.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bond%1:21:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1244,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0026079999999999996,"input_uncached_usd":0.0024879999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4250799170695245}
{"item_id":"semeval2013.d002.s012.t006","gold_sense_keys":["yield%1:21:00::"],"candidates":[{"index":1,"sense_key":"yield%1:04:00::","synset_id":"output.n.02"},{"index":2,"sense_key":"yield%1:21:00::","synset_id":"return.n.06"},{"index":3,"sense_key":"yield%1:06:00::","synset_id":"yield.n.03"},{"index":4,"sense_key":"yield%1:23:00::","synset_id":"output.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"yield%1:21:00::","call_ids":["semeval2013.d002.s012.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"yield%1:21:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":725,"cached_input_tokens":0,"output_tokens":69,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00214,"input_uncached_usd":0.00145,"input_cached_usd":0.0,"output_usd":0.0006900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4758424159372225}
{"item_id":"semeval2013.d002.s012.t007","gold_sense_keys":["evening%1:28:00::"],"candidates":[{"index":1,"sense_key":"evening%1:28:00::","synset_id":"evening.n.01"},{"index":2,"sense_key":"evening%1:28:01::","synset_id":"evening.n.02"},{"index":3,"sense_key":"evening%1:28:02::","synset_id":"evening.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"evening%1:28:00::","call_ids":["semeval2013.d002.s012.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"evening%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":688,"cached_input_tokens":0,"output_tokens":111,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002486,"input_uncached_usd":0.0013759999999999998,"input_cached_usd":0.0,"output_usd":0.00111,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9207332079531625}
{"item_id":"semeval2013.d003.s000.t000","gold_sense_keys":["deal%1:10:00::"],"candidates":[{"index":1,"sense_key":"deal%1:04:02::","synset_id":"deal.n.01"},{"index":2,"sense_key":"deal%1:10:00::","synset_id":"bargain.n.01"},{"index":3,"sense_key":"deal%1:23:00::","synset_id":"batch.n.02"},{"index":4,"sense_key":"deal%1:27:00::","synset_id":"deal.n.04"},{"index":5,"sense_key":"deal%1:27:01::","synset_id":"softwood.n.01"},{"index":6,"sense_key":"deal%1:14:00::","synset_id":"hand.n.06"},{"index":7,"sense_key":"deal%1:11:00::","synset_id":"deal.n.07"},{"index":8,"sense_key":"deal%1:04:00::","synset_id":"deal.n.08"},{"index":9,"sense_key":"deal%1:04:01::","synset_id":"deal.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"deal%1:10:00::","call_ids":["semeval2013.d003.s000.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"deal%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":874,"cached_input_tokens":0,"output_tokens":120,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002948,"input_uncached_usd":0.001748,"input_cached_usd":0.0,"output_usd":0.0012000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1360414170194417}
{"item_id":"semeval2013.d003.s001.t000","gold_sense_keys":["concern%1:12:01::"],"candidates":[{"index":1,"sense_key":"concern%1:09:00::","synset_id":"concern.n.01"},{"index":2,"sense_key":"concern%1:12:01::","synset_id":"concern.n.02"},{"index":3,"sense_key":"concern%1:12:00::","synset_id":"concern.n.03"},{"index":4,"sense_key":"concern%1:09:01::","synset_id":"concern.n.04"},{"index":5,"sense_key":"concern%1:14:00::","synset_id":"business.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"concern%1:12:01::","call_ids":["semeval2013.d003.s001.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"concern%1:12:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":736,"cached_input_tokens":0,"output_tokens":243,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003902,"input_uncached_usd":0.001472,"input_cached_usd":0.0,"output_usd":0.0024300000000000003,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.5077895409194753}
{"item_id":"semeval2013.d003.s001.t001","gold_sense_keys":["violence%1:26:00::"],"candidates":[{"index":1,"sense_key":"violence%1:04:01::","synset_id":"violence.n.01"},{"index":2,"sense_key":"violence%1:07:00::","synset_id":"ferocity.n.01"},{"index":3,"sense_key":"violence%1:26:00::","synset_id":"violence.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"violence%1:26:00::","call_ids":["semeval2013.d003.s001.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"violence%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":594,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0013080000000000001,"input_uncached_usd":0.001188,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.289574124966748}
{"item_id":"semeval2013.d003.s001.t002","gold_sense_keys":["instability%1:07:00::","instability%1:07:01::","instability%1:26:00::"],"candidates":[{"index":1,"sense_key":"instability%1:26:00::","synset_id":"instability.n.01"},{"index":2,"sense_key":"instability%1:07:01::","synset_id":"instability.n.02"},{"index":3,"sense_key":"instability%1:26:01::","synset_id":"imbalance.n.01"},{"index":4,"sense_key":"instability%1:07:00::","synset_id":"instability.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"instability%1:26:00::","call_ids":["semeval2013.d003.s001.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"instability%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":575,"cached_input_tokens":0,"output_tokens":106,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00221,"input_uncached_usd":0.00115,"input_cached_usd":0.0,"output_usd":0.0010600000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1543225839268416}
{"item_id":"semeval2013.d003.s001.t003","gold_sense_keys":["government%1:14:00::"],"candidates":[{"index":1,"sense_key":"government%1:14:00::","synset_id":"government.n.01"},{"index":2,"sense_key":"government%1:04:00::","synset_id":"government.n.02"},{"index":3,"sense_key":"government%1:09:00::","synset_id":"government.n.03"},{"index":4,"sense_key":"government%1:09:01::","synset_id":"politics.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"government%1:14:00::","call_ids":["semeval2013.d003.s001.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"government%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":639,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001398,"input_uncached_usd":0.0012779999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1316469580633566}
{"item_id":"semeval2013.d003.s001.t006","gold_sense_keys":["auction%1:04:00::"],"candidates":[{"index":1,"sense_key":"auction%1:04:01::","synset_id":"auction.n.01"},{"index":2,"sense_key":"auction%1:04:00::","synset_id":"auction.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"auction%1:04:00::","call_ids":["semeval2013.d003.s001.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"auction%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":522,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001164,"input_uncached_usd":0.001044,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0796658750623465}
{"item_id":"semeval2013.d003.s002.t000","gold_sense_keys":["deal%1:10:00::"],"candidates":[{"index":1,"sense_key":"deal%1:04:02::","synset_id":"deal.n.01"},{"index":2,"sense_key":"deal%1:10:00::","synset_id":"bargain.n.01"},{"index":3,"sense_key":"deal%1:23:00::","synset_id":"batch.n.02"},{"index":4,"sense_key":"deal%1:27:00::","synset_id":"deal.n.04"},{"index":5,"sense_key":"deal%1:27:01::","synset_id":"softwood.n.01"},{"index":6,"sense_key":"deal%1:14:00::","synset_id":"hand.n.06"},{"index":7,"sense_key":"deal%1:11:00::","synset_id":"deal.n.07"},{"index":8,"sense_key":"deal%1:04:00::","synset_id":"deal.n.08"},{"index":9,"sense_key":"deal%1:04:01::","synset_id":"deal.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"deal%1:10:00::","call_ids":["semeval2013.d003.s002.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"deal%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":981,"cached_input_tokens":0,"output_tokens":109,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003052,"input_uncached_usd":0.0019619999999999998,"input_cached_usd":0.0,"output_usd":0.00109,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.040651125018485}
{"item_id":"semeval2013.d003.s002.t002","gold_sense_keys":["china%1:15:00::"],"candidates":[{"index":1,"sense_key":"china%1:15:00::","synset_id":"china.n.01"},{"index":2,"sense_key":"china%1:06:00::","synset_id":"china.n.02"},{"index":3,"sense_key":"china%1:15:01::","synset_id":"taiwan.n.01"},{"index":4,"sense_key":"china%1:06:01::","synset_id":"chinaware.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"china%1:15:00::","call_ids":["semeval2013.d003.s002.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"china%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":683,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0014860000000000001,"input_uncached_usd":0.001366,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8562903749989346}
{"item_id":"semeval2013.d003.s002.t003","gold_sense_keys":["russia%1:15:04::"],"candidates":[{"index":1,"sense_key":"russia%1:15:00::","synset_id":"soviet_union.n.01"},{"index":2,"sense_key":"russia%1:15:01::","synset_id":"soviet_russia.n.01"},{"index":3,"sense_key":"russia%1:15:02::","synset_id":"russia.n.03"},{"index":4,"sense_key":"russia%1:15:04::","synset_id":"russia.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"russia%1:15:04::","call_ids":["semeval2013.d003.s002.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"russia%1:15:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":825,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00224,"input_uncached_usd":0.00165,"input_cached_usd":0.0,"output_usd":0.00059,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1267341669881716}
{"item_id":"semeval2013.d003.s002.t004","gold_sense_keys":["oil%1:27:01::"],"candidates":[{"index":1,"sense_key":"oil%1:27:00::","synset_id":"oil.n.01"},{"index":2,"sense_key":"oil%1:06:01::","synset_id":"oil.n.02"},{"index":3,"sense_key":"oil%1:27:01::","synset_id":"petroleum.n.01"},{"index":4,"sense_key":"oil%1:13:00::","synset_id":"vegetable_oil.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"oil%1:27:01::","call_ids":["semeval2013.d003.s002.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"oil%1:27:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":648,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001416,"input_uncached_usd":0.0012959999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.974611375015229}
{"item_id":"semeval2013.d003.s002.t006","gold_sense_keys":["role%1:04:00::"],"candidates":[{"index":1,"sense_key":"role%1:04:00::","synset_id":"function.n.03"},{"index":2,"sense_key":"role%1:09:00::","synset_id":"character.n.04"},{"index":3,"sense_key":"role%1:07:00::","synset_id":"function.n.02"},{"index":4,"sense_key":"role%1:04:01::","synset_id":"role.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"role%1:04:00::","call_ids":["semeval2013.d003.s002.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"role%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":680,"cached_input_tokens":0,"output_tokens":181,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00317,"input_uncached_usd":0.0013599999999999999,"input_cached_usd":0.0,"output_usd":0.0018100000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.77061441692058}
{"item_id":"semeval2013.d003.s002.t009","gold_sense_keys":["decade%1:28:00::"],"candidates":[{"index":1,"sense_key":"decade%1:28:00::","synset_id":"decade.n.01"},{"index":2,"sense_key":"decade%1:23:00::","synset_id":"ten.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"decade%1:28:00::","call_ids":["semeval2013.d003.s002.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"decade%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":531,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001182,"input_uncached_usd":0.001062,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":11.179319791030139}
{"item_id":"semeval2013.d003.s002.t010","gold_sense_keys":["war%1:04:00::","war%1:26:00::"],"candidates":[{"index":1,"sense_key":"war%1:04:00::","synset_id":"war.n.01"},{"index":2,"sense_key":"war%1:26:00::","synset_id":"war.n.02"},{"index":3,"sense_key":"war%1:04:02::","synset_id":"war.n.03"},{"index":4,"sense_key":"war%1:04:01::","synset_id":"war.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"war%1:04:00::","call_ids":["semeval2013.d003.s002.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"war%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":672,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001464,"input_uncached_usd":0.001344,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8287089590448886}
{"item_id":"semeval2013.d003.s002.t011","gold_sense_keys":["sanction%1:04:02::"],"candidates":[{"index":1,"sense_key":"sanction%1:10:00::","synset_id":"sanction.n.01"},{"index":2,"sense_key":"sanction%1:04:02::","synset_id":"sanction.n.02"},{"index":3,"sense_key":"sanction%1:07:00::","synset_id":"authority.n.06"},{"index":4,"sense_key":"sanction%1:04:01::","synset_id":"sanction.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"sanction%1:04:02::","call_ids":["semeval2013.d003.s002.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"sanction%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":653,"cached_input_tokens":0,"output_tokens":118,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002486,"input_uncached_usd":0.0013059999999999999,"input_cached_usd":0.0,"output_usd":0.00118,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.098903250065632}
{"item_id":"semeval2013.d003.s003.t000","gold_sense_keys":["company%1:14:01::"],"candidates":[{"index":1,"sense_key":"company%1:14:01::","synset_id":"company.n.01"},{"index":2,"sense_key":"company%1:14:03::","synset_id":"company.n.02"},{"index":3,"sense_key":"company%1:26:00::","synset_id":"company.n.03"},{"index":4,"sense_key":"company%1:14:02::","synset_id":"company.n.04"},{"index":5,"sense_key":"company%1:18:00::","synset_id":"caller.n.01"},{"index":6,"sense_key":"company%1:14:00::","synset_id":"company.n.06"},{"index":7,"sense_key":"company%1:14:06::","synset_id":"party.n.03"},{"index":8,"sense_key":"company%1:14:04::","synset_id":"ship's_company.n.01"},{"index":9,"sense_key":"company%1:14:05::","synset_id":"company.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"company%1:14:01::","call_ids":["semeval2013.d003.s003.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"company%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":945,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00201,"input_uncached_usd":0.00189,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0634606250096112}
{"item_id":"semeval2013.d003.s003.t001","gold_sense_keys":["stake%1:21:02::"],"candidates":[{"index":1,"sense_key":"stake%1:21:02::","synset_id":"interest.n.05"},{"index":2,"sense_key":"stake%1:10:00::","synset_id":"post.n.09"},{"index":3,"sense_key":"stake%1:06:01::","synset_id":"stake.n.03"},{"index":4,"sense_key":"stake%1:21:00::","synset_id":"stake.n.04"},{"index":5,"sense_key":"stake%1:06:00::","synset_id":"stake.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"stake%1:21:02::","call_ids":["semeval2013.d003.s003.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"stake%1:21:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":746,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001612,"input_uncached_usd":0.0014919999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0904835419496521}
{"item_id":"semeval2013.d003.s003.t002","gold_sense_keys":["field%1:15:05::"],"candidates":[{"index":1,"sense_key":"field%1:15:00::","synset_id":"field.n.01"},{"index":2,"sense_key":"field%1:15:04::","synset_id":"battlefield.n.01"},{"index":3,"sense_key":"field%1:15:01::","synset_id":"field.n.03"},{"index":4,"sense_key":"field%1:09:00::","synset_id":"discipline.n.01"},{"index":5,"sense_key":"field%1:19:00::","synset_id":"field.n.05"},{"index":6,"sense_key":"field%1:04:00::","synset_id":"field.n.06"},{"index":7,"sense_key":"field%1:26:00::","synset_id":"sphere.n.01"},{"index":8,"sense_key":"field%1:15:02::","synset_id":"playing_field.n.02"},{"index":9,"sense_key":"field%1:17:00::","synset_id":"plain.n.01"},{"index":10,"sense_key":"field%1:14:00::","synset_id":"field.n.10"},{"index":11,"sense_key":"field%1:15:03::","synset_id":"field.n.11"},{"index":12,"sense_key":"field%1:14:01::","synset_id":"field.n.12"},{"index":13,"sense_key":"field%1:14:02::","synset_id":"field.n.13"},{"index":14,"sense_key":"field%1:15:05::","synset_id":"field.n.14"},{"index":15,"sense_key":"field%1:14:03::","synset_id":"field.n.15"},{"index":16,"sense_key":"field%1:09:01::","synset_id":"field.n.16"},{"index":17,"sense_key":"field%1:06:00::","synset_id":"airfield.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":14,"chosen_sense_key":"field%1:15:05::","call_ids":["semeval2013.d003.s003.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":14,"predicted_sense_key":"field%1:15:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1527,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0031739999999999997,"input_uncached_usd":0.003054,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8422806250164285}
{"item_id":"semeval2013.d003.s004.t000","gold_sense_keys":["company%1:14:01::"],"candidates":[{"index":1,"sense_key":"company%1:14:01::","synset_id":"company.n.01"},{"index":2,"sense_key":"company%1:14:03::","synset_id":"company.n.02"},{"index":3,"sense_key":"company%1:26:00::","synset_id":"company.n.03"},{"index":4,"sense_key":"company%1:14:02::","synset_id":"company.n.04"},{"index":5,"sense_key":"company%1:18:00::","synset_id":"caller.n.01"},{"index":6,"sense_key":"company%1:14:00::","synset_id":"company.n.06"},{"index":7,"sense_key":"company%1:14:06::","synset_id":"party.n.03"},{"index":8,"sense_key":"company%1:14:04::","synset_id":"ship's_company.n.01"},{"index":9,"sense_key":"company%1:14:05::","synset_id":"company.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"company%1:14:01::","call_ids":["semeval2013.d003.s004.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"company%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":958,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002036,"input_uncached_usd":0.001916,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.3614610839867964}
{"item_id":"semeval2013.d003.s004.t001","gold_sense_keys":["auction%1:04:00::"],"candidates":[{"index":1,"sense_key":"auction%1:04:01::","synset_id":"auction.n.01"},{"index":2,"sense_key":"auction%1:04:00::","synset_id":"auction.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"auction%1:04:00::","call_ids":["semeval2013.d003.s004.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"auction%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":583,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001286,"input_uncached_usd":0.001166,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3434452909277752}
{"item_id":"semeval2013.d003.s005.t000","gold_sense_keys":["bid%1:10:00::"],"candidates":[{"index":1,"sense_key":"bid%1:10:03::","synset_id":"command.n.01"},{"index":2,"sense_key":"bid%1:04:00::","synset_id":"bid.n.02"},{"index":3,"sense_key":"bid%1:10:00::","synset_id":"bid.n.03"},{"index":4,"sense_key":"bid%1:10:01::","synset_id":"bid.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"bid%1:10:00::","call_ids":["semeval2013.d003.s005.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"bid%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":690,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0015,"input_uncached_usd":0.00138,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.039409750024788}
{"item_id":"semeval2013.d003.s006.t000","gold_sense_keys":["company%1:14:01::"],"candidates":[{"index":1,"sense_key":"company%1:14:01::","synset_id":"company.n.01"},{"index":2,"sense_key":"company%1:14:03::","synset_id":"company.n.02"},{"index":3,"sense_key":"company%1:26:00::","synset_id":"company.n.03"},{"index":4,"sense_key":"company%1:14:02::","synset_id":"company.n.04"},{"index":5,"sense_key":"company%1:18:00::","synset_id":"caller.n.01"},{"index":6,"sense_key":"company%1:14:00::","synset_id":"company.n.06"},{"index":7,"sense_key":"company%1:14:06::","synset_id":"party.n.03"},{"index":8,"sense_key":"company%1:14:04::","synset_id":"ship's_company.n.01"},{"index":9,"sense_key":"company%1:14:05::","synset_id":"company.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"company%1:14:01::","call_ids":["semeval2013.d003.s006.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"company%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":979,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002078,"input_uncached_usd":0.001958,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0135749580804259}
{"item_id":"semeval2013.d003.s006.t001","gold_sense_keys":["deal%1:10:00::"],"candidates":[{"index":1,"sense_key":"deal%1:04:02::","synset_id":"deal.n.01"},{"index":2,"sense_key":"deal%1:10:00::","synset_id":"bargain.n.01"},{"index":3,"sense_key":"deal%1:23:00::","synset_id":"batch.n.02"},{"index":4,"sense_key":"deal%1:27:00::","synset_id":"deal.n.04"},{"index":5,"sense_key":"deal%1:27:01::","synset_id":"softwood.n.01"},{"index":6,"sense_key":"deal%1:14:00::","synset_id":"hand.n.06"},{"index":7,"sense_key":"deal%1:11:00::","synset_id":"deal.n.07"},{"index":8,"sense_key":"deal%1:04:00::","synset_id":"deal.n.08"},{"index":9,"sense_key":"deal%1:04:01::","synset_id":"deal.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"deal%1:10:00::","call_ids":["semeval2013.d003.s006.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"deal%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1039,"cached_input_tokens":0,"output_tokens":136,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003438,"input_uncached_usd":0.002078,"input_cached_usd":0.0,"output_usd":0.00136,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":6.467540042009205}
{"item_id":"semeval2013.d003.s006.t002","gold_sense_keys":["field%1:15:05::"],"candidates":[{"index":1,"sense_key":"field%1:15:00::","synset_id":"field.n.01"},{"index":2,"sense_key":"field%1:15:04::","synset_id":"battlefield.n.01"},{"index":3,"sense_key":"field%1:15:01::","synset_id":"field.n.03"},{"index":4,"sense_key":"field%1:09:00::","synset_id":"discipline.n.01"},{"index":5,"sense_key":"field%1:19:00::","synset_id":"field.n.05"},{"index":6,"sense_key":"field%1:04:00::","synset_id":"field.n.06"},{"index":7,"sense_key":"field%1:26:00::","synset_id":"sphere.n.01"},{"index":8,"sense_key":"field%1:15:02::","synset_id":"playing_field.n.02"},{"index":9,"sense_key":"field%1:17:00::","synset_id":"plain.n.01"},{"index":10,"sense_key":"field%1:14:00::","synset_id":"field.n.10"},{"index":11,"sense_key":"field%1:15:03::","synset_id":"field.n.11"},{"index":12,"sense_key":"field%1:14:01::","synset_id":"field.n.12"},{"index":13,"sense_key":"field%1:14:02::","synset_id":"field.n.13"},{"index":14,"sense_key":"field%1:15:05::","synset_id":"field.n.14"},{"index":15,"sense_key":"field%1:14:03::","synset_id":"field.n.15"},{"index":16,"sense_key":"field%1:09:01::","synset_id":"field.n.16"},{"index":17,"sense_key":"field%1:06:00::","synset_id":"airfield.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":14,"chosen_sense_key":"field%1:15:05::","call_ids":["semeval2013.d003.s006.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":14,"predicted_sense_key":"field%1:15:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1561,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0032419999999999997,"input_uncached_usd":0.0031219999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.06619120889809}
{"item_id":"semeval2013.d003.s007.t000","gold_sense_keys":["representation%1:04:01::"],"candidates":[{"index":1,"sense_key":"representation%1:09:00::","synset_id":"representation.n.01"},{"index":2,"sense_key":"representation%1:06:00::","synset_id":"representation.n.02"},{"index":3,"sense_key":"representation%1:04:01::","synset_id":"representation.n.03"},{"index":4,"sense_key":"representation%1:26:00::","synset_id":"representation.n.04"},{"index":5,"sense_key":"representation%1:14:00::","synset_id":"representation.n.05"},{"index":6,"sense_key":"representation%1:10:00::","synset_id":"representation.n.06"},{"index":7,"sense_key":"representation%1:10:01::","synset_id":"theatrical_performance.n.01"},{"index":8,"sense_key":"representation%1:10:02::","synset_id":"representation.n.08"},{"index":9,"sense_key":"representation%1:07:00::","synset_id":"representation.n.09"},{"index":10,"sense_key":"representation%1:04:00::","synset_id":"representation.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"representation%1:14:00::","call_ids":["semeval2013.d003.s007.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"representation%1:14:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1017,"cached_input_tokens":0,"output_tokens":324,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.005274,"input_uncached_usd":0.002034,"input_cached_usd":0.0,"output_usd":0.0032400000000000003,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.969380083028227}
{"item_id":"semeval2013.d003.s007.t001","gold_sense_keys":["oil%1:27:01::"],"candidates":[{"index":1,"sense_key":"oil%1:27:00::","synset_id":"oil.n.01"},{"index":2,"sense_key":"oil%1:06:01::","synset_id":"oil.n.02"},{"index":3,"sense_key":"oil%1:27:01::","synset_id":"petroleum.n.01"},{"index":4,"sense_key":"oil%1:13:00::","synset_id":"vegetable_oil.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"oil%1:27:01::","call_ids":["semeval2013.d003.s007.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"oil%1:27:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":721,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001562,"input_uncached_usd":0.001442,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0378377499291673}
{"item_id":"semeval2013.d003.s007.t002","gold_sense_keys":["giant%1:14:00::"],"candidates":[{"index":1,"sense_key":"giant%1:05:00::","synset_id":"giant.n.01"},{"index":2,"sense_key":"giant%1:18:01::","synset_id":"colossus.n.02"},{"index":3,"sense_key":"giant%1:14:00::","synset_id":"giant.n.03"},{"index":4,"sense_key":"giant%1:18:03::","synset_id":"giant.n.04"},{"index":5,"sense_key":"giant%1:18:00::","synset_id":"giant.n.05"},{"index":6,"sense_key":"giant%1:18:02::","synset_id":"giant.n.06"},{"index":7,"sense_key":"giant%1:17:00::","synset_id":"giant_star.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"giant%1:14:00::","call_ids":["semeval2013.d003.s007.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"giant%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":847,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001814,"input_uncached_usd":0.001694,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2682529159355909}
{"item_id":"semeval2013.d003.s007.t005","gold_sense_keys":["analyst%1:18:00::","analyst%1:18:02::"],"candidates":[{"index":1,"sense_key":"analyst%1:18:00::","synset_id":"analyst.n.01"},{"index":2,"sense_key":"analyst%1:18:02::","synset_id":"analyst.n.02"},{"index":3,"sense_key":"analyst%1:18:01::","synset_id":"analyst.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"analyst%1:18:02::","call_ids":["semeval2013.d003.s007.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"analyst%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":651,"cached_input_tokens":0,"output_tokens":177,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003072,"input_uncached_usd":0.001302,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.4680152910295874}
{"item_id":"semeval2013.d003.s008.t001","gold_sense_keys":["door%1:07:00::"],"candidates":[{"index":1,"sense_key":"door%1:06:00::","synset_id":"door.n.01"},{"index":2,"sense_key":"door%1:06:01::","synset_id":"doorway.n.01"},{"index":3,"sense_key":"door%1:07:00::","synset_id":"door.n.03"},{"index":4,"sense_key":"door%1:06:04::","synset_id":"door.n.04"},{"index":5,"sense_key":"door%1:06:03::","synset_id":"door.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"door%1:07:00::","call_ids":["semeval2013.d003.s008.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"door%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":756,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002142,"input_uncached_usd":0.0015119999999999999,"input_cached_usd":0.0,"output_usd":0.00063,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.332088917028159}
{"item_id":"semeval2013.d003.s008.t002","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["semeval2013.d003.s008.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":695,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00151,"input_uncached_usd":0.00139,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.969360625022091}
{"item_id":"semeval2013.d003.s008.t003","gold_sense_keys":["war%1:04:00::","war%1:26:00::"],"candidates":[{"index":1,"sense_key":"war%1:04:00::","synset_id":"war.n.01"},{"index":2,"sense_key":"war%1:26:00::","synset_id":"war.n.02"},{"index":3,"sense_key":"war%1:04:02::","synset_id":"war.n.03"},{"index":4,"sense_key":"war%1:04:01::","synset_id":"war.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"war%1:04:00::","call_ids":["semeval2013.d003.s008.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"war%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":684,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001488,"input_uncached_usd":0.0013679999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0634874160168692}
{"item_id":"semeval2013.d003.s008.t004","gold_sense_keys":["u.s.%1:15:00::"],"candidates":[{"index":1,"sense_key":"u.s.%1:14:00::","synset_id":"united_states_government.n.01"},{"index":2,"sense_key":"us%1:15:00::","synset_id":"united_states.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"us%1:15:00::","call_ids":["semeval2013.d003.s008.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"us%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":613,"cached_input_tokens":0,"output_tokens":86,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002086,"input_uncached_usd":0.0012259999999999999,"input_cached_usd":0.0,"output_usd":0.0008600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7432725409744307}
{"item_id":"semeval2013.d003.s008.t005","gold_sense_keys":["company%1:14:01::"],"candidates":[{"index":1,"sense_key":"company%1:14:01::","synset_id":"company.n.01"},{"index":2,"sense_key":"company%1:14:03::","synset_id":"company.n.02"},{"index":3,"sense_key":"company%1:26:00::","synset_id":"company.n.03"},{"index":4,"sense_key":"company%1:14:02::","synset_id":"company.n.04"},{"index":5,"sense_key":"company%1:18:00::","synset_id":"caller.n.01"},{"index":6,"sense_key":"company%1:14:00::","synset_id":"company.n.06"},{"index":7,"sense_key":"company%1:14:06::","synset_id":"party.n.03"},{"index":8,"sense_key":"company%1:14:04::","synset_id":"ship's_company.n.01"},{"index":9,"sense_key":"company%1:14:05::","synset_id":"company.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"company%1:14:01::","call_ids":["semeval2013.d003.s008.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"company%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":933,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001986,"input_uncached_usd":0.0018659999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0641715839738026}
{"item_id":"semeval2013.d003.s008.t007","gold_sense_keys":["editor%1:18:00::"],"candidates":[{"index":1,"sense_key":"editor%1:18:00::","synset_id":"editor.n.01"},{"index":2,"sense_key":"editor%1:10:00::","synset_id":"editor_program.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"editor%1:18:00::","call_ids":["semeval2013.d003.s008.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"editor%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":576,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0012720000000000001,"input_uncached_usd":0.001152,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.057744082994759}
{"item_id":"semeval2013.d003.s008.t008","gold_sense_keys":["news%1:10:00::","news%1:10:01::"],"candidates":[{"index":1,"sense_key":"news%1:10:00::","synset_id":"news.n.01"},{"index":2,"sense_key":"news%1:10:01::","synset_id":"news.n.02"},{"index":3,"sense_key":"news%1:10:02::","synset_id":"news_program.n.01"},{"index":4,"sense_key":"news%1:10:03::","synset_id":"news.n.04"},{"index":5,"sense_key":"news%1:07:00::","synset_id":"newsworthiness.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"news%1:10:00::","call_ids":["semeval2013.d003.s008.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"news%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":750,"cached_input_tokens":0,"output_tokens":115,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0026500000000000004,"input_uncached_usd":0.0015,"input_cached_usd":0.0,"output_usd":0.0011500000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7105887499637902}
{"item_id":"semeval2013.d003.s008.t009","gold_sense_keys":["outlet%1:06:01::"],"candidates":[{"index":1,"sense_key":"outlet%1:06:01::","synset_id":"mercantile_establishment.n.01"},{"index":2,"sense_key":"outlet%1:06:02::","synset_id":"wall_socket.n.01"},{"index":3,"sense_key":"outlet%1:06:00::","synset_id":"exit.n.01"},{"index":4,"sense_key":"outlet%1:04:00::","synset_id":"release.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"outlet%1:06:01::","call_ids":["semeval2013.d003.s008.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"outlet%1:06:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":724,"cached_input_tokens":0,"output_tokens":152,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002968,"input_uncached_usd":0.0014479999999999999,"input_cached_usd":0.0,"output_usd":0.00152,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.429292832966894}
{"item_id":"semeval2013.d003.s010.t000","gold_sense_keys":["concern%1:12:01::"],"candidates":[{"index":1,"sense_key":"concern%1:09:00::","synset_id":"concern.n.01"},{"index":2,"sense_key":"concern%1:12:01::","synset_id":"concern.n.02"},{"index":3,"sense_key":"concern%1:12:00::","synset_id":"concern.n.03"},{"index":4,"sense_key":"concern%1:09:01::","synset_id":"concern.n.04"},{"index":5,"sense_key":"concern%1:14:00::","synset_id":"business.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"concern%1:12:01::","call_ids":["semeval2013.d003.s010.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"concern%1:12:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":886,"cached_input_tokens":0,"output_tokens":165,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003422,"input_uncached_usd":0.001772,"input_cached_usd":0.0,"output_usd":0.0016500000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9735547919990495}
{"item_id":"semeval2013.d003.s010.t001","gold_sense_keys":["security%1:04:00::","security%1:26:00::"],"candidates":[{"index":1,"sense_key":"security%1:26:00::","synset_id":"security.n.01"},{"index":2,"sense_key":"security%1:21:01::","synset_id":"security.n.02"},{"index":3,"sense_key":"security%1:12:00::","synset_id":"security.n.03"},{"index":4,"sense_key":"security%1:21:04::","synset_id":"security.n.04"},{"index":5,"sense_key":"security%1:21:00::","synset_id":"security.n.05"},{"index":6,"sense_key":"security%1:14:00::","synset_id":"security.n.06"},{"index":7,"sense_key":"security%1:10:00::","synset_id":"security.n.07"},{"index":8,"sense_key":"security%1:06:00::","synset_id":"security_system.n.02"},{"index":9,"sense_key":"security%1:04:00::","synset_id":"security.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"security%1:26:00::","call_ids":["semeval2013.d003.s010.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"security%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1093,"cached_input_tokens":0,"output_tokens":106,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003246,"input_uncached_usd":0.002186,"input_cached_usd":0.0,"output_usd":0.0010600000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.838008291902952}
{"item_id":"semeval2013.d003.s010.t002","gold_sense_keys":["bombing%1:04:01::"],"candidates":[{"index":1,"sense_key":"bombing%1:04:00::","synset_id":"bombing.n.01"},{"index":2,"sense_key":"bombing%1:04:01::","synset_id":"bombing.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bombing%1:04:01::","call_ids":["semeval2013.d003.s010.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bombing%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":642,"cached_input_tokens":0,"output_tokens":75,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0020340000000000002,"input_uncached_usd":0.001284,"input_cached_usd":0.0,"output_usd":0.00075,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.184207540936768}
{"item_id":"semeval2013.d003.s010.t004","gold_sense_keys":["instability%1:07:00::","instability%1:07:01::","instability%1:26:00::"],"candidates":[{"index":1,"sense_key":"instability%1:26:00::","synset_id":"instability.n.01"},{"index":2,"sense_key":"instability%1:07:01::","synset_id":"instability.n.02"},{"index":3,"sense_key":"instability%1:26:01::","synset_id":"imbalance.n.01"},{"index":4,"sense_key":"instability%1:07:00::","synset_id":"instability.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"instability%1:26:00::","call_ids":["semeval2013.d003.s010.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"instability%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":724,"cached_input_tokens":0,"output_tokens":165,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003098,"input_uncached_usd":0.0014479999999999999,"input_cached_usd":0.0,"output_usd":0.0016500000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.3279725830070674}
{"item_id":"semeval2013.d003.s010.t005","gold_sense_keys":["u.s.%1:14:00::"],"candidates":[{"index":1,"sense_key":"u.s.%1:14:00::","synset_id":"united_states_government.n.01"},{"index":2,"sense_key":"us%1:15:00::","synset_id":"united_states.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"u.s.%1:14:00::","call_ids":["semeval2013.d003.s010.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"u.s.%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":725,"cached_input_tokens":0,"output_tokens":168,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00313,"input_uncached_usd":0.00145,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.5263166669756174}
{"item_id":"semeval2013.d003.s010.t009","gold_sense_keys":["world%1:17:00::"],"candidates":[{"index":1,"sense_key":"world%1:17:01::","synset_id":"universe.n.01"},{"index":2,"sense_key":"world%1:14:01::","synset_id":"world.n.02"},{"index":3,"sense_key":"world%1:09:00::","synset_id":"world.n.03"},{"index":4,"sense_key":"world%1:17:00::","synset_id":"earth.n.01"},{"index":5,"sense_key":"world%1:14:02::","synset_id":"populace.n.01"},{"index":6,"sense_key":"world%1:17:02::","synset_id":"world.n.06"},{"index":7,"sense_key":"world%1:09:01::","synset_id":"worldly_concern.n.01"},{"index":8,"sense_key":"world%1:05:00::","synset_id":"world.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"world%1:17:00::","call_ids":["semeval2013.d003.s010.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"world%1:17:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1102,"cached_input_tokens":0,"output_tokens":91,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003114,"input_uncached_usd":0.002204,"input_cached_usd":0.0,"output_usd":0.0009100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.296426499960944}
{"item_id":"semeval2013.d003.s010.t011","gold_sense_keys":["analyst%1:18:00::","analyst%1:18:02::"],"candidates":[{"index":1,"sense_key":"analyst%1:18:00::","synset_id":"analyst.n.01"},{"index":2,"sense_key":"analyst%1:18:02::","synset_id":"analyst.n.02"},{"index":3,"sense_key":"analyst%1:18:01::","synset_id":"analyst.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"analyst%1:18:02::","call_ids":["semeval2013.d003.s010.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"analyst%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":702,"cached_input_tokens":0,"output_tokens":117,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002574,"input_uncached_usd":0.0014039999999999999,"input_cached_usd":0.0,"output_usd":0.00117,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.3282693750225008}
{"item_id":"semeval2013.d003.s011.t000","gold_sense_keys":["u.s.%1:15:00::"],"candidates":[{"index":1,"sense_key":"u.s.%1:14:00::","synset_id":"united_states_government.n.01"},{"index":2,"sense_key":"us%1:15:00::","synset_id":"united_states.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"us%1:15:00::","call_ids":["semeval2013.d003.s011.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"us%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":752,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002124,"input_uncached_usd":0.001504,"input_cached_usd":0.0,"output_usd":0.00062,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4465062919771299}
{"item_id":"semeval2013.d003.s011.t002","gold_sense_keys":["case%1:11:00::"],"candidates":[{"index":1,"sense_key":"case%1:11:00::","synset_id":"case.n.01"},{"index":2,"sense_key":"case%1:26:00::","synset_id":"event.n.02"},{"index":3,"sense_key":"case%1:04:00::","synset_id":"lawsuit.n.01"},{"index":4,"sense_key":"case%1:09:00::","synset_id":"case.n.04"},{"index":5,"sense_key":"case%1:06:00::","synset_id":"case.n.05"},{"index":6,"sense_key":"case%1:18:03::","synset_id":"case.n.06"},{"index":7,"sense_key":"case%1:18:02::","synset_id":"subject.n.06"},{"index":8,"sense_key":"case%1:10:02::","synset_id":"case.n.08"},{"index":9,"sense_key":"case%1:10:01::","synset_id":"case.n.09"},{"index":10,"sense_key":"case%1:23:00::","synset_id":"case.n.10"},{"index":11,"sense_key":"case%1:10:00::","synset_id":"case.n.11"},{"index":12,"sense_key":"case%1:26:02::","synset_id":"case.n.12"},{"index":13,"sense_key":"case%1:18:04::","synset_id":"character.n.05"},{"index":14,"sense_key":"case%1:10:03::","synset_id":"font.n.01"},{"index":15,"sense_key":"case%1:08:00::","synset_id":"sheath.n.02"},{"index":16,"sense_key":"case%1:06:06::","synset_id":"shell.n.08"},{"index":17,"sense_key":"case%1:06:07::","synset_id":"casing.n.03"},{"index":18,"sense_key":"case%1:06:03::","synset_id":"case.n.18"},{"index":19,"sense_key":"case%1:06:02::","synset_id":"case.n.19"},{"index":20,"sense_key":"case%1:06:01::","synset_id":"case.n.20"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"case%1:11:00::","call_ids":["semeval2013.d003.s011.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"case%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1829,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004228,"input_uncached_usd":0.003658,"input_cached_usd":0.0,"output_usd":0.0005700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6282837090548128}
{"item_id":"semeval2013.d003.s011.t005","gold_sense_keys":["government%1:14:00::"],"candidates":[{"index":1,"sense_key":"government%1:14:00::","synset_id":"government.n.01"},{"index":2,"sense_key":"government%1:04:00::","synset_id":"government.n.02"},{"index":3,"sense_key":"government%1:09:00::","synset_id":"government.n.03"},{"index":4,"sense_key":"government%1:09:01::","synset_id":"politics.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"government%1:14:00::","call_ids":["semeval2013.d003.s011.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"government%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":815,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00229,"input_uncached_usd":0.00163,"input_cached_usd":0.0,"output_usd":0.0006600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8787552089197561}
{"item_id":"semeval2013.d003.s011.t006","gold_sense_keys":["energy%1:19:01::"],"candidates":[{"index":1,"sense_key":"energy%1:19:00::","synset_id":"energy.n.01"},{"index":2,"sense_key":"energy%1:07:00::","synset_id":"energy.n.02"},{"index":3,"sense_key":"energy%1:07:02::","synset_id":"energy.n.03"},{"index":4,"sense_key":"energy%1:07:01::","synset_id":"energy.n.04"},{"index":5,"sense_key":"energy%1:26:00::","synset_id":"energy.n.05"},{"index":6,"sense_key":"energy%1:19:01::","synset_id":"energy.n.06"},{"index":7,"sense_key":"energy%1:14:00::","synset_id":"department_of_energy.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"energy%1:19:01::","call_ids":["semeval2013.d003.s011.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"energy%1:19:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1078,"cached_input_tokens":0,"output_tokens":109,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0032459999999999998,"input_uncached_usd":0.002156,"input_cached_usd":0.0,"output_usd":0.00109,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1169487501028925}
{"item_id":"semeval2013.d003.s012.t000","gold_sense_keys":["company%1:14:01::"],"candidates":[{"index":1,"sense_key":"company%1:14:01::","synset_id":"company.n.01"},{"index":2,"sense_key":"company%1:14:03::","synset_id":"company.n.02"},{"index":3,"sense_key":"company%1:26:00::","synset_id":"company.n.03"},{"index":4,"sense_key":"company%1:14:02::","synset_id":"company.n.04"},{"index":5,"sense_key":"company%1:18:00::","synset_id":"caller.n.01"},{"index":6,"sense_key":"company%1:14:00::","synset_id":"company.n.06"},{"index":7,"sense_key":"company%1:14:06::","synset_id":"party.n.03"},{"index":8,"sense_key":"company%1:14:04::","synset_id":"ship's_company.n.01"},{"index":9,"sense_key":"company%1:14:05::","synset_id":"company.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"company%1:14:01::","call_ids":["semeval2013.d003.s012.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"company%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1122,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0023639999999999998,"input_uncached_usd":0.002244,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1856569580268115}
{"item_id":"semeval2013.d003.s012.t001","gold_sense_keys":["deal%1:10:00::"],"candidates":[{"index":1,"sense_key":"deal%1:04:02::","synset_id":"deal.n.01"},{"index":2,"sense_key":"deal%1:10:00::","synset_id":"bargain.n.01"},{"index":3,"sense_key":"deal%1:23:00::","synset_id":"batch.n.02"},{"index":4,"sense_key":"deal%1:27:00::","synset_id":"deal.n.04"},{"index":5,"sense_key":"deal%1:27:01::","synset_id":"softwood.n.01"},{"index":6,"sense_key":"deal%1:14:00::","synset_id":"hand.n.06"},{"index":7,"sense_key":"deal%1:11:00::","synset_id":"deal.n.07"},{"index":8,"sense_key":"deal%1:04:00::","synset_id":"deal.n.08"},{"index":9,"sense_key":"deal%1:04:01::","synset_id":"deal.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"deal%1:10:00::","call_ids":["semeval2013.d003.s012.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"deal%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1182,"cached_input_tokens":0,"output_tokens":121,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003574,"input_uncached_usd":0.0023639999999999998,"input_cached_usd":0.0,"output_usd":0.0012100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9960524579510093}
{"item_id":"semeval2013.d003.s012.t002","gold_sense_keys":["ministry%1:14:00::"],"candidates":[{"index":1,"sense_key":"ministry%1:14:01::","synset_id":"ministry.n.01"},{"index":2,"sense_key":"ministry%1:06:00::","synset_id":"ministry.n.02"},{"index":3,"sense_key":"ministry%1:14:00::","synset_id":"ministry.n.03"},{"index":4,"sense_key":"ministry%1:04:00::","synset_id":"ministry.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"ministry%1:14:00::","call_ids":["semeval2013.d003.s012.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"ministry%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":783,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001686,"input_uncached_usd":0.001566,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4368069170741364}
{"item_id":"semeval2013.d003.s013.t000","gold_sense_keys":["u.s.%1:15:00::"],"candidates":[{"index":1,"sense_key":"u.s.%1:14:00::","synset_id":"united_states_government.n.01"},{"index":2,"sense_key":"us%1:15:00::","synset_id":"united_states.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"us%1:15:00::","call_ids":["semeval2013.d003.s013.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"us%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":811,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002252,"input_uncached_usd":0.001622,"input_cached_usd":0.0,"output_usd":0.00063,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6883849590085447}
{"item_id":"semeval2013.d003.s013.t002","gold_sense_keys":["tie%1:26:01::"],"candidates":[{"index":1,"sense_key":"tie%1:06:01::","synset_id":"necktie.n.01"},{"index":2,"sense_key":"tie%1:26:01::","synset_id":"affiliation.n.01"},{"index":3,"sense_key":"tie%1:26:00::","synset_id":"tie.n.03"},{"index":4,"sense_key":"tie%1:06:02::","synset_id":"tie.n.04"},{"index":5,"sense_key":"tie%1:06:04::","synset_id":"link.n.02"},{"index":6,"sense_key":"tie%1:11:00::","synset_id":"draw.n.03"},{"index":7,"sense_key":"tie%1:10:00::","synset_id":"tie.n.07"},{"index":8,"sense_key":"tie%1:06:03::","synset_id":"tie.n.08"},{"index":9,"sense_key":"tie%1:06:00::","synset_id":"tie.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"tie%1:26:01::","call_ids":["semeval2013.d003.s013.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"tie%1:26:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1281,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002682,"input_uncached_usd":0.002562,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5078294159611687}
{"item_id":"semeval2013.d003.s013.t004","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["semeval2013.d003.s013.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":893,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001906,"input_uncached_usd":0.0017859999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.516048499965109}
{"item_id":"semeval2013.d003.s014.t000","gold_sense_keys":["company%1:14:01::"],"candidates":[{"index":1,"sense_key":"company%1:14:01::","synset_id":"company.n.01"},{"index":2,"sense_key":"company%1:14:03::","synset_id":"company.n.02"},{"index":3,"sense_key":"company%1:26:00::","synset_id":"company.n.03"},{"index":4,"sense_key":"company%1:14:02::","synset_id":"company.n.04"},{"index":5,"sense_key":"company%1:18:00::","synset_id":"caller.n.01"},{"index":6,"sense_key":"company%1:14:00::","synset_id":"company.n.06"},{"index":7,"sense_key":"company%1:14:06::","synset_id":"party.n.03"},{"index":8,"sense_key":"company%1:14:04::","synset_id":"ship's_company.n.01"},{"index":9,"sense_key":"company%1:14:05::","synset_id":"company.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"company%1:14:01::","call_ids":["semeval2013.d003.s014.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"company%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1102,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0023239999999999997,"input_uncached_usd":0.002204,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":6.352856916957535}
{"item_id":"semeval2013.d003.s014.t001","gold_sense_keys":["contract%1:10:00::"],"candidates":[{"index":1,"sense_key":"contract%1:10:00::","synset_id":"contract.n.01"},{"index":2,"sense_key":"contract%1:10:01::","synset_id":"contract.n.02"},{"index":3,"sense_key":"contract%1:04:01::","synset_id":"contract.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"contract%1:10:00::","call_ids":["semeval2013.d003.s014.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"contract%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":776,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001672,"input_uncached_usd":0.001552,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8901238749967888}
{"item_id":"semeval2013.d003.s015.t000","gold_sense_keys":["bid%2:40:00::"],"candidates":[{"index":1,"sense_key":"bid%2:40:00::","synset_id":"offer.v.07"},{"index":2,"sense_key":"bid%2:32:01::","synset_id":"wish.v.06"},{"index":3,"sense_key":"bid%2:32:02::","synset_id":"bid.v.03"},{"index":4,"sense_key":"bid%2:40:01::","synset_id":"bid.v.04"},{"index":5,"sense_key":"bid%2:40:04::","synset_id":"bid.v.05"},{"index":6,"sense_key":"bid%2:32:00::","synset_id":"invite.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bid%2:40:00::","call_ids":["semeval2013.d003.s015.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bid%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":961,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002582,"input_uncached_usd":0.0019219999999999999,"input_cached_usd":0.0,"output_usd":0.0006600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.418500707950443}
{"item_id":"semeval2013.d003.s015.t001","gold_sense_keys":["contract%1:10:00::"],"candidates":[{"index":1,"sense_key":"contract%1:10:00::","synset_id":"contract.n.01"},{"index":2,"sense_key":"contract%1:10:01::","synset_id":"contract.n.02"},{"index":3,"sense_key":"contract%1:04:01::","synset_id":"contract.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"contract%1:10:00::","call_ids":["semeval2013.d003.s015.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"contract%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":796,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001712,"input_uncached_usd":0.0015919999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3371090000728145}
{"item_id":"semeval2013.d003.s015.t002","gold_sense_keys":["company%1:14:01::"],"candidates":[{"index":1,"sense_key":"company%1:14:01::","synset_id":"company.n.01"},{"index":2,"sense_key":"company%1:14:03::","synset_id":"company.n.02"},{"index":3,"sense_key":"company%1:26:00::","synset_id":"company.n.03"},{"index":4,"sense_key":"company%1:14:02::","synset_id":"company.n.04"},{"index":5,"sense_key":"company%1:18:00::","synset_id":"caller.n.01"},{"index":6,"sense_key":"company%1:14:00::","synset_id":"company.n.06"},{"index":7,"sense_key":"company%1:14:06::","synset_id":"party.n.03"},{"index":8,"sense_key":"company%1:14:04::","synset_id":"ship's_company.n.01"},{"index":9,"sense_key":"company%1:14:05::","synset_id":"company.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"company%1:14:01::","call_ids":["semeval2013.d003.s015.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"company%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1121,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002362,"input_uncached_usd":0.002242,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1297069580759853}
{"item_id":"semeval2013.d003.s015.t003","gold_sense_keys":["stake%1:21:02::"],"candidates":[{"index":1,"sense_key":"stake%1:21:02::","synset_id":"interest.n.05"},{"index":2,"sense_key":"stake%1:10:00::","synset_id":"post.n.09"},{"index":3,"sense_key":"stake%1:06:01::","synset_id":"stake.n.03"},{"index":4,"sense_key":"stake%1:21:00::","synset_id":"stake.n.04"},{"index":5,"sense_key":"stake%1:06:00::","synset_id":"stake.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"stake%1:21:02::","call_ids":["semeval2013.d003.s015.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"stake%1:21:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":923,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001966,"input_uncached_usd":0.001846,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.144844791968353}
{"item_id":"semeval2013.d003.s015.t004","gold_sense_keys":["contract%1:10:00::"],"candidates":[{"index":1,"sense_key":"contract%1:10:00::","synset_id":"contract.n.01"},{"index":2,"sense_key":"contract%1:10:01::","synset_id":"contract.n.02"},{"index":3,"sense_key":"contract%1:04:01::","synset_id":"contract.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"contract%1:10:00::","call_ids":["semeval2013.d003.s015.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"contract%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":796,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001712,"input_uncached_usd":0.0015919999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9513455840060487}
{"item_id":"semeval2013.d003.s015.t005","gold_sense_keys":["field%1:15:05::"],"candidates":[{"index":1,"sense_key":"field%1:15:00::","synset_id":"field.n.01"},{"index":2,"sense_key":"field%1:15:04::","synset_id":"battlefield.n.01"},{"index":3,"sense_key":"field%1:15:01::","synset_id":"field.n.03"},{"index":4,"sense_key":"field%1:09:00::","synset_id":"discipline.n.01"},{"index":5,"sense_key":"field%1:19:00::","synset_id":"field.n.05"},{"index":6,"sense_key":"field%1:04:00::","synset_id":"field.n.06"},{"index":7,"sense_key":"field%1:26:00::","synset_id":"sphere.n.01"},{"index":8,"sense_key":"field%1:15:02::","synset_id":"playing_field.n.02"},{"index":9,"sense_key":"field%1:17:00::","synset_id":"plain.n.01"},{"index":10,"sense_key":"field%1:14:00::","synset_id":"field.n.10"},{"index":11,"sense_key":"field%1:15:03::","synset_id":"field.n.11"},{"index":12,"sense_key":"field%1:14:01::","synset_id":"field.n.12"},{"index":13,"sense_key":"field%1:14:02::","synset_id":"field.n.13"},{"index":14,"sense_key":"field%1:15:05::","synset_id":"field.n.14"},{"index":15,"sense_key":"field%1:14:03::","synset_id":"field.n.15"},{"index":16,"sense_key":"field%1:09:01::","synset_id":"field.n.16"},{"index":17,"sense_key":"field%1:06:00::","synset_id":"airfield.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":14,"chosen_sense_key":"field%1:15:05::","call_ids":["semeval2013.d003.s015.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":14,"predicted_sense_key":"field%1:15:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1704,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004008,"input_uncached_usd":0.003408,"input_cached_usd":0.0,"output_usd":0.0006000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7766118749277666}
{"item_id":"semeval2013.d003.s016.t000","gold_sense_keys":["china%1:15:00::"],"candidates":[{"index":1,"sense_key":"china%1:15:00::","synset_id":"china.n.01"},{"index":2,"sense_key":"china%1:06:00::","synset_id":"china.n.02"},{"index":3,"sense_key":"china%1:15:01::","synset_id":"taiwan.n.01"},{"index":4,"sense_key":"china%1:06:01::","synset_id":"chinaware.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"china%1:15:00::","call_ids":["semeval2013.d003.s016.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"china%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":834,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001788,"input_uncached_usd":0.001668,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8729364579776302}
{"item_id":"semeval2013.d003.s016.t002","gold_sense_keys":["power%1:14:00::"],"candidates":[{"index":1,"sense_key":"power%1:07:00::","synset_id":"power.n.01"},{"index":2,"sense_key":"power%1:19:00::","synset_id":"power.n.02"},{"index":3,"sense_key":"power%1:09:01::","synset_id":"ability.n.02"},{"index":4,"sense_key":"power%1:26:00::","synset_id":"office.n.04"},{"index":5,"sense_key":"power%1:18:00::","synset_id":"power.n.05"},{"index":6,"sense_key":"power%1:10:00::","synset_id":"exponent.n.03"},{"index":7,"sense_key":"power%1:07:02::","synset_id":"might.n.01"},{"index":8,"sense_key":"power%1:14:00::","synset_id":"world_power.n.01"},{"index":9,"sense_key":"power%1:18:01::","synset_id":"baron.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"power%1:14:00::","call_ids":["semeval2013.d003.s016.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"power%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1112,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002894,"input_uncached_usd":0.002224,"input_cached_usd":0.0,"output_usd":0.00067,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7965006660670042}
{"item_id":"semeval2013.d003.s017.t000","gold_sense_keys":["company%1:14:01::"],"candidates":[{"index":1,"sense_key":"company%1:14:01::","synset_id":"company.n.01"},{"index":2,"sense_key":"company%1:14:03::","synset_id":"company.n.02"},{"index":3,"sense_key":"company%1:26:00::","synset_id":"company.n.03"},{"index":4,"sense_key":"company%1:14:02::","synset_id":"company.n.04"},{"index":5,"sense_key":"company%1:18:00::","synset_id":"caller.n.01"},{"index":6,"sense_key":"company%1:14:00::","synset_id":"company.n.06"},{"index":7,"sense_key":"company%1:14:06::","synset_id":"party.n.03"},{"index":8,"sense_key":"company%1:14:04::","synset_id":"ship's_company.n.01"},{"index":9,"sense_key":"company%1:14:05::","synset_id":"company.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"company%1:14:01::","call_ids":["semeval2013.d003.s017.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"company%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1050,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0022199999999999998,"input_uncached_usd":0.0021,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1514007500372827}
{"item_id":"semeval2013.d003.s017.t001","gold_sense_keys":["par%1:26:00::"],"candidates":[{"index":1,"sense_key":"par%1:23:00::","synset_id":"par.n.01"},{"index":2,"sense_key":"par%1:26:00::","synset_id":"equality.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"par%1:26:00::","call_ids":["semeval2013.d003.s017.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"par%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":689,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001498,"input_uncached_usd":0.0013779999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0084560419199988}
{"item_id":"semeval2013.d003.s017.t003","gold_sense_keys":["commitment%1:04:02::"],"candidates":[{"index":1,"sense_key":"commitment%1:07:01::","synset_id":"committedness.n.01"},{"index":2,"sense_key":"commitment%1:04:00::","synset_id":"commitment.n.02"},{"index":3,"sense_key":"commitment%1:04:02::","synset_id":"commitment.n.03"},{"index":4,"sense_key":"commitment%1:10:00::","synset_id":"commitment.n.04"},{"index":5,"sense_key":"commitment%1:04:01::","synset_id":"commitment.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"commitment%1:04:02::","call_ids":["semeval2013.d003.s017.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"commitment%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":865,"cached_input_tokens":0,"output_tokens":76,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00249,"input_uncached_usd":0.00173,"input_cached_usd":0.0,"output_usd":0.00076,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9283264580881223}
{"item_id":"semeval2013.d003.s018.t000","gold_sense_keys":["company%1:14:01::"],"candidates":[{"index":1,"sense_key":"company%1:14:01::","synset_id":"company.n.01"},{"index":2,"sense_key":"company%1:14:03::","synset_id":"company.n.02"},{"index":3,"sense_key":"company%1:26:00::","synset_id":"company.n.03"},{"index":4,"sense_key":"company%1:14:02::","synset_id":"company.n.04"},{"index":5,"sense_key":"company%1:18:00::","synset_id":"caller.n.01"},{"index":6,"sense_key":"company%1:14:00::","synset_id":"company.n.06"},{"index":7,"sense_key":"company%1:14:06::","synset_id":"party.n.03"},{"index":8,"sense_key":"company%1:14:04::","synset_id":"ship's_company.n.01"},{"index":9,"sense_key":"company%1:14:05::","synset_id":"company.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"company%1:14:01::","call_ids":["semeval2013.d003.s018.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"company%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1012,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0021439999999999996,"input_uncached_usd":0.0020239999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1517538749612868}
{"item_id":"semeval2013.d003.s018.t001","gold_sense_keys":["bid%1:10:00::"],"candidates":[{"index":1,"sense_key":"bid%1:10:03::","synset_id":"command.n.01"},{"index":2,"sense_key":"bid%1:04:00::","synset_id":"bid.n.02"},{"index":3,"sense_key":"bid%1:10:00::","synset_id":"bid.n.03"},{"index":4,"sense_key":"bid%1:10:01::","synset_id":"bid.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"bid%1:10:00::","call_ids":["semeval2013.d003.s018.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"bid%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":724,"cached_input_tokens":0,"output_tokens":75,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002198,"input_uncached_usd":0.0014479999999999999,"input_cached_usd":0.0,"output_usd":0.00075,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8737929170019925}
{"item_id":"semeval2013.d003.s018.t002","gold_sense_keys":["offer%1:10:00::"],"candidates":[{"index":1,"sense_key":"offer%1:10:01::","synset_id":"offer.n.01"},{"index":2,"sense_key":"offer%1:10:00::","synset_id":"offer.n.02"},{"index":3,"sense_key":"offer%1:04:01::","synset_id":"crack.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"offer%1:10:00::","call_ids":["semeval2013.d003.s018.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"offer%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":700,"cached_input_tokens":0,"output_tokens":88,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00228,"input_uncached_usd":0.0014,"input_cached_usd":0.0,"output_usd":0.00088,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.744360041921027}
{"item_id":"semeval2013.d003.s018.t003","gold_sense_keys":["barrel%1:23:01::"],"candidates":[{"index":1,"sense_key":"barrel%1:06:01::","synset_id":"barrel.n.01"},{"index":2,"sense_key":"barrel%1:06:00::","synset_id":"barrel.n.02"},{"index":3,"sense_key":"barrel%1:25:00::","synset_id":"barrel.n.03"},{"index":4,"sense_key":"barrel%1:23:04::","synset_id":"barrel.n.04"},{"index":5,"sense_key":"barrel%1:23:01::","synset_id":"barrel.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"barrel%1:23:01::","call_ids":["semeval2013.d003.s018.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"barrel%1:23:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":787,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0021739999999999997,"input_uncached_usd":0.0015739999999999999,"input_cached_usd":0.0,"output_usd":0.0006000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7496612078975886}
{"item_id":"semeval2013.d003.s018.t004","gold_sense_keys":["fee%1:21:00::"],"candidates":[{"index":1,"sense_key":"fee%1:21:00::","synset_id":"fee.n.01"},{"index":2,"sense_key":"fee%1:21:01::","synset_id":"fee.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"fee%1:21:00::","call_ids":["semeval2013.d003.s018.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"fee%1:21:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":587,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001294,"input_uncached_usd":0.001174,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.325190250063315}
{"item_id":"semeval2013.d003.s018.t005","gold_sense_keys":["ministry%1:14:00::"],"candidates":[{"index":1,"sense_key":"ministry%1:14:01::","synset_id":"ministry.n.01"},{"index":2,"sense_key":"ministry%1:06:00::","synset_id":"ministry.n.02"},{"index":3,"sense_key":"ministry%1:14:00::","synset_id":"ministry.n.03"},{"index":4,"sense_key":"ministry%1:04:00::","synset_id":"ministry.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"ministry%1:14:00::","call_ids":["semeval2013.d003.s018.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"ministry%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":672,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001464,"input_uncached_usd":0.001344,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0029376670718193}
{"item_id":"semeval2013.d003.s018.t006","gold_sense_keys":["output%1:04:00::","output%1:06:01::","output%1:23:00::"],"candidates":[{"index":1,"sense_key":"output%1:06:00::","synset_id":"end_product.n.01"},{"index":2,"sense_key":"output%1:04:00::","synset_id":"output.n.02"},{"index":3,"sense_key":"output%1:10:02::","synset_id":"output_signal.n.01"},{"index":4,"sense_key":"output%1:23:00::","synset_id":"output.n.04"},{"index":5,"sense_key":"output%1:06:01::","synset_id":"output.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"output%1:23:00::","call_ids":["semeval2013.d003.s018.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"output%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":740,"cached_input_tokens":0,"output_tokens":146,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00294,"input_uncached_usd":0.00148,"input_cached_usd":0.0,"output_usd":0.0014600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9742842910345644}
{"item_id":"semeval2013.d003.s018.t007","gold_sense_keys":["level%1:07:00::"],"candidates":[{"index":1,"sense_key":"level%1:07:00::","synset_id":"degree.n.01"},{"index":2,"sense_key":"level%1:26:01::","synset_id":"grade.n.02"},{"index":3,"sense_key":"level%1:26:00::","synset_id":"degree.n.02"},{"index":4,"sense_key":"level%1:07:01::","synset_id":"level.n.04"},{"index":5,"sense_key":"level%1:06:00::","synset_id":"level.n.05"},{"index":6,"sense_key":"level%1:06:03::","synset_id":"horizontal_surface.n.01"},{"index":7,"sense_key":"level%1:09:00::","synset_id":"level.n.07"},{"index":8,"sense_key":"level%1:06:01::","synset_id":"floor.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"level%1:07:00::","call_ids":["semeval2013.d003.s018.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"level%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1037,"cached_input_tokens":0,"output_tokens":41,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002484,"input_uncached_usd":0.002074,"input_cached_usd":0.0,"output_usd":0.00041000000000000005,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6147712080273777}
{"item_id":"semeval2013.d003.s018.t008","gold_sense_keys":["field%1:15:05::"],"candidates":[{"index":1,"sense_key":"field%1:15:00::","synset_id":"field.n.01"},{"index":2,"sense_key":"field%1:15:04::","synset_id":"battlefield.n.01"},{"index":3,"sense_key":"field%1:15:01::","synset_id":"field.n.03"},{"index":4,"sense_key":"field%1:09:00::","synset_id":"discipline.n.01"},{"index":5,"sense_key":"field%1:19:00::","synset_id":"field.n.05"},{"index":6,"sense_key":"field%1:04:00::","synset_id":"field.n.06"},{"index":7,"sense_key":"field%1:26:00::","synset_id":"sphere.n.01"},{"index":8,"sense_key":"field%1:15:02::","synset_id":"playing_field.n.02"},{"index":9,"sense_key":"field%1:17:00::","synset_id":"plain.n.01"},{"index":10,"sense_key":"field%1:14:00::","synset_id":"field.n.10"},{"index":11,"sense_key":"field%1:15:03::","synset_id":"field.n.11"},{"index":12,"sense_key":"field%1:14:01::","synset_id":"field.n.12"},{"index":13,"sense_key":"field%1:14:02::","synset_id":"field.n.13"},{"index":14,"sense_key":"field%1:15:05::","synset_id":"field.n.14"},{"index":15,"sense_key":"field%1:14:03::","synset_id":"field.n.15"},{"index":16,"sense_key":"field%1:09:01::","synset_id":"field.n.16"},{"index":17,"sense_key":"field%1:06:00::","synset_id":"airfield.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":14,"chosen_sense_key":"field%1:15:05::","call_ids":["semeval2013.d003.s018.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":14,"predicted_sense_key":"field%1:15:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1592,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0033039999999999996,"input_uncached_usd":0.0031839999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2647039589937776}
{"item_id":"semeval2013.d004.s000.t000","gold_sense_keys":["bank%1:14:00::"],"candidates":[{"index":1,"sense_key":"bank%1:17:01::","synset_id":"bank.n.01"},{"index":2,"sense_key":"bank%1:14:00::","synset_id":"depository_financial_institution.n.01"},{"index":3,"sense_key":"bank%1:17:00::","synset_id":"bank.n.03"},{"index":4,"sense_key":"bank%1:14:01::","synset_id":"bank.n.04"},{"index":5,"sense_key":"bank%1:21:00::","synset_id":"bank.n.05"},{"index":6,"sense_key":"bank%1:21:01::","synset_id":"bank.n.06"},{"index":7,"sense_key":"bank%1:17:02::","synset_id":"bank.n.07"},{"index":8,"sense_key":"bank%1:06:01::","synset_id":"savings_bank.n.02"},{"index":9,"sense_key":"bank%1:06:00::","synset_id":"bank.n.09"},{"index":10,"sense_key":"bank%1:04:00::","synset_id":"bank.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bank%1:14:00::","call_ids":["semeval2013.d004.s000.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bank%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":910,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00194,"input_uncached_usd":0.00182,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1623807090800256}
{"item_id":"semeval2013.d004.s000.t001","gold_sense_keys":["state%1:14:01::"],"candidates":[{"index":1,"sense_key":"state%1:15:01::","synset_id":"state.n.01"},{"index":2,"sense_key":"state%1:03:00::","synset_id":"state.n.02"},{"index":3,"sense_key":"state%1:14:01::","synset_id":"state.n.03"},{"index":4,"sense_key":"state%1:14:00::","synset_id":"state.n.04"},{"index":5,"sense_key":"state%1:26:02::","synset_id":"state_of_matter.n.01"},{"index":6,"sense_key":"state%1:26:01::","synset_id":"state.n.06"},{"index":7,"sense_key":"state%1:15:00::","synset_id":"country.n.02"},{"index":8,"sense_key":"state%1:14:02::","synset_id":"department_of_state.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"state%1:14:01::","call_ids":["semeval2013.d004.s000.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"state%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":791,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0021219999999999998,"input_uncached_usd":0.0015819999999999999,"input_cached_usd":0.0,"output_usd":0.00054,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9743479159660637}
{"item_id":"semeval2013.d004.s000.t002","gold_sense_keys":["support%1:21:01::"],"candidates":[{"index":1,"sense_key":"support%1:04:04::","synset_id":"support.n.01"},{"index":2,"sense_key":"support%1:04:00::","synset_id":"support.n.02"},{"index":3,"sense_key":"support%1:09:00::","synset_id":"support.n.03"},{"index":4,"sense_key":"support%1:04:03::","synset_id":"support.n.04"},{"index":5,"sense_key":"support%1:04:02::","synset_id":"documentation.n.03"},{"index":6,"sense_key":"support%1:21:00::","synset_id":"support.n.06"},{"index":7,"sense_key":"support%1:06:01::","synset_id":"support.n.07"},{"index":8,"sense_key":"support%1:04:01::","synset_id":"support.n.08"},{"index":9,"sense_key":"support%1:10:01::","synset_id":"accompaniment.n.02"},{"index":10,"sense_key":"support%1:06:00::","synset_id":"support.n.10"},{"index":11,"sense_key":"support%1:21:01::","synset_id":"support.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"support%1:21:01::","call_ids":["semeval2013.d004.s000.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"support%1:21:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":987,"cached_input_tokens":0,"output_tokens":126,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003234,"input_uncached_usd":0.001974,"input_cached_usd":0.0,"output_usd":0.00126,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.4852970000356436}
{"item_id":"semeval2013.d004.s001.t001","gold_sense_keys":["bank%1:14:00::"],"candidates":[{"index":1,"sense_key":"bank%1:17:01::","synset_id":"bank.n.01"},{"index":2,"sense_key":"bank%1:14:00::","synset_id":"depository_financial_institution.n.01"},{"index":3,"sense_key":"bank%1:17:00::","synset_id":"bank.n.03"},{"index":4,"sense_key":"bank%1:14:01::","synset_id":"bank.n.04"},{"index":5,"sense_key":"bank%1:21:00::","synset_id":"bank.n.05"},{"index":6,"sense_key":"bank%1:21:01::","synset_id":"bank.n.06"},{"index":7,"sense_key":"bank%1:17:02::","synset_id":"bank.n.07"},{"index":8,"sense_key":"bank%1:06:01::","synset_id":"savings_bank.n.02"},{"index":9,"sense_key":"bank%1:06:00::","synset_id":"bank.n.09"},{"index":10,"sense_key":"bank%1:04:00::","synset_id":"bank.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bank%1:14:00::","call_ids":["semeval2013.d004.s001.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bank%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":965,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0020499999999999997,"input_uncached_usd":0.0019299999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.7046960419975221}
{"item_id":"semeval2013.d004.s001.t002","gold_sense_keys":["state%1:14:00::","state%1:14:01::"],"candidates":[{"index":1,"sense_key":"state%1:15:01::","synset_id":"state.n.01"},{"index":2,"sense_key":"state%1:03:00::","synset_id":"state.n.02"},{"index":3,"sense_key":"state%1:14:01::","synset_id":"state.n.03"},{"index":4,"sense_key":"state%1:14:00::","synset_id":"state.n.04"},{"index":5,"sense_key":"state%1:26:02::","synset_id":"state_of_matter.n.01"},{"index":6,"sense_key":"state%1:26:01::","synset_id":"state.n.06"},{"index":7,"sense_key":"state%1:15:00::","synset_id":"country.n.02"},{"index":8,"sense_key":"state%1:14:02::","synset_id":"department_of_state.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"state%1:14:01::","call_ids":["semeval2013.d004.s001.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"state%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":846,"cached_input_tokens":0,"output_tokens":44,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0021319999999999998,"input_uncached_usd":0.001692,"input_cached_usd":0.0,"output_usd":0.00044,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2843824999872595}
{"item_id":"semeval2013.d004.s001.t003","gold_sense_keys":["shackle%1:06:00::"],"candidates":[{"index":1,"sense_key":"shackle%1:06:00::","synset_id":"shackle.n.01"},{"index":2,"sense_key":"shackle%1:06:01::","synset_id":"shackle.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"shackle%1:06:00::","call_ids":["semeval2013.d004.s001.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"shackle%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":440,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001,"input_uncached_usd":0.0008799999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.1329335840418935}
{"item_id":"semeval2013.d004.s002.t001","gold_sense_keys":["support%1:21:01::"],"candidates":[{"index":1,"sense_key":"support%1:04:04::","synset_id":"support.n.01"},{"index":2,"sense_key":"support%1:04:00::","synset_id":"support.n.02"},{"index":3,"sense_key":"support%1:09:00::","synset_id":"support.n.03"},{"index":4,"sense_key":"support%1:04:03::","synset_id":"support.n.04"},{"index":5,"sense_key":"support%1:04:02::","synset_id":"documentation.n.03"},{"index":6,"sense_key":"support%1:21:00::","synset_id":"support.n.06"},{"index":7,"sense_key":"support%1:06:01::","synset_id":"support.n.07"},{"index":8,"sense_key":"support%1:04:01::","synset_id":"support.n.08"},{"index":9,"sense_key":"support%1:10:01::","synset_id":"accompaniment.n.02"},{"index":10,"sense_key":"support%1:06:00::","synset_id":"support.n.10"},{"index":11,"sense_key":"support%1:21:01::","synset_id":"support.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"support%1:21:01::","call_ids":["semeval2013.d004.s002.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"support%1:21:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1076,"cached_input_tokens":0,"output_tokens":106,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003212,"input_uncached_usd":0.002152,"input_cached_usd":0.0,"output_usd":0.0010600000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.220363207976334}
{"item_id":"semeval2013.d004.s002.t002","gold_sense_keys":["bank%1:14:00::"],"candidates":[{"index":1,"sense_key":"bank%1:17:01::","synset_id":"bank.n.01"},{"index":2,"sense_key":"bank%1:14:00::","synset_id":"depository_financial_institution.n.01"},{"index":3,"sense_key":"bank%1:17:00::","synset_id":"bank.n.03"},{"index":4,"sense_key":"bank%1:14:01::","synset_id":"bank.n.04"},{"index":5,"sense_key":"bank%1:21:00::","synset_id":"bank.n.05"},{"index":6,"sense_key":"bank%1:21:01::","synset_id":"bank.n.06"},{"index":7,"sense_key":"bank%1:17:02::","synset_id":"bank.n.07"},{"index":8,"sense_key":"bank%1:06:01::","synset_id":"savings_bank.n.02"},{"index":9,"sense_key":"bank%1:06:00::","synset_id":"bank.n.09"},{"index":10,"sense_key":"bank%1:04:00::","synset_id":"bank.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bank%1:14:00::","call_ids":["semeval2013.d004.s002.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bank%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":999,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0026379999999999997,"input_uncached_usd":0.0019979999999999998,"input_cached_usd":0.0,"output_usd":0.00064,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4730462500592694}
{"item_id":"semeval2013.d004.s002.t003","gold_sense_keys":["protection%1:21:00::"],"candidates":[{"index":1,"sense_key":"protection%1:04:00::","synset_id":"protection.n.01"},{"index":2,"sense_key":"protection%1:06:00::","synset_id":"protective_covering.n.01"},{"index":3,"sense_key":"protection%1:21:00::","synset_id":"security.n.02"},{"index":4,"sense_key":"protection%1:26:00::","synset_id":"protection.n.04"},{"index":5,"sense_key":"protection%1:04:02::","synset_id":"auspices.n.01"},{"index":6,"sense_key":"protection%1:04:03::","synset_id":"protection.n.06"},{"index":7,"sense_key":"protection%1:04:01::","synset_id":"protection.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"protection%1:21:00::","call_ids":["semeval2013.d004.s002.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"protection%1:21:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":833,"cached_input_tokens":0,"output_tokens":46,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002126,"input_uncached_usd":0.001666,"input_cached_usd":0.0,"output_usd":0.00046,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.649016167037189}
{"item_id":"semeval2013.d004.s002.t004","gold_sense_keys":["fund%1:21:00::"],"candidates":[{"index":1,"sense_key":"fund%1:21:00::","synset_id":"fund.n.01"},{"index":2,"sense_key":"fund%1:21:01::","synset_id":"store.n.02"},{"index":3,"sense_key":"fund%1:14:00::","synset_id":"investment_company.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"fund%1:21:00::","call_ids":["semeval2013.d004.s002.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"fund%1:21:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":524,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001168,"input_uncached_usd":0.0010479999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4499634170206264}
{"item_id":"semeval2013.d004.s002.t005","gold_sense_keys":["amount%1:21:00::"],"candidates":[{"index":1,"sense_key":"amount%1:21:00::","synset_id":"sum.n.01"},{"index":2,"sense_key":"amount%1:07:00::","synset_id":"amount.n.02"},{"index":3,"sense_key":"amount%1:03:00::","synset_id":"measure.n.02"},{"index":4,"sense_key":"amount%1:09:00::","synset_id":"sum.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"amount%1:21:00::","call_ids":["semeval2013.d004.s002.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"amount%1:21:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":578,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001276,"input_uncached_usd":0.001156,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4556429999647662}
{"item_id":"semeval2013.d004.s003.t000","gold_sense_keys":["report%1:10:00::"],"candidates":[{"index":1,"sense_key":"report%1:10:03::","synset_id":"report.n.01"},{"index":2,"sense_key":"report%1:10:01::","synset_id":"report.n.02"},{"index":3,"sense_key":"report%1:10:00::","synset_id":"report.n.03"},{"index":4,"sense_key":"report%1:11:00::","synset_id":"report.n.04"},{"index":5,"sense_key":"report%1:10:04::","synset_id":"report_card.n.01"},{"index":6,"sense_key":"report%1:10:02::","synset_id":"composition.n.08"},{"index":7,"sense_key":"report%1:09:00::","synset_id":"reputation.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"report%1:10:00::","call_ids":["semeval2013.d004.s003.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"report%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":860,"cached_input_tokens":0,"output_tokens":119,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0029100000000000003,"input_uncached_usd":0.00172,"input_cached_usd":0.0,"output_usd":0.00119,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9769566670292988}
{"item_id":"semeval2013.d004.s003.t002","gold_sense_keys":["broadcaster%1:18:00::"],"candidates":[{"index":1,"sense_key":"broadcaster%1:18:00::","synset_id":"broadcaster.n.01"},{"index":2,"sense_key":"broadcaster%1:06:00::","synset_id":"broadcaster.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"broadcaster%1:18:00::","call_ids":["semeval2013.d004.s003.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"broadcaster%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":521,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001162,"input_uncached_usd":0.001042,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.060141084017232}
{"item_id":"semeval2013.d004.s003.t003","gold_sense_keys":["state%1:14:01::"],"candidates":[{"index":1,"sense_key":"state%1:15:01::","synset_id":"state.n.01"},{"index":2,"sense_key":"state%1:03:00::","synset_id":"state.n.02"},{"index":3,"sense_key":"state%1:14:01::","synset_id":"state.n.03"},{"index":4,"sense_key":"state%1:14:00::","synset_id":"state.n.04"},{"index":5,"sense_key":"state%1:26:02::","synset_id":"state_of_matter.n.01"},{"index":6,"sense_key":"state%1:26:01::","synset_id":"state.n.06"},{"index":7,"sense_key":"state%1:15:00::","synset_id":"country.n.02"},{"index":8,"sense_key":"state%1:14:02::","synset_id":"department_of_state.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"state%1:14:01::","call_ids":["semeval2013.d004.s003.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"state%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":942,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002464,"input_uncached_usd":0.0018839999999999998,"input_cached_usd":0.0,"output_usd":0.00058,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.033785792067647}
{"item_id":"semeval2013.d004.s003.t004","gold_sense_keys":["support%1:21:01::"],"candidates":[{"index":1,"sense_key":"support%1:04:04::","synset_id":"support.n.01"},{"index":2,"sense_key":"support%1:04:00::","synset_id":"support.n.02"},{"index":3,"sense_key":"support%1:09:00::","synset_id":"support.n.03"},{"index":4,"sense_key":"support%1:04:03::","synset_id":"support.n.04"},{"index":5,"sense_key":"support%1:04:02::","synset_id":"documentation.n.03"},{"index":6,"sense_key":"support%1:21:00::","synset_id":"support.n.06"},{"index":7,"sense_key":"support%1:06:01::","synset_id":"support.n.07"},{"index":8,"sense_key":"support%1:04:01::","synset_id":"support.n.08"},{"index":9,"sense_key":"support%1:10:01::","synset_id":"accompaniment.n.02"},{"index":10,"sense_key":"support%1:06:00::","synset_id":"support.n.10"},{"index":11,"sense_key":"support%1:21:01::","synset_id":"support.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"support%1:21:01::","call_ids":["semeval2013.d004.s003.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"support%1:21:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1138,"cached_input_tokens":0,"output_tokens":98,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003256,"input_uncached_usd":0.002276,"input_cached_usd":0.0,"output_usd":0.0009800000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9616068329196423}
{"item_id":"semeval2013.d004.s004.t000","gold_sense_keys":["institution%1:14:00::"],"candidates":[{"index":1,"sense_key":"institution%1:14:00::","synset_id":"institution.n.01"},{"index":2,"sense_key":"institution%1:06:00::","synset_id":"institution.n.02"},{"index":3,"sense_key":"institution%1:09:00::","synset_id":"institution.n.03"},{"index":4,"sense_key":"institution%1:04:00::","synset_id":"initiation.n.02"},{"index":5,"sense_key":"institution%1:06:01::","synset_id":"mental_hospital.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"institution%1:14:00::","call_ids":["semeval2013.d004.s004.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"institution%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":773,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001666,"input_uncached_usd":0.0015459999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0970580420689657}
{"item_id":"semeval2013.d004.s004.t001","gold_sense_keys":["restriction%1:09:00::"],"candidates":[{"index":1,"sense_key":"restriction%1:09:00::","synset_id":"restriction.n.01"},{"index":2,"sense_key":"restriction%1:04:00::","synset_id":"limitation.n.05"},{"index":3,"sense_key":"restriction%1:04:01::","synset_id":"restriction.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"restriction%1:09:00::","call_ids":["semeval2013.d004.s004.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"restriction%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":624,"cached_input_tokens":0,"output_tokens":116,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002408,"input_uncached_usd":0.001248,"input_cached_usd":0.0,"output_usd":0.00116,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8311389159644023}
{"item_id":"semeval2013.d004.s004.t002","gold_sense_keys":["acceptance%1:04:03::"],"candidates":[{"index":1,"sense_key":"acceptance%1:09:00::","synset_id":"credence.n.01"},{"index":2,"sense_key":"acceptance%1:04:00::","synset_id":"adoption.n.01"},{"index":3,"sense_key":"acceptance%1:26:00::","synset_id":"acceptance.n.03"},{"index":4,"sense_key":"acceptance%1:10:01::","synset_id":"acceptance.n.04"},{"index":5,"sense_key":"acceptance%1:21:00::","synset_id":"acceptance.n.05"},{"index":6,"sense_key":"acceptance%1:07:00::","synset_id":"toleration.n.01"},{"index":7,"sense_key":"acceptance%1:04:03::","synset_id":"acceptance.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"acceptance%1:04:03::","call_ids":["semeval2013.d004.s004.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"acceptance%1:04:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":915,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00195,"input_uncached_usd":0.0018299999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2737389160320163}
{"item_id":"semeval2013.d004.s004.t003","gold_sense_keys":["money%1:21:00::"],"candidates":[{"index":1,"sense_key":"money%1:21:00::","synset_id":"money.n.01"},{"index":2,"sense_key":"money%1:21:02::","synset_id":"money.n.02"},{"index":3,"sense_key":"money%1:21:01::","synset_id":"money.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"money%1:21:00::","call_ids":["semeval2013.d004.s004.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"money%1:21:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":581,"cached_input_tokens":0,"output_tokens":107,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002232,"input_uncached_usd":0.001162,"input_cached_usd":0.0,"output_usd":0.00107,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9501074999570847}
{"item_id":"semeval2013.d004.s004.t004","gold_sense_keys":["limit%1:23:00::"],"candidates":[{"index":1,"sense_key":"limit%1:07:00::","synset_id":"limit.n.01"},{"index":2,"sense_key":"limit%1:28:00::","synset_id":"terminus_ad_quem.n.01"},{"index":3,"sense_key":"limit%1:15:02::","synset_id":"limit.n.03"},{"index":4,"sense_key":"limit%1:15:00::","synset_id":"limit.n.04"},{"index":5,"sense_key":"limit%1:23:01::","synset_id":"limit.n.05"},{"index":6,"sense_key":"limit%1:23:00::","synset_id":"limit.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"limit%1:23:00::","call_ids":["semeval2013.d004.s004.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"limit%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":780,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00168,"input_uncached_usd":0.00156,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.170765749993734}
{"item_id":"semeval2013.d004.s004.t005","gold_sense_keys":["manager%1:18:00::"],"candidates":[{"index":1,"sense_key":"manager%1:18:00::","synset_id":"director.n.01"},{"index":2,"sense_key":"manager%1:18:01::","synset_id":"coach.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"manager%1:18:00::","call_ids":["semeval2013.d004.s004.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"manager%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":534,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001188,"input_uncached_usd":0.001068,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0639380000066012}
{"item_id":"semeval2013.d004.s004.t006","gold_sense_keys":["bonus%1:21:00::"],"candidates":[{"index":1,"sense_key":"bonus%1:09:00::","synset_id":"bonus.n.01"},{"index":2,"sense_key":"bonus%1:21:00::","synset_id":"bonus.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bonus%1:21:00::","call_ids":["semeval2013.d004.s004.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bonus%1:21:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":551,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001222,"input_uncached_usd":0.001102,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9878822090104222}
{"item_id":"semeval2013.d004.s004.t007","gold_sense_keys":["charge%1:21:02::"],"candidates":[{"index":1,"sense_key":"charge%1:04:01::","synset_id":"charge.n.01"},{"index":2,"sense_key":"charge%1:10:00::","synset_id":"charge.n.02"},{"index":3,"sense_key":"charge%1:21:02::","synset_id":"charge.n.03"},{"index":4,"sense_key":"charge%1:19:00::","synset_id":"charge.n.04"},{"index":5,"sense_key":"charge%1:04:00::","synset_id":"care.n.05"},{"index":6,"sense_key":"charge%1:04:03::","synset_id":"mission.n.03"},{"index":7,"sense_key":"charge%1:18:00::","synset_id":"charge.n.07"},{"index":8,"sense_key":"charge%1:21:01::","synset_id":"charge.n.08"},{"index":9,"sense_key":"charge%1:16:00::","synset_id":"cathexis.n.01"},{"index":10,"sense_key":"charge%1:12:00::","synset_id":"bang.n.04"},{"index":11,"sense_key":"charge%1:10:03::","synset_id":"charge.n.11"},{"index":12,"sense_key":"charge%1:10:01::","synset_id":"commission.n.06"},{"index":13,"sense_key":"charge%1:10:04::","synset_id":"accusation.n.02"},{"index":14,"sense_key":"charge%1:06:01::","synset_id":"charge.n.14"},{"index":15,"sense_key":"charge%1:06:00::","synset_id":"charge.n.15"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"charge%1:21:02::","call_ids":["semeval2013.d004.s004.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"charge%1:21:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1423,"cached_input_tokens":0,"output_tokens":133,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004176,"input_uncached_usd":0.002846,"input_cached_usd":0.0,"output_usd":0.00133,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5822625000728294}
{"item_id":"semeval2013.d004.s005.t001","gold_sense_keys":["loan%1:21:00::"],"candidates":[{"index":1,"sense_key":"loan%1:21:00::","synset_id":"loan.n.01"},{"index":2,"sense_key":"loan%1:10:00::","synset_id":"loanword.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"loan%1:21:00::","call_ids":["semeval2013.d004.s005.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"loan%1:21:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":567,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0012540000000000001,"input_uncached_usd":0.001134,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5715009999694303}
{"item_id":"semeval2013.d004.s006.t000","gold_sense_keys":["rest%1:24:00::"],"candidates":[{"index":1,"sense_key":"rest%1:24:00::","synset_id":"remainder.n.01"},{"index":2,"sense_key":"rest%1:04:00::","synset_id":"rest.n.02"},{"index":3,"sense_key":"rest%1:28:00::","synset_id":"respite.n.04"},{"index":4,"sense_key":"rest%1:26:00::","synset_id":"rest.n.04"},{"index":5,"sense_key":"rest%1:26:01::","synset_id":"rest.n.05"},{"index":6,"sense_key":"rest%1:06:00::","synset_id":"rest.n.06"},{"index":7,"sense_key":"rest%1:10:00::","synset_id":"rest.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"rest%1:24:00::","call_ids":["semeval2013.d004.s006.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"rest%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":959,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002038,"input_uncached_usd":0.001918,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5632033339934424}
{"item_id":"semeval2013.d004.s006.t001","gold_sense_keys":["amount%1:21:00::"],"candidates":[{"index":1,"sense_key":"amount%1:21:00::","synset_id":"sum.n.01"},{"index":2,"sense_key":"amount%1:07:00::","synset_id":"amount.n.02"},{"index":3,"sense_key":"amount%1:03:00::","synset_id":"measure.n.02"},{"index":4,"sense_key":"amount%1:09:00::","synset_id":"sum.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"amount%1:21:00::","call_ids":["semeval2013.d004.s006.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"amount%1:21:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":741,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002102,"input_uncached_usd":0.001482,"input_cached_usd":0.0,"output_usd":0.00062,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7080809170147404}
{"item_id":"semeval2013.d004.s006.t003","gold_sense_keys":["sale%1:04:00::"],"candidates":[{"index":1,"sense_key":"sale%1:04:00::","synset_id":"sale.n.01"},{"index":2,"sense_key":"sale%1:04:02::","synset_id":"sale.n.02"},{"index":3,"sense_key":"sale%1:04:01::","synset_id":"sale.n.03"},{"index":4,"sense_key":"sale%1:26:00::","synset_id":"sale.n.04"},{"index":5,"sense_key":"sale%1:10:00::","synset_id":"sale.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"sale%1:04:02::","call_ids":["semeval2013.d004.s006.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"sale%1:04:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":906,"cached_input_tokens":0,"output_tokens":184,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003652,"input_uncached_usd":0.001812,"input_cached_usd":0.0,"output_usd":0.00184,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2099928749958053}
{"item_id":"semeval2013.d004.s007.t000","gold_sense_keys":["bank%1:14:00::"],"candidates":[{"index":1,"sense_key":"bank%1:17:01::","synset_id":"bank.n.01"},{"index":2,"sense_key":"bank%1:14:00::","synset_id":"depository_financial_institution.n.01"},{"index":3,"sense_key":"bank%1:17:00::","synset_id":"bank.n.03"},{"index":4,"sense_key":"bank%1:14:01::","synset_id":"bank.n.04"},{"index":5,"sense_key":"bank%1:21:00::","synset_id":"bank.n.05"},{"index":6,"sense_key":"bank%1:21:01::","synset_id":"bank.n.06"},{"index":7,"sense_key":"bank%1:17:02::","synset_id":"bank.n.07"},{"index":8,"sense_key":"bank%1:06:01::","synset_id":"savings_bank.n.02"},{"index":9,"sense_key":"bank%1:06:00::","synset_id":"bank.n.09"},{"index":10,"sense_key":"bank%1:04:00::","synset_id":"bank.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bank%1:14:00::","call_ids":["semeval2013.d004.s007.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bank%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1175,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00294,"input_uncached_usd":0.00235,"input_cached_usd":0.0,"output_usd":0.00059,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5202106249053031}
{"item_id":"semeval2013.d004.s008.t000","gold_sense_keys":["bank%1:14:00::"],"candidates":[{"index":1,"sense_key":"bank%1:17:01::","synset_id":"bank.n.01"},{"index":2,"sense_key":"bank%1:14:00::","synset_id":"depository_financial_institution.n.01"},{"index":3,"sense_key":"bank%1:17:00::","synset_id":"bank.n.03"},{"index":4,"sense_key":"bank%1:14:01::","synset_id":"bank.n.04"},{"index":5,"sense_key":"bank%1:21:00::","synset_id":"bank.n.05"},{"index":6,"sense_key":"bank%1:21:01::","synset_id":"bank.n.06"},{"index":7,"sense_key":"bank%1:17:02::","synset_id":"bank.n.07"},{"index":8,"sense_key":"bank%1:06:01::","synset_id":"savings_bank.n.02"},{"index":9,"sense_key":"bank%1:06:00::","synset_id":"bank.n.09"},{"index":10,"sense_key":"bank%1:04:00::","synset_id":"bank.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bank%1:14:00::","call_ids":["semeval2013.d004.s008.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bank%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1151,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0024219999999999997,"input_uncached_usd":0.002302,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.7255937920417637}
{"item_id":"semeval2013.d004.s008.t001","gold_sense_keys":["height%1:26:00::"],"candidates":[{"index":1,"sense_key":"height%1:07:00::","synset_id":"height.n.01"},{"index":2,"sense_key":"height%1:26:00::","synset_id":"acme.n.01"},{"index":3,"sense_key":"height%1:07:01::","synset_id":"stature.n.02"},{"index":4,"sense_key":"height%1:07:02::","synset_id":"altitude.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"height%1:26:00::","call_ids":["semeval2013.d004.s008.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"height%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":756,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001632,"input_uncached_usd":0.0015119999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.413165207952261}
{"item_id":"semeval2013.d004.s008.t002","gold_sense_keys":["crisis%1:26:00::"],"candidates":[{"index":1,"sense_key":"crisis%1:26:00::","synset_id":"crisis.n.01"},{"index":2,"sense_key":"crisis%1:11:00::","synset_id":"crisis.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"crisis%1:26:00::","call_ids":["semeval2013.d004.s008.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"crisis%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":619,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001358,"input_uncached_usd":0.001238,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9657243749825284}
{"item_id":"semeval2013.d004.s008.t003","gold_sense_keys":["investment%1:04:00::","investment%1:21:00::"],"candidates":[{"index":1,"sense_key":"investment%1:04:00::","synset_id":"investing.n.01"},{"index":2,"sense_key":"investment%1:21:00::","synset_id":"investment.n.02"},{"index":3,"sense_key":"investment%1:07:00::","synset_id":"investment.n.03"},{"index":4,"sense_key":"investment%1:05:00::","synset_id":"investment.n.04"},{"index":5,"sense_key":"investment%1:04:01::","synset_id":"investment.n.05"},{"index":6,"sense_key":"investment%1:04:02::","synset_id":"investment.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"investment%1:04:00::","call_ids":["semeval2013.d004.s008.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"investment%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":832,"cached_input_tokens":0,"output_tokens":146,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003124,"input_uncached_usd":0.001664,"input_cached_usd":0.0,"output_usd":0.0014600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8409727920079604}
{"item_id":"semeval2013.d004.s008.t004","gold_sense_keys":["bank%1:14:00::"],"candidates":[{"index":1,"sense_key":"bank%1:17:01::","synset_id":"bank.n.01"},{"index":2,"sense_key":"bank%1:14:00::","synset_id":"depository_financial_institution.n.01"},{"index":3,"sense_key":"bank%1:17:00::","synset_id":"bank.n.03"},{"index":4,"sense_key":"bank%1:14:01::","synset_id":"bank.n.04"},{"index":5,"sense_key":"bank%1:21:00::","synset_id":"bank.n.05"},{"index":6,"sense_key":"bank%1:21:01::","synset_id":"bank.n.06"},{"index":7,"sense_key":"bank%1:17:02::","synset_id":"bank.n.07"},{"index":8,"sense_key":"bank%1:06:01::","synset_id":"savings_bank.n.02"},{"index":9,"sense_key":"bank%1:06:00::","synset_id":"bank.n.09"},{"index":10,"sense_key":"bank%1:04:00::","synset_id":"bank.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bank%1:14:00::","call_ids":["semeval2013.d004.s008.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bank%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1151,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0024219999999999997,"input_uncached_usd":0.002302,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.993003583047539}
{"item_id":"semeval2013.d004.s009.t000","gold_sense_keys":["cash%1:21:00::"],"candidates":[{"index":1,"sense_key":"cash%1:21:00::","synset_id":"cash.n.01"},{"index":2,"sense_key":"cash%1:21:02::","synset_id":"cash.n.02"},{"index":3,"sense_key":"cash%1:18:00::","synset_id":"cash.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cash%1:21:00::","call_ids":["semeval2013.d004.s009.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cash%1:21:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":685,"cached_input_tokens":0,"output_tokens":111,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00248,"input_uncached_usd":0.00137,"input_cached_usd":0.0,"output_usd":0.00111,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.60981100006029}
{"item_id":"semeval2013.d004.s009.t001","gold_sense_keys":["liability%1:26:00::"],"candidates":[{"index":1,"sense_key":"liability%1:26:01::","synset_id":"liability.n.01"},{"index":2,"sense_key":"liability%1:26:00::","synset_id":"indebtedness.n.01"},{"index":3,"sense_key":"liability%1:07:00::","synset_id":"liability.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"liability%1:26:00::","call_ids":["semeval2013.d004.s009.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"liability%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":633,"cached_input_tokens":0,"output_tokens":70,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001966,"input_uncached_usd":0.001266,"input_cached_usd":0.0,"output_usd":0.0007000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9147675411077216}
{"item_id":"semeval2013.d004.s009.t002","gold_sense_keys":["state%1:14:01::"],"candidates":[{"index":1,"sense_key":"state%1:15:01::","synset_id":"state.n.01"},{"index":2,"sense_key":"state%1:03:00::","synset_id":"state.n.02"},{"index":3,"sense_key":"state%1:14:01::","synset_id":"state.n.03"},{"index":4,"sense_key":"state%1:14:00::","synset_id":"state.n.04"},{"index":5,"sense_key":"state%1:26:02::","synset_id":"state_of_matter.n.01"},{"index":6,"sense_key":"state%1:26:01::","synset_id":"state.n.06"},{"index":7,"sense_key":"state%1:15:00::","synset_id":"country.n.02"},{"index":8,"sense_key":"state%1:14:02::","synset_id":"department_of_state.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"state%1:14:01::","call_ids":["semeval2013.d004.s009.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"state%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1043,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002666,"input_uncached_usd":0.0020859999999999997,"input_cached_usd":0.0,"output_usd":0.00058,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6918639169307426}
{"item_id":"semeval2013.d004.s010.t002","gold_sense_keys":["bank%1:14:00::"],"candidates":[{"index":1,"sense_key":"bank%1:17:01::","synset_id":"bank.n.01"},{"index":2,"sense_key":"bank%1:14:00::","synset_id":"depository_financial_institution.n.01"},{"index":3,"sense_key":"bank%1:17:00::","synset_id":"bank.n.03"},{"index":4,"sense_key":"bank%1:14:01::","synset_id":"bank.n.04"},{"index":5,"sense_key":"bank%1:21:00::","synset_id":"bank.n.05"},{"index":6,"sense_key":"bank%1:21:01::","synset_id":"bank.n.06"},{"index":7,"sense_key":"bank%1:17:02::","synset_id":"bank.n.07"},{"index":8,"sense_key":"bank%1:06:01::","synset_id":"savings_bank.n.02"},{"index":9,"sense_key":"bank%1:06:00::","synset_id":"bank.n.09"},{"index":10,"sense_key":"bank%1:04:00::","synset_id":"bank.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bank%1:14:00::","call_ids":["semeval2013.d004.s010.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bank%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1116,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002352,"input_uncached_usd":0.002232,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9901623749174178}
{"item_id":"semeval2013.d004.s010.t003","gold_sense_keys":["capital%1:21:00::"],"candidates":[{"index":1,"sense_key":"capital%1:21:01::","synset_id":"capital.n.01"},{"index":2,"sense_key":"capital%1:21:00::","synset_id":"capital.n.02"},{"index":3,"sense_key":"capital%1:15:00::","synset_id":"capital.n.03"},{"index":4,"sense_key":"capital%1:10:00::","synset_id":"capital.n.04"},{"index":5,"sense_key":"capital%1:15:01::","synset_id":"capital.n.05"},{"index":6,"sense_key":"capital%1:14:00::","synset_id":"capital.n.06"},{"index":7,"sense_key":"capital%1:10:01::","synset_id":"das_kapital.n.01"},{"index":8,"sense_key":"capital%1:06:00::","synset_id":"capital.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"capital%1:21:00::","call_ids":["semeval2013.d004.s010.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"capital%1:21:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":926,"cached_input_tokens":0,"output_tokens":114,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002992,"input_uncached_usd":0.001852,"input_cached_usd":0.0,"output_usd":0.0011400000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1774770419579}
{"item_id":"semeval2013.d004.s010.t004","gold_sense_keys":["increase%1:07:00::","increase%1:11:00::"],"candidates":[{"index":1,"sense_key":"increase%1:23:00::","synset_id":"addition.n.03"},{"index":2,"sense_key":"increase%1:11:00::","synset_id":"increase.n.02"},{"index":3,"sense_key":"increase%1:22:00::","synset_id":"increase.n.03"},{"index":4,"sense_key":"increase%1:07:00::","synset_id":"increase.n.04"},{"index":5,"sense_key":"increase%1:04:00::","synset_id":"increase.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"increase%1:04:00::","call_ids":["semeval2013.d004.s010.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"increase%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":772,"cached_input_tokens":0,"output_tokens":134,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0028840000000000003,"input_uncached_usd":0.001544,"input_cached_usd":0.0,"output_usd":0.00134,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.331993833067827}
{"item_id":"semeval2013.d004.s010.t006","gold_sense_keys":["circle%1:14:00::"],"candidates":[{"index":1,"sense_key":"circle%1:25:00::","synset_id":"circle.n.01"},{"index":2,"sense_key":"circle%1:14:00::","synset_id":"set.n.05"},{"index":3,"sense_key":"circle%1:25:01::","synset_id":"circle.n.03"},{"index":4,"sense_key":"circle%1:04:00::","synset_id":"lap.n.05"},{"index":5,"sense_key":"circle%1:06:01::","synset_id":"traffic_circle.n.01"},{"index":6,"sense_key":"circle%1:06:03::","synset_id":"r-2.n.01"},{"index":7,"sense_key":"circle%1:06:02::","synset_id":"circle.n.07"},{"index":8,"sense_key":"circle%1:06:00::","synset_id":"circle.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"circle%1:14:00::","call_ids":["semeval2013.d004.s010.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"circle%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":993,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002106,"input_uncached_usd":0.001986,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4669816660461947}
{"item_id":"semeval2013.d004.s012.t000","gold_sense_keys":["support%1:21:01::"],"candidates":[{"index":1,"sense_key":"support%1:04:04::","synset_id":"support.n.01"},{"index":2,"sense_key":"support%1:04:00::","synset_id":"support.n.02"},{"index":3,"sense_key":"support%1:09:00::","synset_id":"support.n.03"},{"index":4,"sense_key":"support%1:04:03::","synset_id":"support.n.04"},{"index":5,"sense_key":"support%1:04:02::","synset_id":"documentation.n.03"},{"index":6,"sense_key":"support%1:21:00::","synset_id":"support.n.06"},{"index":7,"sense_key":"support%1:06:01::","synset_id":"support.n.07"},{"index":8,"sense_key":"support%1:04:01::","synset_id":"support.n.08"},{"index":9,"sense_key":"support%1:10:01::","synset_id":"accompaniment.n.02"},{"index":10,"sense_key":"support%1:06:00::","synset_id":"support.n.10"},{"index":11,"sense_key":"support%1:21:01::","synset_id":"support.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"support%1:21:01::","call_ids":["semeval2013.d004.s012.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"support%1:21:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1174,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0029879999999999998,"input_uncached_usd":0.002348,"input_cached_usd":0.0,"output_usd":0.00064,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4721485000336543}
{"item_id":"semeval2013.d005.s000.t000","gold_sense_keys":["appeal%1:04:00::"],"candidates":[{"index":1,"sense_key":"appeal%1:10:00::","synset_id":"entreaty.n.01"},{"index":2,"sense_key":"appeal%1:07:00::","synset_id":"appeal.n.02"},{"index":3,"sense_key":"appeal%1:04:00::","synset_id":"appeal.n.03"},{"index":4,"sense_key":"appeal%1:10:02::","synset_id":"solicitation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"appeal%1:04:00::","call_ids":["semeval2013.d005.s000.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"appeal%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":579,"cached_input_tokens":0,"output_tokens":124,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002398,"input_uncached_usd":0.001158,"input_cached_usd":0.0,"output_usd":0.00124,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9221339999930933}
{"item_id":"semeval2013.d005.s001.t002","gold_sense_keys":["row%1:14:00::"],"candidates":[{"index":1,"sense_key":"row%1:14:00::","synset_id":"row.n.01"},{"index":2,"sense_key":"row%1:10:00::","synset_id":"quarrel.n.01"},{"index":3,"sense_key":"row%1:17:00::","synset_id":"row.n.03"},{"index":4,"sense_key":"row%1:06:00::","synset_id":"course.n.08"},{"index":5,"sense_key":"row%1:14:01::","synset_id":"row.n.05"},{"index":6,"sense_key":"row%1:07:00::","synset_id":"row.n.06"},{"index":7,"sense_key":"row%1:04:00::","synset_id":"rowing.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"row%1:14:00::","call_ids":["semeval2013.d005.s001.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"row%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":660,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0018800000000000002,"input_uncached_usd":0.00132,"input_cached_usd":0.0,"output_usd":0.0005600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5088179169688374}
{"item_id":"semeval2013.d005.s002.t000","gold_sense_keys":["report%1:10:00::"],"candidates":[{"index":1,"sense_key":"report%1:10:03::","synset_id":"report.n.01"},{"index":2,"sense_key":"report%1:10:01::","synset_id":"report.n.02"},{"index":3,"sense_key":"report%1:10:00::","synset_id":"report.n.03"},{"index":4,"sense_key":"report%1:11:00::","synset_id":"report.n.04"},{"index":5,"sense_key":"report%1:10:04::","synset_id":"report_card.n.01"},{"index":6,"sense_key":"report%1:10:02::","synset_id":"composition.n.08"},{"index":7,"sense_key":"report%1:09:00::","synset_id":"reputation.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"report%1:10:00::","call_ids":["semeval2013.d005.s002.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"report%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":799,"cached_input_tokens":0,"output_tokens":75,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0023480000000000003,"input_uncached_usd":0.001598,"input_cached_usd":0.0,"output_usd":0.00075,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9777068749535829}
{"item_id":"semeval2013.d005.s002.t001","gold_sense_keys":["testimony%1:10:00::"],"candidates":[{"index":1,"sense_key":"testimony%1:10:00::","synset_id":"testimony.n.01"},{"index":2,"sense_key":"testimony%1:10:01::","synset_id":"testimony.n.02"},{"index":3,"sense_key":"testimony%1:10:02::","synset_id":"testimony.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"testimony%1:10:00::","call_ids":["semeval2013.d005.s002.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"testimony%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":499,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0016380000000000001,"input_uncached_usd":0.000998,"input_cached_usd":0.0,"output_usd":0.00064,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5797950419364497}
{"item_id":"semeval2013.d005.s003.t000","gold_sense_keys":["in_the_end%4:02:00::"],"candidates":[{"index":1,"sense_key":"in_the_end%4:02:02::","synset_id":"in_the_long_run.r.01"},{"index":2,"sense_key":"in_the_end%4:02:00::","synset_id":"ultimately.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"in_the_end%4:02:00::","call_ids":["semeval2013.d005.s003.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"in_the_end%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":568,"cached_input_tokens":0,"output_tokens":100,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002136,"input_uncached_usd":0.0011359999999999999,"input_cached_usd":0.0,"output_usd":0.001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9131157499505207}
{"item_id":"semeval2013.d005.s003.t001","gold_sense_keys":["defense%1:14:00::"],"candidates":[{"index":1,"sense_key":"defense%1:04:00::","synset_id":"defense.n.01"},{"index":2,"sense_key":"defense%1:04:03::","synset_id":"defense.n.02"},{"index":3,"sense_key":"defense%1:14:01::","synset_id":"defense.n.03"},{"index":4,"sense_key":"defense%1:10:00::","synset_id":"defense.n.04"},{"index":5,"sense_key":"defense%1:22:00::","synset_id":"defense_mechanism.n.01"},{"index":6,"sense_key":"defense%1:14:02::","synset_id":"department_of_defense.n.01"},{"index":7,"sense_key":"defense%1:14:00::","synset_id":"defense.n.07"},{"index":8,"sense_key":"defense%1:10:01::","synset_id":"refutation.n.01"},{"index":9,"sense_key":"defense%1:14:03::","synset_id":"defense.n.09"},{"index":10,"sense_key":"defense%1:06:00::","synset_id":"defensive_structure.n.01"},{"index":11,"sense_key":"defense%1:04:02::","synset_id":"defense.n.11"},{"index":12,"sense_key":"defense%1:04:01::","synset_id":"defense.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"defense%1:14:00::","call_ids":["semeval2013.d005.s003.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"defense%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1286,"cached_input_tokens":0,"output_tokens":39,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0029620000000000002,"input_uncached_usd":0.002572,"input_cached_usd":0.0,"output_usd":0.00039000000000000005,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1674165419535711}
{"item_id":"semeval2013.d005.s003.t002","gold_sense_keys":["message%1:10:01::"],"candidates":[{"index":1,"sense_key":"message%1:10:01::","synset_id":"message.n.01"},{"index":2,"sense_key":"message%1:10:00::","synset_id":"message.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"message%1:10:01::","call_ids":["semeval2013.d005.s003.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"message%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":527,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0011740000000000001,"input_uncached_usd":0.001054,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.281961957924068}
{"item_id":"semeval2013.d005.s003.t003","gold_sense_keys":["trial%1:04:02::"],"candidates":[{"index":1,"sense_key":"trial%1:04:00::","synset_id":"test.n.05"},{"index":2,"sense_key":"trial%1:09:00::","synset_id":"trial.n.02"},{"index":3,"sense_key":"trial%1:04:03::","synset_id":"test.n.04"},{"index":4,"sense_key":"trial%1:04:02::","synset_id":"trial.n.04"},{"index":5,"sense_key":"trial%1:11:01::","synset_id":"trial.n.05"},{"index":6,"sense_key":"trial%1:11:00::","synset_id":"trial.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"trial%1:04:02::","call_ids":["semeval2013.d005.s003.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"trial%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":865,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00185,"input_uncached_usd":0.00173,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3283379580825567}
{"item_id":"semeval2013.d005.s003.t004","gold_sense_keys":["rape%1:04:00::"],"candidates":[{"index":1,"sense_key":"rape%1:20:00::","synset_id":"rape.n.01"},{"index":2,"sense_key":"rape%1:04:01::","synset_id":"rape.n.02"},{"index":3,"sense_key":"rape%1:04:00::","synset_id":"rape.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"rape%1:04:00::","call_ids":["semeval2013.d005.s003.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"rape%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":594,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0013080000000000001,"input_uncached_usd":0.001188,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.7971510000061244}
{"item_id":"semeval2013.d005.s003.t005","gold_sense_keys":["child%1:18:00::"],"candidates":[{"index":1,"sense_key":"child%1:18:00::","synset_id":"child.n.01"},{"index":2,"sense_key":"child%1:18:01::","synset_id":"child.n.02"},{"index":3,"sense_key":"child%1:18:06::","synset_id":"child.n.03"},{"index":4,"sense_key":"child%1:18:07::","synset_id":"child.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"child%1:18:00::","call_ids":["semeval2013.d005.s003.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"child%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":665,"cached_input_tokens":0,"output_tokens":123,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00256,"input_uncached_usd":0.00133,"input_cached_usd":0.0,"output_usd":0.0012300000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6649384999182075}
{"item_id":"semeval2013.d005.s003.t006","gold_sense_keys":["bed%1:06:00::"],"candidates":[{"index":1,"sense_key":"bed%1:06:00::","synset_id":"bed.n.01"},{"index":2,"sense_key":"bed%1:06:01::","synset_id":"bed.n.02"},{"index":3,"sense_key":"bed%1:17:00::","synset_id":"bed.n.03"},{"index":4,"sense_key":"bed%1:17:01::","synset_id":"bed.n.04"},{"index":5,"sense_key":"bed%1:15:00::","synset_id":"seam.n.03"},{"index":6,"sense_key":"bed%1:06:04::","synset_id":"layer.n.01"},{"index":7,"sense_key":"bed%1:06:03::","synset_id":"bed.n.07"},{"index":8,"sense_key":"bed%1:06:02::","synset_id":"bed.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bed%1:06:00::","call_ids":["semeval2013.d005.s003.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bed%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":912,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001944,"input_uncached_usd":0.0018239999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4089403749676421}
{"item_id":"semeval2013.d005.s003.t007","gold_sense_keys":["house%1:06:00::"],"candidates":[{"index":1,"sense_key":"house%1:06:00::","synset_id":"house.n.01"},{"index":2,"sense_key":"house%1:14:01::","synset_id":"firm.n.01"},{"index":3,"sense_key":"house%1:14:04::","synset_id":"house.n.03"},{"index":4,"sense_key":"house%1:14:03::","synset_id":"house.n.04"},{"index":5,"sense_key":"house%1:14:02::","synset_id":"house.n.05"},{"index":6,"sense_key":"house%1:14:00::","synset_id":"house.n.06"},{"index":7,"sense_key":"house%1:04:00::","synset_id":"house.n.07"},{"index":8,"sense_key":"house%1:15:00::","synset_id":"sign_of_the_zodiac.n.01"},{"index":9,"sense_key":"house%1:14:06::","synset_id":"house.n.09"},{"index":10,"sense_key":"house%1:14:05::","synset_id":"family.n.01"},{"index":11,"sense_key":"house%1:06:02::","synset_id":"theater.n.01"},{"index":12,"sense_key":"house%1:06:01::","synset_id":"house.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"house%1:06:00::","call_ids":["semeval2013.d005.s003.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"house%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1075,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00227,"input_uncached_usd":0.00215,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5628249159781262}
{"item_id":"semeval2013.d005.s004.t001","gold_sense_keys":["trial%1:04:02::"],"candidates":[{"index":1,"sense_key":"trial%1:04:00::","synset_id":"test.n.05"},{"index":2,"sense_key":"trial%1:09:00::","synset_id":"trial.n.02"},{"index":3,"sense_key":"trial%1:04:03::","synset_id":"test.n.04"},{"index":4,"sense_key":"trial%1:04:02::","synset_id":"trial.n.04"},{"index":5,"sense_key":"trial%1:11:01::","synset_id":"trial.n.05"},{"index":6,"sense_key":"trial%1:11:00::","synset_id":"trial.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"trial%1:04:02::","call_ids":["semeval2013.d005.s004.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"trial%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":910,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00194,"input_uncached_usd":0.00182,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.121358875068836}
{"item_id":"semeval2013.d005.s004.t002","gold_sense_keys":["member%1:18:00::"],"candidates":[{"index":1,"sense_key":"member%1:18:00::","synset_id":"member.n.01"},{"index":2,"sense_key":"member%1:24:00::","synset_id":"member.n.02"},{"index":3,"sense_key":"member%1:08:00::","synset_id":"extremity.n.01"},{"index":4,"sense_key":"member%1:14:00::","synset_id":"member.n.04"},{"index":5,"sense_key":"member%1:08:01::","synset_id":"penis.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"member%1:18:00::","call_ids":["semeval2013.d005.s004.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"member%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":802,"cached_input_tokens":0,"output_tokens":132,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0029240000000000004,"input_uncached_usd":0.001604,"input_cached_usd":0.0,"output_usd":0.0013200000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.5043169589480385}
{"item_id":"semeval2013.d005.s004.t003","gold_sense_keys":["family%1:14:00::","family%1:14:02::"],"candidates":[{"index":1,"sense_key":"family%1:14:02::","synset_id":"family.n.01"},{"index":2,"sense_key":"family%1:14:00::","synset_id":"family.n.02"},{"index":3,"sense_key":"family%1:14:04::","synset_id":"class.n.01"},{"index":4,"sense_key":"family%1:14:01::","synset_id":"family.n.04"},{"index":5,"sense_key":"family%1:18:00::","synset_id":"kin.n.01"},{"index":6,"sense_key":"family%1:14:03::","synset_id":"family.n.06"},{"index":7,"sense_key":"family%1:14:06::","synset_id":"syndicate.n.01"},{"index":8,"sense_key":"family%1:14:05::","synset_id":"family.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"family%1:14:00::","call_ids":["semeval2013.d005.s004.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"family%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":991,"cached_input_tokens":0,"output_tokens":126,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003242,"input_uncached_usd":0.001982,"input_cached_usd":0.0,"output_usd":0.00126,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8550770409638062}
{"item_id":"semeval2013.d005.s004.t004","gold_sense_keys":["end%1:11:00::","end%1:28:00::"],"candidates":[{"index":1,"sense_key":"end%1:15:00::","synset_id":"end.n.01"},{"index":2,"sense_key":"end%1:28:00::","synset_id":"end.n.02"},{"index":3,"sense_key":"end%1:11:00::","synset_id":"end.n.03"},{"index":4,"sense_key":"end%1:09:02::","synset_id":"goal.n.01"},{"index":5,"sense_key":"end%1:09:00::","synset_id":"end.n.05"},{"index":6,"sense_key":"end%1:26:00::","synset_id":"end.n.06"},{"index":7,"sense_key":"end%1:15:02::","synset_id":"end.n.07"},{"index":8,"sense_key":"end%1:18:00::","synset_id":"end.n.08"},{"index":9,"sense_key":"end%1:15:01::","synset_id":"end.n.09"},{"index":10,"sense_key":"end%1:15:03::","synset_id":"end.n.10"},{"index":11,"sense_key":"end%1:04:00::","synset_id":"end.n.11"},{"index":12,"sense_key":"end%1:10:00::","synset_id":"conclusion.n.08"},{"index":13,"sense_key":"end%1:06:02::","synset_id":"end.n.13"},{"index":14,"sense_key":"end%1:04:02::","synset_id":"end.n.14"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"end%1:28:00::","call_ids":["semeval2013.d005.s004.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"end%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1281,"cached_input_tokens":0,"output_tokens":262,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.005182000000000001,"input_uncached_usd":0.002562,"input_cached_usd":0.0,"output_usd":0.0026200000000000004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.325811624992639}
{"item_id":"semeval2013.d005.s004.t005","gold_sense_keys":["judge%1:18:00::"],"candidates":[{"index":1,"sense_key":"judge%1:18:00::","synset_id":"judge.n.01"},{"index":2,"sense_key":"judge%1:18:01::","synset_id":"evaluator.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"judge%1:18:00::","call_ids":["semeval2013.d005.s004.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"judge%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":575,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00127,"input_uncached_usd":0.00115,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2536118749994785}
{"item_id":"semeval2013.d005.s004.t006","gold_sense_keys":["imposition%1:04:02::"],"candidates":[{"index":1,"sense_key":"imposition%1:04:02::","synset_id":"imposition.n.01"},{"index":2,"sense_key":"imposition%1:09:00::","synset_id":"imposition.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"imposition%1:04:02::","call_ids":["semeval2013.d005.s004.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"imposition%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":572,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0012640000000000001,"input_uncached_usd":0.001144,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.265349542023614}
{"item_id":"semeval2013.d005.s004.t007","gold_sense_keys":["jury%1:14:00::"],"candidates":[{"index":1,"sense_key":"jury%1:14:00::","synset_id":"jury.n.01"},{"index":2,"sense_key":"jury%1:14:01::","synset_id":"jury.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"jury%1:14:00::","call_ids":["semeval2013.d005.s004.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"jury%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":567,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0012540000000000001,"input_uncached_usd":0.001134,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0702836660202593}
{"item_id":"semeval2013.d005.s005.t002","gold_sense_keys":["filing%1:10:00::"],"candidates":[{"index":1,"sense_key":"filing%1:10:00::","synset_id":"filing.n.01"},{"index":2,"sense_key":"filing%1:17:00::","synset_id":"filing.n.02"},{"index":3,"sense_key":"filing%1:04:01::","synset_id":"filing.n.03"},{"index":4,"sense_key":"filing%1:04:00::","synset_id":"filing.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"filing%1:10:00::","call_ids":["semeval2013.d005.s005.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"filing%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":744,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001608,"input_uncached_usd":0.001488,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.45268358301837}
{"item_id":"semeval2013.d005.s005.t003","gold_sense_keys":["appeal%1:04:00::"],"candidates":[{"index":1,"sense_key":"appeal%1:10:00::","synset_id":"entreaty.n.01"},{"index":2,"sense_key":"appeal%1:07:00::","synset_id":"appeal.n.02"},{"index":3,"sense_key":"appeal%1:04:00::","synset_id":"appeal.n.03"},{"index":4,"sense_key":"appeal%1:10:02::","synset_id":"solicitation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"appeal%1:04:00::","call_ids":["semeval2013.d005.s005.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"appeal%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":866,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001852,"input_uncached_usd":0.0017319999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5220271250000224}
{"item_id":"semeval2013.d005.s005.t004","gold_sense_keys":["argument%1:10:02::"],"candidates":[{"index":1,"sense_key":"argument%1:10:02::","synset_id":"argument.n.01"},{"index":2,"sense_key":"argument%1:10:03::","synset_id":"controversy.n.01"},{"index":3,"sense_key":"argument%1:10:00::","synset_id":"argument.n.03"},{"index":4,"sense_key":"argument%1:10:01::","synset_id":"argument.n.04"},{"index":5,"sense_key":"argument%1:10:04::","synset_id":"argument.n.05"},{"index":6,"sense_key":"argument%1:09:00::","synset_id":"argument.n.06"},{"index":7,"sense_key":"argument%1:09:01::","synset_id":"argumentation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"argument%1:10:02::","call_ids":["semeval2013.d005.s005.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"argument%1:10:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1080,"cached_input_tokens":0,"output_tokens":246,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004620000000000001,"input_uncached_usd":0.00216,"input_cached_usd":0.0,"output_usd":0.0024600000000000004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.565394458011724}
{"item_id":"semeval2013.d005.s005.t005","gold_sense_keys":["court%1:14:00::"],"candidates":[{"index":1,"sense_key":"court%1:14:00::","synset_id":"court.n.01"},{"index":2,"sense_key":"court%1:06:02::","synset_id":"court.n.02"},{"index":3,"sense_key":"court%1:14:01::","synset_id":"court.n.03"},{"index":4,"sense_key":"court%1:06:01::","synset_id":"court.n.04"},{"index":5,"sense_key":"court%1:18:00::","synset_id":"court.n.05"},{"index":6,"sense_key":"court%1:14:02::","synset_id":"court.n.06"},{"index":7,"sense_key":"court%1:06:03::","synset_id":"motor_hotel.n.01"},{"index":8,"sense_key":"court%1:06:05::","synset_id":"court.n.08"},{"index":9,"sense_key":"court%1:06:04::","synset_id":"court.n.09"},{"index":10,"sense_key":"court%1:06:00::","synset_id":"court.n.10"},{"index":11,"sense_key":"court%1:04:00::","synset_id":"court.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"court%1:06:05::","call_ids":["semeval2013.d005.s005.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"court%1:06:05::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1176,"cached_input_tokens":0,"output_tokens":232,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004672,"input_uncached_usd":0.002352,"input_cached_usd":0.0,"output_usd":0.00232,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.357045333017595}
{"item_id":"semeval2013.d005.s005.t006","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["semeval2013.d005.s005.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":816,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001752,"input_uncached_usd":0.001632,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9740926249651238}
{"item_id":"semeval2013.d005.s006.t001","gold_sense_keys":["trial%1:04:02::"],"candidates":[{"index":1,"sense_key":"trial%1:04:00::","synset_id":"test.n.05"},{"index":2,"sense_key":"trial%1:09:00::","synset_id":"trial.n.02"},{"index":3,"sense_key":"trial%1:04:03::","synset_id":"test.n.04"},{"index":4,"sense_key":"trial%1:04:02::","synset_id":"trial.n.04"},{"index":5,"sense_key":"trial%1:11:01::","synset_id":"trial.n.05"},{"index":6,"sense_key":"trial%1:11:00::","synset_id":"trial.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"trial%1:04:02::","call_ids":["semeval2013.d005.s006.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"trial%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1009,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0021379999999999997,"input_uncached_usd":0.002018,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.971589665976353}
{"item_id":"semeval2013.d005.s006.t003","gold_sense_keys":["use%1:04:00::"],"candidates":[{"index":1,"sense_key":"use%1:04:00::","synset_id":"use.n.01"},{"index":2,"sense_key":"use%1:07:00::","synset_id":"function.n.02"},{"index":3,"sense_key":"use%1:07:02::","synset_id":"use.n.03"},{"index":4,"sense_key":"use%1:22:00::","synset_id":"consumption.n.03"},{"index":5,"sense_key":"use%1:04:01::","synset_id":"habit.n.02"},{"index":6,"sense_key":"use%1:04:02::","synset_id":"manipulation.n.01"},{"index":7,"sense_key":"use%1:07:03::","synset_id":"use.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%1:04:00::","call_ids":["semeval2013.d005.s006.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1099,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0023179999999999997,"input_uncached_usd":0.002198,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.7772878329269588}
{"item_id":"semeval2013.d005.s006.t004","gold_sense_keys":["circus%1:04:01::"],"candidates":[{"index":1,"sense_key":"circus%1:14:00::","synset_id":"circus.n.01"},{"index":2,"sense_key":"circus%1:04:00::","synset_id":"circus.n.02"},{"index":3,"sense_key":"circus%1:04:01::","synset_id":"circus.n.03"},{"index":4,"sense_key":"circus%1:06:01::","synset_id":"circus.n.04"},{"index":5,"sense_key":"circus%1:06:00::","synset_id":"circus.n.05"},{"index":6,"sense_key":"circus%1:05:00::","synset_id":"circus.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"circus%1:04:01::","call_ids":["semeval2013.d005.s006.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"circus%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":947,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0020139999999999997,"input_uncached_usd":0.0018939999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8403273749863729}
{"item_id":"semeval2013.d005.s006.t005","gold_sense_keys":["atmosphere%1:26:01::"],"candidates":[{"index":1,"sense_key":"atmosphere%1:26:01::","synset_id":"atmosphere.n.01"},{"index":2,"sense_key":"atmosphere%1:23:00::","synset_id":"standard_atmosphere.n.01"},{"index":3,"sense_key":"atmosphere%1:15:00::","synset_id":"atmosphere.n.03"},{"index":4,"sense_key":"atmosphere%1:26:00::","synset_id":"atmosphere.n.04"},{"index":5,"sense_key":"atmosphere%1:17:00::","synset_id":"atmosphere.n.05"},{"index":6,"sense_key":"atmosphere%1:07:00::","synset_id":"air.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"atmosphere%1:26:01::","call_ids":["semeval2013.d005.s006.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"atmosphere%1:26:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":987,"cached_input_tokens":0,"output_tokens":117,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003144,"input_uncached_usd":0.001974,"input_cached_usd":0.0,"output_usd":0.00117,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8834057910135016}
{"item_id":"semeval2013.d005.s006.t006","gold_sense_keys":["detail%1:09:00::"],"candidates":[{"index":1,"sense_key":"detail%1:09:00::","synset_id":"detail.n.01"},{"index":2,"sense_key":"detail%1:24:00::","synset_id":"detail.n.02"},{"index":3,"sense_key":"detail%1:10:00::","synset_id":"detail.n.03"},{"index":4,"sense_key":"detail%1:14:01::","synset_id":"detail.n.04"},{"index":5,"sense_key":"detail%1:14:00::","synset_id":"contingent.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"detail%1:09:00::","call_ids":["semeval2013.d005.s006.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"detail%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":848,"cached_input_tokens":0,"output_tokens":117,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002866,"input_uncached_usd":0.001696,"input_cached_usd":0.0,"output_usd":0.00117,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.947911584051326}
{"item_id":"semeval2013.d005.s006.t007","gold_sense_keys":["jury%1:14:00::"],"candidates":[{"index":1,"sense_key":"jury%1:14:00::","synset_id":"jury.n.01"},{"index":2,"sense_key":"jury%1:14:01::","synset_id":"jury.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"jury%1:14:00::","call_ids":["semeval2013.d005.s006.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"jury%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":666,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001452,"input_uncached_usd":0.0013319999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.124030499951914}
{"item_id":"semeval2013.d005.s006.t008","gold_sense_keys":["passion%1:12:00::"],"candidates":[{"index":1,"sense_key":"passion%1:12:00::","synset_id":"passion.n.01"},{"index":2,"sense_key":"passion%1:07:00::","synset_id":"heat.n.04"},{"index":3,"sense_key":"passion%1:26:01::","synset_id":"rage.n.03"},{"index":4,"sense_key":"passion%1:16:00::","synset_id":"mania.n.01"},{"index":5,"sense_key":"passion%1:12:02::","synset_id":"passion.n.05"},{"index":6,"sense_key":"passion%1:09:00::","synset_id":"love.n.02"},{"index":7,"sense_key":"passion%1:26:02::","synset_id":"passion.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"passion%1:12:00::","call_ids":["semeval2013.d005.s006.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"passion%1:12:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":915,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00195,"input_uncached_usd":0.0018299999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.669385917019099}
{"item_id":"semeval2013.d005.s007.t000","gold_sense_keys":["claim%1:10:02::"],"candidates":[{"index":1,"sense_key":"claim%1:10:00::","synset_id":"claim.n.01"},{"index":2,"sense_key":"claim%1:10:02::","synset_id":"claim.n.02"},{"index":3,"sense_key":"claim%1:04:00::","synset_id":"claim.n.03"},{"index":4,"sense_key":"claim%1:07:02::","synset_id":"claim.n.04"},{"index":5,"sense_key":"claim%1:07:01::","synset_id":"title.n.07"},{"index":6,"sense_key":"claim%1:10:01::","synset_id":"call.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"claim%1:10:02::","call_ids":["semeval2013.d005.s007.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"claim%1:10:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":962,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002654,"input_uncached_usd":0.001924,"input_cached_usd":0.0,"output_usd":0.0007300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3637982909567654}
{"item_id":"semeval2013.d005.s007.t001","gold_sense_keys":["appeal%1:04:00::"],"candidates":[{"index":1,"sense_key":"appeal%1:10:00::","synset_id":"entreaty.n.01"},{"index":2,"sense_key":"appeal%1:07:00::","synset_id":"appeal.n.02"},{"index":3,"sense_key":"appeal%1:04:00::","synset_id":"appeal.n.03"},{"index":4,"sense_key":"appeal%1:10:02::","synset_id":"solicitation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"appeal%1:04:00::","call_ids":["semeval2013.d005.s007.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"appeal%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":943,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0025659999999999997,"input_uncached_usd":0.0018859999999999999,"input_cached_usd":0.0,"output_usd":0.00068,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8230529580032453}
{"item_id":"semeval2013.d005.s007.t002","gold_sense_keys":["judge%1:18:00::"],"candidates":[{"index":1,"sense_key":"judge%1:18:00::","synset_id":"judge.n.01"},{"index":2,"sense_key":"judge%1:18:01::","synset_id":"evaluator.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"judge%1:18:00::","call_ids":["semeval2013.d005.s007.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"judge%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":735,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00159,"input_uncached_usd":0.00147,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4831902920268476}
{"item_id":"semeval2013.d005.s007.t003","gold_sense_keys":["question%1:10:01::"],"candidates":[{"index":1,"sense_key":"question%1:10:00::","synset_id":"question.n.01"},{"index":2,"sense_key":"question%1:10:01::","synset_id":"question.n.02"},{"index":3,"sense_key":"question%1:10:03::","synset_id":"question.n.03"},{"index":4,"sense_key":"question%1:07:00::","synset_id":"doubt.n.02"},{"index":5,"sense_key":"question%1:10:02::","synset_id":"motion.n.05"},{"index":6,"sense_key":"question%1:10:04::","synset_id":"question.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"question%1:10:01::","call_ids":["semeval2013.d005.s007.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"question%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1068,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0022559999999999998,"input_uncached_usd":0.002136,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2961269160732627}
{"item_id":"semeval2013.d005.s007.t005","gold_sense_keys":["rule%1:10:00::"],"candidates":[{"index":1,"sense_key":"rule%1:09:00::","synset_id":"rule.n.01"},{"index":2,"sense_key":"rule%1:09:01::","synset_id":"convention.n.02"},{"index":3,"sense_key":"rule%1:10:00::","synset_id":"rule.n.03"},{"index":4,"sense_key":"rule%1:10:01::","synset_id":"rule.n.04"},{"index":5,"sense_key":"rule%1:09:03::","synset_id":"principle.n.01"},{"index":6,"sense_key":"rule%1:28:00::","synset_id":"rule.n.06"},{"index":7,"sense_key":"rule%1:26:00::","synset_id":"dominion.n.01"},{"index":8,"sense_key":"rule%1:10:02::","synset_id":"rule.n.08"},{"index":9,"sense_key":"rule%1:10:03::","synset_id":"rule.n.09"},{"index":10,"sense_key":"rule%1:09:02::","synset_id":"principle.n.04"},{"index":11,"sense_key":"rule%1:09:04::","synset_id":"rule.n.11"},{"index":12,"sense_key":"rule%1:06:00::","synset_id":"rule.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"rule%1:10:00::","call_ids":["semeval2013.d005.s007.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"rule%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1450,"cached_input_tokens":0,"output_tokens":106,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00396,"input_uncached_usd":0.0029,"input_cached_usd":0.0,"output_usd":0.0010600000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5687559159705415}
{"item_id":"semeval2013.d005.s008.t000","gold_sense_keys":["trial%1:04:02::"],"candidates":[{"index":1,"sense_key":"trial%1:04:00::","synset_id":"test.n.05"},{"index":2,"sense_key":"trial%1:09:00::","synset_id":"trial.n.02"},{"index":3,"sense_key":"trial%1:04:03::","synset_id":"test.n.04"},{"index":4,"sense_key":"trial%1:04:02::","synset_id":"trial.n.04"},{"index":5,"sense_key":"trial%1:11:01::","synset_id":"trial.n.05"},{"index":6,"sense_key":"trial%1:11:00::","synset_id":"trial.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"trial%1:04:02::","call_ids":["semeval2013.d005.s008.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"trial%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1116,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002352,"input_uncached_usd":0.002232,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.018526375060901}
{"item_id":"semeval2013.d005.s008.t001","gold_sense_keys":["danger%1:04:00::","danger%1:26:00::"],"candidates":[{"index":1,"sense_key":"danger%1:26:00::","synset_id":"danger.n.01"},{"index":2,"sense_key":"danger%1:04:00::","synset_id":"risk.n.02"},{"index":3,"sense_key":"danger%1:26:01::","synset_id":"danger.n.03"},{"index":4,"sense_key":"danger%1:15:00::","synset_id":"danger.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"danger%1:04:00::","call_ids":["semeval2013.d005.s008.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"danger%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":913,"cached_input_tokens":0,"output_tokens":222,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004046,"input_uncached_usd":0.001826,"input_cached_usd":0.0,"output_usd":0.00222,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.3852465000236407}
{"item_id":"semeval2013.d005.s008.t002","gold_sense_keys":["inclusion%1:04:00::"],"candidates":[{"index":1,"sense_key":"inclusion%1:26:00::","synset_id":"inclusion.n.01"},{"index":2,"sense_key":"inclusion%1:24:00::","synset_id":"inclusion.n.02"},{"index":3,"sense_key":"inclusion%1:17:00::","synset_id":"inclusion_body.n.01"},{"index":4,"sense_key":"inclusion%1:04:00::","synset_id":"inclusion.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"inclusion%1:04:00::","call_ids":["semeval2013.d005.s008.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"inclusion%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":892,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002334,"input_uncached_usd":0.001784,"input_cached_usd":0.0,"output_usd":0.00055,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6714536250801757}
{"item_id":"semeval2013.d005.s008.t004","gold_sense_keys":["individual%1:03:00::"],"candidates":[{"index":1,"sense_key":"individual%1:03:00::","synset_id":"person.n.01"},{"index":2,"sense_key":"individual%1:18:00::","synset_id":"individual.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"individual%1:03:00::","call_ids":["semeval2013.d005.s008.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"individual%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":766,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001652,"input_uncached_usd":0.001532,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8914930829778314}
{"item_id":"semeval2013.d005.s008.t005","gold_sense_keys":["right%1:07:00::"],"candidates":[{"index":1,"sense_key":"right%1:07:00::","synset_id":"right.n.01"},{"index":2,"sense_key":"right%1:15:00::","synset_id":"right.n.02"},{"index":3,"sense_key":"right%1:06:00::","synset_id":"right_field.n.01"},{"index":4,"sense_key":"right%1:14:00::","synset_id":"right.n.04"},{"index":5,"sense_key":"right%1:08:00::","synset_id":"right.n.05"},{"index":6,"sense_key":"right%1:04:00::","synset_id":"right.n.06"},{"index":7,"sense_key":"right%1:07:01::","synset_id":"right.n.07"},{"index":8,"sense_key":"right%1:21:00::","synset_id":"right.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"right%1:07:00::","call_ids":["semeval2013.d005.s008.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"right%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1208,"cached_input_tokens":0,"output_tokens":114,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003556,"input_uncached_usd":0.0024159999999999997,"input_cached_usd":0.0,"output_usd":0.0011400000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6035790829919279}
{"item_id":"semeval2013.d005.s009.t003","gold_sense_keys":["claim%1:10:02::"],"candidates":[{"index":1,"sense_key":"claim%1:10:00::","synset_id":"claim.n.01"},{"index":2,"sense_key":"claim%1:10:02::","synset_id":"claim.n.02"},{"index":3,"sense_key":"claim%1:04:00::","synset_id":"claim.n.03"},{"index":4,"sense_key":"claim%1:07:02::","synset_id":"claim.n.04"},{"index":5,"sense_key":"claim%1:07:01::","synset_id":"title.n.07"},{"index":6,"sense_key":"claim%1:10:01::","synset_id":"call.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"claim%1:10:02::","call_ids":["semeval2013.d005.s009.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"claim%1:10:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":969,"cached_input_tokens":0,"output_tokens":86,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002798,"input_uncached_usd":0.0019379999999999998,"input_cached_usd":0.0,"output_usd":0.0008600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.780574542004615}
{"item_id":"semeval2013.d005.s009.t005","gold_sense_keys":["week%1:28:02::"],"candidates":[{"index":1,"sense_key":"week%1:28:00::","synset_id":"week.n.01"},{"index":2,"sense_key":"week%1:28:01::","synset_id":"workweek.n.01"},{"index":3,"sense_key":"week%1:28:02::","synset_id":"week.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"week%1:28:02::","call_ids":["semeval2013.d005.s009.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"week%1:28:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":796,"cached_input_tokens":0,"output_tokens":144,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003032,"input_uncached_usd":0.0015919999999999999,"input_cached_usd":0.0,"output_usd":0.00144,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.957202082965523}
{"item_id":"semeval2013.d005.s009.t007","gold_sense_keys":["evidence%1:09:00::"],"candidates":[{"index":1,"sense_key":"evidence%1:09:00::","synset_id":"evidence.n.01"},{"index":2,"sense_key":"evidence%1:10:01::","synset_id":"evidence.n.02"},{"index":3,"sense_key":"evidence%1:10:00::","synset_id":"evidence.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"evidence%1:09:00::","call_ids":["semeval2013.d005.s009.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"evidence%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":830,"cached_input_tokens":0,"output_tokens":180,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0034600000000000004,"input_uncached_usd":0.00166,"input_cached_usd":0.0,"output_usd":0.0018000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.4357657090295106}
{"item_id":"semeval2013.d005.s009.t008","gold_sense_keys":["jury%1:14:00::"],"candidates":[{"index":1,"sense_key":"jury%1:14:00::","synset_id":"jury.n.01"},{"index":2,"sense_key":"jury%1:14:01::","synset_id":"jury.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"jury%1:14:00::","call_ids":["semeval2013.d005.s009.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"jury%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":735,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00159,"input_uncached_usd":0.00147,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.6514722499996424}
{"item_id":"semeval2013.d005.s009.t009","gold_sense_keys":["passion%1:12:00::"],"candidates":[{"index":1,"sense_key":"passion%1:12:00::","synset_id":"passion.n.01"},{"index":2,"sense_key":"passion%1:07:00::","synset_id":"heat.n.04"},{"index":3,"sense_key":"passion%1:26:01::","synset_id":"rage.n.03"},{"index":4,"sense_key":"passion%1:16:00::","synset_id":"mania.n.01"},{"index":5,"sense_key":"passion%1:12:02::","synset_id":"passion.n.05"},{"index":6,"sense_key":"passion%1:09:00::","synset_id":"love.n.02"},{"index":7,"sense_key":"passion%1:26:02::","synset_id":"passion.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"passion%1:12:00::","call_ids":["semeval2013.d005.s009.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"passion%1:12:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":984,"cached_input_tokens":0,"output_tokens":113,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003098,"input_uncached_usd":0.0019679999999999997,"input_cached_usd":0.0,"output_usd":0.0011300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.692322792019695}
{"item_id":"semeval2013.d005.s010.t000","gold_sense_keys":["last_word%1:10:00::"],"candidates":[{"index":1,"sense_key":"last_word%1:10:00::","synset_id":"last_word.n.01"},{"index":2,"sense_key":"last_word%1:10:01::","synset_id":"last_word.n.02"},{"index":3,"sense_key":"last_word%1:07:00::","synset_id":"chic.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"last_word%1:10:01::","call_ids":["semeval2013.d005.s010.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"last_word%1:10:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":761,"cached_input_tokens":0,"output_tokens":128,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002802,"input_uncached_usd":0.001522,"input_cached_usd":0.0,"output_usd":0.00128,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6916820410406217}
{"item_id":"semeval2013.d005.s010.t001","gold_sense_keys":["issue%1:09:01::"],"candidates":[{"index":1,"sense_key":"issue%1:09:01::","synset_id":"issue.n.01"},{"index":2,"sense_key":"issue%1:10:00::","synset_id":"issue.n.02"},{"index":3,"sense_key":"issue%1:09:00::","synset_id":"topic.n.02"},{"index":4,"sense_key":"issue%1:04:01::","synset_id":"issue.n.04"},{"index":5,"sense_key":"issue%1:21:01::","synset_id":"issue.n.05"},{"index":6,"sense_key":"issue%1:21:00::","synset_id":"return.n.06"},{"index":7,"sense_key":"issue%1:19:00::","synset_id":"consequence.n.01"},{"index":8,"sense_key":"issue%1:18:00::","synset_id":"offspring.n.01"},{"index":9,"sense_key":"issue%1:11:00::","synset_id":"emergence.n.02"},{"index":10,"sense_key":"issue%1:06:00::","synset_id":"exit.n.01"},{"index":11,"sense_key":"issue%1:04:00::","synset_id":"issue.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"issue%1:09:01::","call_ids":["semeval2013.d005.s010.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"issue%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1378,"cached_input_tokens":0,"output_tokens":104,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003796,"input_uncached_usd":0.0027559999999999998,"input_cached_usd":0.0,"output_usd":0.0010400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4333622920094058}
{"item_id":"semeval2013.d005.s011.t000","gold_sense_keys":["decade%1:28:00::"],"candidates":[{"index":1,"sense_key":"decade%1:28:00::","synset_id":"decade.n.01"},{"index":2,"sense_key":"decade%1:23:00::","synset_id":"ten.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"decade%1:28:00::","call_ids":["semeval2013.d005.s011.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"decade%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":768,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0016560000000000001,"input_uncached_usd":0.001536,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4181022089906037}
{"item_id":"semeval2013.d005.s011.t002","gold_sense_keys":["coverage%1:10:00::"],"candidates":[{"index":1,"sense_key":"coverage%1:21:00::","synset_id":"coverage.n.01"},{"index":2,"sense_key":"coverage%1:07:00::","synset_id":"coverage.n.02"},{"index":3,"sense_key":"coverage%1:10:00::","synset_id":"coverage.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"coverage%1:10:00::","call_ids":["semeval2013.d005.s011.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"coverage%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":839,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0017980000000000001,"input_uncached_usd":0.001678,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4849085420137271}
{"item_id":"semeval2013.d005.s011.t003","gold_sense_keys":["case%1:04:00::"],"candidates":[{"index":1,"sense_key":"case%1:11:00::","synset_id":"case.n.01"},{"index":2,"sense_key":"case%1:26:00::","synset_id":"event.n.02"},{"index":3,"sense_key":"case%1:04:00::","synset_id":"lawsuit.n.01"},{"index":4,"sense_key":"case%1:09:00::","synset_id":"case.n.04"},{"index":5,"sense_key":"case%1:06:00::","synset_id":"case.n.05"},{"index":6,"sense_key":"case%1:18:03::","synset_id":"case.n.06"},{"index":7,"sense_key":"case%1:18:02::","synset_id":"subject.n.06"},{"index":8,"sense_key":"case%1:10:02::","synset_id":"case.n.08"},{"index":9,"sense_key":"case%1:10:01::","synset_id":"case.n.09"},{"index":10,"sense_key":"case%1:23:00::","synset_id":"case.n.10"},{"index":11,"sense_key":"case%1:10:00::","synset_id":"case.n.11"},{"index":12,"sense_key":"case%1:26:02::","synset_id":"case.n.12"},{"index":13,"sense_key":"case%1:18:04::","synset_id":"character.n.05"},{"index":14,"sense_key":"case%1:10:03::","synset_id":"font.n.01"},{"index":15,"sense_key":"case%1:08:00::","synset_id":"sheath.n.02"},{"index":16,"sense_key":"case%1:06:06::","synset_id":"shell.n.08"},{"index":17,"sense_key":"case%1:06:07::","synset_id":"casing.n.03"},{"index":18,"sense_key":"case%1:06:03::","synset_id":"case.n.18"},{"index":19,"sense_key":"case%1:06:02::","synset_id":"case.n.19"},{"index":20,"sense_key":"case%1:06:01::","synset_id":"case.n.20"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"case%1:04:00::","call_ids":["semeval2013.d005.s011.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"case%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1915,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00439,"input_uncached_usd":0.0038299999999999996,"input_cached_usd":0.0,"output_usd":0.0005600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.440388625022024}
{"item_id":"semeval2013.d005.s011.t004","gold_sense_keys":["question%1:07:00::"],"candidates":[{"index":1,"sense_key":"question%1:10:00::","synset_id":"question.n.01"},{"index":2,"sense_key":"question%1:10:01::","synset_id":"question.n.02"},{"index":3,"sense_key":"question%1:10:03::","synset_id":"question.n.03"},{"index":4,"sense_key":"question%1:07:00::","synset_id":"doubt.n.02"},{"index":5,"sense_key":"question%1:10:02::","synset_id":"motion.n.05"},{"index":6,"sense_key":"question%1:10:04::","synset_id":"question.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"question%1:07:00::","call_ids":["semeval2013.d005.s011.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"question%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1096,"cached_input_tokens":0,"output_tokens":153,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003722,"input_uncached_usd":0.002192,"input_cached_usd":0.0,"output_usd":0.0015300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.002251583035104}
{"item_id":"semeval2013.d005.s011.t005","gold_sense_keys":["trial%1:04:02::"],"candidates":[{"index":1,"sense_key":"trial%1:04:00::","synset_id":"test.n.05"},{"index":2,"sense_key":"trial%1:09:00::","synset_id":"trial.n.02"},{"index":3,"sense_key":"trial%1:04:03::","synset_id":"test.n.04"},{"index":4,"sense_key":"trial%1:04:02::","synset_id":"trial.n.04"},{"index":5,"sense_key":"trial%1:11:01::","synset_id":"trial.n.05"},{"index":6,"sense_key":"trial%1:11:00::","synset_id":"trial.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"trial%1:04:02::","call_ids":["semeval2013.d005.s011.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"trial%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1099,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0023179999999999997,"input_uncached_usd":0.002198,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3021431249799207}
{"item_id":"semeval2013.d005.s011.t006","gold_sense_keys":["fairness%1:07:00::"],"candidates":[{"index":1,"sense_key":"fairness%1:07:00::","synset_id":"fairness.n.01"},{"index":2,"sense_key":"fairness%1:09:00::","synset_id":"fairness.n.02"},{"index":3,"sense_key":"fairness%1:07:01::","synset_id":"paleness.n.02"},{"index":4,"sense_key":"fairness%1:07:02::","synset_id":"comeliness.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"fairness%1:07:00::","call_ids":["semeval2013.d005.s011.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"fairness%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":887,"cached_input_tokens":0,"output_tokens":113,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0029040000000000003,"input_uncached_usd":0.001774,"input_cached_usd":0.0,"output_usd":0.0011300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.4417074589291587}
{"item_id":"semeval2013.d005.s012.t002","gold_sense_keys":["conviction%1:04:00::"],"candidates":[{"index":1,"sense_key":"conviction%1:09:00::","synset_id":"conviction.n.01"},{"index":2,"sense_key":"conviction%1:04:00::","synset_id":"conviction.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"conviction%1:04:00::","call_ids":["semeval2013.d005.s012.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"conviction%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":758,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001636,"input_uncached_usd":0.001516,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":6.465474874945357}
{"item_id":"semeval2013.d005.s012.t003","gold_sense_keys":["cleveland%1:15:00::"],"candidates":[{"index":1,"sense_key":"cleveland%1:15:00::","synset_id":"cleveland.n.01"},{"index":2,"sense_key":"cleveland%1:18:00::","synset_id":"cleveland.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cleveland%1:15:00::","call_ids":["semeval2013.d005.s012.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cleveland%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":753,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001626,"input_uncached_usd":0.001506,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2175752909388393}
{"item_id":"semeval2013.d005.s012.t007","gold_sense_keys":["decision%1:09:00::"],"candidates":[{"index":1,"sense_key":"decision%1:04:00::","synset_id":"decision.n.01"},{"index":2,"sense_key":"decision%1:09:00::","synset_id":"decision.n.02"},{"index":3,"sense_key":"decision%1:11:02::","synset_id":"decision.n.03"},{"index":4,"sense_key":"decision%1:11:01::","synset_id":"decision.n.04"},{"index":5,"sense_key":"decision%1:07:00::","synset_id":"decisiveness.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"decision%1:09:00::","call_ids":["semeval2013.d005.s012.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"decision%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":946,"cached_input_tokens":0,"output_tokens":50,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002392,"input_uncached_usd":0.001892,"input_cached_usd":0.0,"output_usd":0.0005,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4115476659499109}
{"item_id":"semeval2013.d005.s012.t008","gold_sense_keys":["carnival%1:04:02::"],"candidates":[{"index":1,"sense_key":"carnival%1:04:00::","synset_id":"carnival.n.01"},{"index":2,"sense_key":"carnival%1:04:02::","synset_id":"circus.n.03"},{"index":3,"sense_key":"carnival%1:04:01::","synset_id":"carnival.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"carnival%1:04:02::","call_ids":["semeval2013.d005.s012.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"carnival%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":799,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0017180000000000001,"input_uncached_usd":0.001598,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3879651670577005}
{"item_id":"semeval2013.d005.s012.t009","gold_sense_keys":["atmosphere%1:26:01::"],"candidates":[{"index":1,"sense_key":"atmosphere%1:26:01::","synset_id":"atmosphere.n.01"},{"index":2,"sense_key":"atmosphere%1:23:00::","synset_id":"standard_atmosphere.n.01"},{"index":3,"sense_key":"atmosphere%1:15:00::","synset_id":"atmosphere.n.03"},{"index":4,"sense_key":"atmosphere%1:26:00::","synset_id":"atmosphere.n.04"},{"index":5,"sense_key":"atmosphere%1:17:00::","synset_id":"atmosphere.n.05"},{"index":6,"sense_key":"atmosphere%1:07:00::","synset_id":"air.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"atmosphere%1:26:01::","call_ids":["semeval2013.d005.s012.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"atmosphere%1:26:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1024,"cached_input_tokens":0,"output_tokens":122,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003268,"input_uncached_usd":0.002048,"input_cached_usd":0.0,"output_usd":0.0012200000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.3702906250255182}
{"item_id":"semeval2013.d005.s012.t010","gold_sense_keys":["trial%1:04:02::"],"candidates":[{"index":1,"sense_key":"trial%1:04:00::","synset_id":"test.n.05"},{"index":2,"sense_key":"trial%1:09:00::","synset_id":"trial.n.02"},{"index":3,"sense_key":"trial%1:04:03::","synset_id":"test.n.04"},{"index":4,"sense_key":"trial%1:04:02::","synset_id":"trial.n.04"},{"index":5,"sense_key":"trial%1:11:01::","synset_id":"trial.n.05"},{"index":6,"sense_key":"trial%1:11:00::","synset_id":"trial.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"trial%1:04:02::","call_ids":["semeval2013.d005.s012.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"trial%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1046,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002212,"input_uncached_usd":0.002092,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8899676250293851}
{"item_id":"semeval2013.d005.s013.t000","gold_sense_keys":["crime%1:04:00::"],"candidates":[{"index":1,"sense_key":"crime%1:04:00::","synset_id":"crime.n.01"},{"index":2,"sense_key":"crime%1:04:01::","synset_id":"crime.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"crime%1:04:00::","call_ids":["semeval2013.d005.s013.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"crime%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":794,"cached_input_tokens":0,"output_tokens":52,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002108,"input_uncached_usd":0.001588,"input_cached_usd":0.0,"output_usd":0.0005200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.751845041057095}
{"item_id":"semeval2013.d005.s013.t001","gold_sense_keys":["inspiration%1:09:02::"],"candidates":[{"index":1,"sense_key":"inspiration%1:09:02::","synset_id":"inspiration.n.01"},{"index":2,"sense_key":"inspiration%1:06:00::","synset_id":"inspiration.n.02"},{"index":3,"sense_key":"inspiration%1:09:01::","synset_id":"inspiration.n.03"},{"index":4,"sense_key":"inspiration%1:09:00::","synset_id":"divine_guidance.n.01"},{"index":5,"sense_key":"inspiration%1:04:01::","synset_id":"inspiration.n.05"},{"index":6,"sense_key":"inspiration%1:04:00::","synset_id":"inhalation.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"inspiration%1:09:02::","call_ids":["semeval2013.d005.s013.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"inspiration%1:09:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1026,"cached_input_tokens":0,"output_tokens":162,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003672,"input_uncached_usd":0.002052,"input_cached_usd":0.0,"output_usd":0.0016200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.1055124580161646}
{"item_id":"semeval2013.d005.s014.t002","gold_sense_keys":["conviction%1:04:00::"],"candidates":[{"index":1,"sense_key":"conviction%1:09:00::","synset_id":"conviction.n.01"},{"index":2,"sense_key":"conviction%1:04:00::","synset_id":"conviction.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"conviction%1:04:00::","call_ids":["semeval2013.d005.s014.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"conviction%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":808,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0017360000000000001,"input_uncached_usd":0.001616,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.032828207942657}
{"item_id":"semeval2013.d005.s014.t003","gold_sense_keys":["half%1:23:00::"],"candidates":[{"index":1,"sense_key":"half%1:23:00::","synset_id":"one-half.n.01"},{"index":2,"sense_key":"half%1:28:00::","synset_id":"half.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"half%1:23:00::","call_ids":["semeval2013.d005.s014.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"half%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":766,"cached_input_tokens":0,"output_tokens":91,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002442,"input_uncached_usd":0.001532,"input_cached_usd":0.0,"output_usd":0.0009100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1520610000006855}
{"item_id":"semeval2013.d005.s014.t004","gold_sense_keys":["century%1:28:00::"],"candidates":[{"index":1,"sense_key":"century%1:28:00::","synset_id":"century.n.01"},{"index":2,"sense_key":"century%1:23:00::","synset_id":"hundred.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"century%1:28:00::","call_ids":["semeval2013.d005.s014.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"century%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":729,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001578,"input_uncached_usd":0.0014579999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0007970000151545}
{"item_id":"semeval2013.d005.s014.t005","gold_sense_keys":["argument%1:09:01::","argument%1:10:02::"],"candidates":[{"index":1,"sense_key":"argument%1:10:02::","synset_id":"argument.n.01"},{"index":2,"sense_key":"argument%1:10:03::","synset_id":"controversy.n.01"},{"index":3,"sense_key":"argument%1:10:00::","synset_id":"argument.n.03"},{"index":4,"sense_key":"argument%1:10:01::","synset_id":"argument.n.04"},{"index":5,"sense_key":"argument%1:10:04::","synset_id":"argument.n.05"},{"index":6,"sense_key":"argument%1:09:00::","synset_id":"argument.n.06"},{"index":7,"sense_key":"argument%1:09:01::","synset_id":"argumentation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"argument%1:10:02::","call_ids":["semeval2013.d005.s014.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"argument%1:10:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1181,"cached_input_tokens":0,"output_tokens":123,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003592,"input_uncached_usd":0.002362,"input_cached_usd":0.0,"output_usd":0.0012300000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9600220000138506}
{"item_id":"semeval2013.d005.s014.t006","gold_sense_keys":["rule%1:10:00::"],"candidates":[{"index":1,"sense_key":"rule%1:09:00::","synset_id":"rule.n.01"},{"index":2,"sense_key":"rule%1:09:01::","synset_id":"convention.n.02"},{"index":3,"sense_key":"rule%1:10:00::","synset_id":"rule.n.03"},{"index":4,"sense_key":"rule%1:10:01::","synset_id":"rule.n.04"},{"index":5,"sense_key":"rule%1:09:03::","synset_id":"principle.n.01"},{"index":6,"sense_key":"rule%1:28:00::","synset_id":"rule.n.06"},{"index":7,"sense_key":"rule%1:26:00::","synset_id":"dominion.n.01"},{"index":8,"sense_key":"rule%1:10:02::","synset_id":"rule.n.08"},{"index":9,"sense_key":"rule%1:10:03::","synset_id":"rule.n.09"},{"index":10,"sense_key":"rule%1:09:02::","synset_id":"principle.n.04"},{"index":11,"sense_key":"rule%1:09:04::","synset_id":"rule.n.11"},{"index":12,"sense_key":"rule%1:06:00::","synset_id":"rule.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"rule%1:10:00::","call_ids":["semeval2013.d005.s014.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"rule%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1475,"cached_input_tokens":0,"output_tokens":139,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00434,"input_uncached_usd":0.00295,"input_cached_usd":0.0,"output_usd":0.0013900000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7891274589346722}
{"item_id":"semeval2013.d005.s014.t007","gold_sense_keys":["way%1:04:01::","way%1:07:01::"],"candidates":[{"index":1,"sense_key":"way%1:07:01::","synset_id":"manner.n.01"},{"index":2,"sense_key":"way%1:04:01::","synset_id":"means.n.01"},{"index":3,"sense_key":"way%1:15:00::","synset_id":"direction.n.01"},{"index":4,"sense_key":"way%1:26:00::","synset_id":"way.n.04"},{"index":5,"sense_key":"way%1:04:00::","synset_id":"way.n.05"},{"index":6,"sense_key":"way%1:06:00::","synset_id":"way.n.06"},{"index":7,"sense_key":"way%1:04:02::","synset_id":"way.n.07"},{"index":8,"sense_key":"way%1:23:00::","synset_id":"room.n.02"},{"index":9,"sense_key":"way%1:07:02::","synset_id":"way.n.09"},{"index":10,"sense_key":"way%1:09:00::","synset_id":"way.n.10"},{"index":11,"sense_key":"way%1:09:01::","synset_id":"way.n.11"},{"index":12,"sense_key":"way%1:21:00::","synset_id":"way.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"way%1:07:01::","call_ids":["semeval2013.d005.s014.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"way%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1278,"cached_input_tokens":0,"output_tokens":142,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0039759999999999995,"input_uncached_usd":0.0025559999999999997,"input_cached_usd":0.0,"output_usd":0.00142,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.929936125059612}
{"item_id":"semeval2013.d005.s014.t010","gold_sense_keys":["case%1:04:00::"],"candidates":[{"index":1,"sense_key":"case%1:11:00::","synset_id":"case.n.01"},{"index":2,"sense_key":"case%1:26:00::","synset_id":"event.n.02"},{"index":3,"sense_key":"case%1:04:00::","synset_id":"lawsuit.n.01"},{"index":4,"sense_key":"case%1:09:00::","synset_id":"case.n.04"},{"index":5,"sense_key":"case%1:06:00::","synset_id":"case.n.05"},{"index":6,"sense_key":"case%1:18:03::","synset_id":"case.n.06"},{"index":7,"sense_key":"case%1:18:02::","synset_id":"subject.n.06"},{"index":8,"sense_key":"case%1:10:02::","synset_id":"case.n.08"},{"index":9,"sense_key":"case%1:10:01::","synset_id":"case.n.09"},{"index":10,"sense_key":"case%1:23:00::","synset_id":"case.n.10"},{"index":11,"sense_key":"case%1:10:00::","synset_id":"case.n.11"},{"index":12,"sense_key":"case%1:26:02::","synset_id":"case.n.12"},{"index":13,"sense_key":"case%1:18:04::","synset_id":"character.n.05"},{"index":14,"sense_key":"case%1:10:03::","synset_id":"font.n.01"},{"index":15,"sense_key":"case%1:08:00::","synset_id":"sheath.n.02"},{"index":16,"sense_key":"case%1:06:06::","synset_id":"shell.n.08"},{"index":17,"sense_key":"case%1:06:07::","synset_id":"casing.n.03"},{"index":18,"sense_key":"case%1:06:03::","synset_id":"case.n.18"},{"index":19,"sense_key":"case%1:06:02::","synset_id":"case.n.19"},{"index":20,"sense_key":"case%1:06:01::","synset_id":"case.n.20"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"case%1:04:00::","call_ids":["semeval2013.d005.s014.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"case%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1912,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004494,"input_uncached_usd":0.0038239999999999997,"input_cached_usd":0.0,"output_usd":0.00067,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3861202088883147}
{"item_id":"semeval2013.d005.s015.t000","gold_sense_keys":["reason%1:09:01::","reason%1:10:00::","reason%1:10:01::","reason%1:16:00::"],"candidates":[{"index":1,"sense_key":"reason%1:16:00::","synset_id":"reason.n.01"},{"index":2,"sense_key":"reason%1:10:01::","synset_id":"reason.n.02"},{"index":3,"sense_key":"reason%1:09:00::","synset_id":"reason.n.03"},{"index":4,"sense_key":"reason%1:26:00::","synset_id":"rationality.n.01"},{"index":5,"sense_key":"reason%1:10:00::","synset_id":"cause.n.02"},{"index":6,"sense_key":"reason%1:09:01::","synset_id":"reason.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"reason%1:10:00::","call_ids":["semeval2013.d005.s015.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"reason%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1047,"cached_input_tokens":0,"output_tokens":159,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0036839999999999998,"input_uncached_usd":0.002094,"input_cached_usd":0.0,"output_usd":0.00159,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.672900917008519}
{"item_id":"semeval2013.d005.s015.t001","gold_sense_keys":["concern%1:12:01::"],"candidates":[{"index":1,"sense_key":"concern%1:09:00::","synset_id":"concern.n.01"},{"index":2,"sense_key":"concern%1:12:01::","synset_id":"concern.n.02"},{"index":3,"sense_key":"concern%1:12:00::","synset_id":"concern.n.03"},{"index":4,"sense_key":"concern%1:09:01::","synset_id":"concern.n.04"},{"index":5,"sense_key":"concern%1:14:00::","synset_id":"business.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"concern%1:12:01::","call_ids":["semeval2013.d005.s015.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"concern%1:12:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":969,"cached_input_tokens":0,"output_tokens":111,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003048,"input_uncached_usd":0.0019379999999999998,"input_cached_usd":0.0,"output_usd":0.00111,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1470605420181528}
{"item_id":"semeval2013.d005.s015.t002","gold_sense_keys":["impact%1:19:00::"],"candidates":[{"index":1,"sense_key":"impact%1:11:00::","synset_id":"impact.n.01"},{"index":2,"sense_key":"impact%1:19:00::","synset_id":"impact.n.02"},{"index":3,"sense_key":"impact%1:04:00::","synset_id":"impingement.n.01"},{"index":4,"sense_key":"impact%1:04:01::","synset_id":"shock.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"impact%1:19:00::","call_ids":["semeval2013.d005.s015.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"impact%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":856,"cached_input_tokens":0,"output_tokens":89,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002602,"input_uncached_usd":0.001712,"input_cached_usd":0.0,"output_usd":0.0008900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5081944579724222}
{"item_id":"semeval2013.d005.s015.t003","gold_sense_keys":["trial%1:04:02::"],"candidates":[{"index":1,"sense_key":"trial%1:04:00::","synset_id":"test.n.05"},{"index":2,"sense_key":"trial%1:09:00::","synset_id":"trial.n.02"},{"index":3,"sense_key":"trial%1:04:03::","synset_id":"test.n.04"},{"index":4,"sense_key":"trial%1:04:02::","synset_id":"trial.n.04"},{"index":5,"sense_key":"trial%1:11:01::","synset_id":"trial.n.05"},{"index":6,"sense_key":"trial%1:11:00::","synset_id":"trial.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"trial%1:04:02::","call_ids":["semeval2013.d005.s015.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"trial%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1070,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00226,"input_uncached_usd":0.00214,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9656783330719918}
{"item_id":"semeval2013.d005.s015.t005","gold_sense_keys":["connecticut%1:15:00::"],"candidates":[{"index":1,"sense_key":"connecticut%1:15:00::","synset_id":"connecticut.n.01"},{"index":2,"sense_key":"connecticut%1:17:00::","synset_id":"connecticut.n.02"},{"index":3,"sense_key":"connecticut%1:15:01::","synset_id":"connecticut.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"connecticut%1:15:00::","call_ids":["semeval2013.d005.s015.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"connecticut%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":837,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002324,"input_uncached_usd":0.001674,"input_cached_usd":0.0,"output_usd":0.0006500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.6286197500303388}
{"item_id":"semeval2013.d005.s015.t007","gold_sense_keys":["report%1:10:03::"],"candidates":[{"index":1,"sense_key":"report%1:10:03::","synset_id":"report.n.01"},{"index":2,"sense_key":"report%1:10:01::","synset_id":"report.n.02"},{"index":3,"sense_key":"report%1:10:00::","synset_id":"report.n.03"},{"index":4,"sense_key":"report%1:11:00::","synset_id":"report.n.04"},{"index":5,"sense_key":"report%1:10:04::","synset_id":"report_card.n.01"},{"index":6,"sense_key":"report%1:10:02::","synset_id":"composition.n.08"},{"index":7,"sense_key":"report%1:09:00::","synset_id":"reputation.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"report%1:10:03::","call_ids":["semeval2013.d005.s015.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"report%1:10:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1081,"cached_input_tokens":0,"output_tokens":75,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0029119999999999997,"input_uncached_usd":0.002162,"input_cached_usd":0.0,"output_usd":0.00075,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.977849791990593}
{"item_id":"semeval2013.d005.s016.t001","gold_sense_keys":["appeal%1:04:00::"],"candidates":[{"index":1,"sense_key":"appeal%1:10:00::","synset_id":"entreaty.n.01"},{"index":2,"sense_key":"appeal%1:07:00::","synset_id":"appeal.n.02"},{"index":3,"sense_key":"appeal%1:04:00::","synset_id":"appeal.n.03"},{"index":4,"sense_key":"appeal%1:10:02::","synset_id":"solicitation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"appeal%1:04:00::","call_ids":["semeval2013.d005.s016.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"appeal%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":973,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002606,"input_uncached_usd":0.0019459999999999998,"input_cached_usd":0.0,"output_usd":0.0006600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0841809160774574}
{"item_id":"semeval2013.d005.s016.t002","gold_sense_keys":["judge%1:18:00::"],"candidates":[{"index":1,"sense_key":"judge%1:18:00::","synset_id":"judge.n.01"},{"index":2,"sense_key":"judge%1:18:01::","synset_id":"evaluator.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"judge%1:18:00::","call_ids":["semeval2013.d005.s016.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"judge%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":765,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00165,"input_uncached_usd":0.00153,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.7932200410868973}
{"item_id":"semeval2013.d005.s016.t003","gold_sense_keys":["idea%1:09:00::"],"candidates":[{"index":1,"sense_key":"idea%1:09:00::","synset_id":"idea.n.01"},{"index":2,"sense_key":"idea%1:09:01::","synset_id":"mind.n.06"},{"index":3,"sense_key":"idea%1:09:03::","synset_id":"idea.n.03"},{"index":4,"sense_key":"idea%1:09:02::","synset_id":"estimate.n.01"},{"index":5,"sense_key":"idea%1:10:00::","synset_id":"theme.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"idea%1:09:00::","call_ids":["semeval2013.d005.s016.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"idea%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1015,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00215,"input_uncached_usd":0.00203,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0558309580665082}
{"item_id":"semeval2013.d005.s016.t004","gold_sense_keys":["communication%1:10:01::"],"candidates":[{"index":1,"sense_key":"communication%1:10:01::","synset_id":"communication.n.01"},{"index":2,"sense_key":"communication%1:03:00::","synset_id":"communication.n.02"},{"index":3,"sense_key":"communication%1:24:00::","synset_id":"communication.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"communication%1:10:01::","call_ids":["semeval2013.d005.s016.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"communication%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":835,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00225,"input_uncached_usd":0.0016699999999999998,"input_cached_usd":0.0,"output_usd":0.00058,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.05299641599413}
{"item_id":"semeval2013.d005.s016.t006","gold_sense_keys":["trial%1:04:02::"],"candidates":[{"index":1,"sense_key":"trial%1:04:00::","synset_id":"test.n.05"},{"index":2,"sense_key":"trial%1:09:00::","synset_id":"trial.n.02"},{"index":3,"sense_key":"trial%1:04:03::","synset_id":"test.n.04"},{"index":4,"sense_key":"trial%1:04:02::","synset_id":"trial.n.04"},{"index":5,"sense_key":"trial%1:11:01::","synset_id":"trial.n.05"},{"index":6,"sense_key":"trial%1:11:00::","synset_id":"trial.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"trial%1:04:02::","call_ids":["semeval2013.d005.s016.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"trial%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1101,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002322,"input_uncached_usd":0.002202,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.592244583996944}
{"item_id":"semeval2013.d005.s017.t000","gold_sense_keys":["issue%1:09:01::"],"candidates":[{"index":1,"sense_key":"issue%1:09:01::","synset_id":"issue.n.01"},{"index":2,"sense_key":"issue%1:10:00::","synset_id":"issue.n.02"},{"index":3,"sense_key":"issue%1:09:00::","synset_id":"topic.n.02"},{"index":4,"sense_key":"issue%1:04:01::","synset_id":"issue.n.04"},{"index":5,"sense_key":"issue%1:21:01::","synset_id":"issue.n.05"},{"index":6,"sense_key":"issue%1:21:00::","synset_id":"return.n.06"},{"index":7,"sense_key":"issue%1:19:00::","synset_id":"consequence.n.01"},{"index":8,"sense_key":"issue%1:18:00::","synset_id":"offspring.n.01"},{"index":9,"sense_key":"issue%1:11:00::","synset_id":"emergence.n.02"},{"index":10,"sense_key":"issue%1:06:00::","synset_id":"exit.n.01"},{"index":11,"sense_key":"issue%1:04:00::","synset_id":"issue.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"issue%1:09:01::","call_ids":["semeval2013.d005.s017.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"issue%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1481,"cached_input_tokens":0,"output_tokens":86,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003822,"input_uncached_usd":0.002962,"input_cached_usd":0.0,"output_usd":0.0008600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4691541249630973}
{"item_id":"semeval2013.d005.s017.t002","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["semeval2013.d005.s017.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":962,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002044,"input_uncached_usd":0.001924,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1831582080340013}
{"item_id":"semeval2013.d005.s017.t003","gold_sense_keys":["execution%1:04:00::"],"candidates":[{"index":1,"sense_key":"execution%1:04:00::","synset_id":"execution.n.01"},{"index":2,"sense_key":"execution%1:04:02::","synset_id":"performance.n.03"},{"index":3,"sense_key":"execution%1:22:00::","synset_id":"execution.n.03"},{"index":4,"sense_key":"execution%1:10:01::","synset_id":"execution.n.04"},{"index":5,"sense_key":"execution%1:10:00::","synset_id":"execution.n.05"},{"index":6,"sense_key":"execution%1:04:03::","synset_id":"execution.n.06"},{"index":7,"sense_key":"execution%1:04:01::","synset_id":"murder.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"execution%1:04:00::","call_ids":["semeval2013.d005.s017.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"execution%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1193,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002506,"input_uncached_usd":0.002386,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.30354754102882}
{"item_id":"semeval2013.d005.s018.t002","gold_sense_keys":["judge%1:18:00::"],"candidates":[{"index":1,"sense_key":"judge%1:18:00::","synset_id":"judge.n.01"},{"index":2,"sense_key":"judge%1:18:01::","synset_id":"evaluator.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"judge%1:18:00::","call_ids":["semeval2013.d005.s018.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"judge%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":763,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0016460000000000001,"input_uncached_usd":0.001526,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9831602909835055}
{"item_id":"semeval2013.d005.s018.t004","gold_sense_keys":["reaction%1:04:00::","reaction%1:09:00::","reaction%1:19:01::"],"candidates":[{"index":1,"sense_key":"reaction%1:22:00::","synset_id":"chemical_reaction.n.01"},{"index":2,"sense_key":"reaction%1:09:00::","synset_id":"reaction.n.02"},{"index":3,"sense_key":"reaction%1:04:00::","synset_id":"reaction.n.03"},{"index":4,"sense_key":"reaction%1:19:00::","synset_id":"reaction.n.04"},{"index":5,"sense_key":"reaction%1:19:01::","synset_id":"reaction.n.05"},{"index":6,"sense_key":"reaction%1:09:01::","synset_id":"reaction.n.06"},{"index":7,"sense_key":"reaction%1:04:01::","synset_id":"reaction.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"reaction%1:19:01::","call_ids":["semeval2013.d005.s018.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"reaction%1:19:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1133,"cached_input_tokens":0,"output_tokens":122,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003486,"input_uncached_usd":0.002266,"input_cached_usd":0.0,"output_usd":0.0012200000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0226428330643103}
{"item_id":"semeval2013.d005.s019.t002","gold_sense_keys":["victim%1:18:00::"],"candidates":[{"index":1,"sense_key":"victim%1:18:00::","synset_id":"victim.n.01"},{"index":2,"sense_key":"victim%1:18:01::","synset_id":"victim.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"victim%1:18:00::","call_ids":["semeval2013.d005.s019.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"victim%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":806,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001732,"input_uncached_usd":0.001612,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.118650084012188}
{"item_id":"semeval2013.d005.s020.t002","gold_sense_keys":["judge%1:18:00::"],"candidates":[{"index":1,"sense_key":"judge%1:18:00::","synset_id":"judge.n.01"},{"index":2,"sense_key":"judge%1:18:01::","synset_id":"evaluator.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"judge%1:18:00::","call_ids":["semeval2013.d005.s020.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"judge%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":747,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001614,"input_uncached_usd":0.0014939999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6057696249336004}
{"item_id":"semeval2013.d005.s020.t005","gold_sense_keys":["pressure%1:07:00::"],"candidates":[{"index":1,"sense_key":"pressure%1:19:00::","synset_id":"pressure.n.01"},{"index":2,"sense_key":"pressure%1:07:00::","synset_id":"pressure.n.02"},{"index":3,"sense_key":"pressure%1:04:00::","synset_id":"press.n.09"},{"index":4,"sense_key":"pressure%1:26:00::","synset_id":"imperativeness.n.01"},{"index":5,"sense_key":"pressure%1:09:00::","synset_id":"pressure.n.05"},{"index":6,"sense_key":"pressure%1:26:01::","synset_id":"pressure.n.06"},{"index":7,"sense_key":"pressure%1:19:01::","synset_id":"atmospheric_pressure.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"pressure%1:07:00::","call_ids":["semeval2013.d005.s020.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"pressure%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1116,"cached_input_tokens":0,"output_tokens":80,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003032,"input_uncached_usd":0.002232,"input_cached_usd":0.0,"output_usd":0.0008,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.509087041951716}
{"item_id":"semeval2013.d005.s020.t007","gold_sense_keys":["decision%1:09:00::"],"candidates":[{"index":1,"sense_key":"decision%1:04:00::","synset_id":"decision.n.01"},{"index":2,"sense_key":"decision%1:09:00::","synset_id":"decision.n.02"},{"index":3,"sense_key":"decision%1:11:02::","synset_id":"decision.n.03"},{"index":4,"sense_key":"decision%1:11:01::","synset_id":"decision.n.04"},{"index":5,"sense_key":"decision%1:07:00::","synset_id":"decisiveness.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"decision%1:04:00::","call_ids":["semeval2013.d005.s020.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"decision%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":983,"cached_input_tokens":0,"output_tokens":185,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003816,"input_uncached_usd":0.001966,"input_cached_usd":0.0,"output_usd":0.00185,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1607112500350922}
{"item_id":"semeval2013.d005.s020.t008","gold_sense_keys":["jury%1:14:00::"],"candidates":[{"index":1,"sense_key":"jury%1:14:00::","synset_id":"jury.n.01"},{"index":2,"sense_key":"jury%1:14:01::","synset_id":"jury.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"jury%1:14:00::","call_ids":["semeval2013.d005.s020.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"jury%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":740,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0016,"input_uncached_usd":0.00148,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.601462958031334}
{"item_id":"semeval2013.d005.s022.t000","gold_sense_keys":["argument%1:09:01::","argument%1:10:02::"],"candidates":[{"index":1,"sense_key":"argument%1:10:02::","synset_id":"argument.n.01"},{"index":2,"sense_key":"argument%1:10:03::","synset_id":"controversy.n.01"},{"index":3,"sense_key":"argument%1:10:00::","synset_id":"argument.n.03"},{"index":4,"sense_key":"argument%1:10:01::","synset_id":"argument.n.04"},{"index":5,"sense_key":"argument%1:10:04::","synset_id":"argument.n.05"},{"index":6,"sense_key":"argument%1:09:00::","synset_id":"argument.n.06"},{"index":7,"sense_key":"argument%1:09:01::","synset_id":"argumentation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"argument%1:10:02::","call_ids":["semeval2013.d005.s022.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"argument%1:10:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1128,"cached_input_tokens":0,"output_tokens":214,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004396,"input_uncached_usd":0.0022559999999999998,"input_cached_usd":0.0,"output_usd":0.00214,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.3356408750405535}
{"item_id":"semeval2013.d005.s022.t001","gold_sense_keys":["effect%1:19:00::"],"candidates":[{"index":1,"sense_key":"effect%1:19:00::","synset_id":"consequence.n.01"},{"index":2,"sense_key":"effect%1:07:00::","synset_id":"impression.n.02"},{"index":3,"sense_key":"effect%1:09:00::","synset_id":"effect.n.03"},{"index":4,"sense_key":"effect%1:10:00::","synset_id":"effect.n.04"},{"index":5,"sense_key":"effect%1:07:01::","synset_id":"effect.n.05"},{"index":6,"sense_key":"effect%1:26:00::","synset_id":"effect.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"effect%1:19:00::","call_ids":["semeval2013.d005.s022.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"effect%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":983,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0020859999999999997,"input_uncached_usd":0.001966,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0708299580728635}
{"item_id":"semeval2013.d005.s022.t002","gold_sense_keys":["coverage%1:10:00::"],"candidates":[{"index":1,"sense_key":"coverage%1:21:00::","synset_id":"coverage.n.01"},{"index":2,"sense_key":"coverage%1:07:00::","synset_id":"coverage.n.02"},{"index":3,"sense_key":"coverage%1:10:00::","synset_id":"coverage.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"coverage%1:10:00::","call_ids":["semeval2013.d005.s022.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"coverage%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":783,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001686,"input_uncached_usd":0.001566,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":12.715897832997143}
{"item_id":"semeval2013.d005.s022.t003","gold_sense_keys":["trial%1:04:02::"],"candidates":[{"index":1,"sense_key":"trial%1:04:00::","synset_id":"test.n.05"},{"index":2,"sense_key":"trial%1:09:00::","synset_id":"trial.n.02"},{"index":3,"sense_key":"trial%1:04:03::","synset_id":"test.n.04"},{"index":4,"sense_key":"trial%1:04:02::","synset_id":"trial.n.04"},{"index":5,"sense_key":"trial%1:11:01::","synset_id":"trial.n.05"},{"index":6,"sense_key":"trial%1:11:00::","synset_id":"trial.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"trial%1:04:02::","call_ids":["semeval2013.d005.s022.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"trial%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1042,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002204,"input_uncached_usd":0.002084,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":4.7152987089939415}
{"item_id":"semeval2013.d005.s022.t004","gold_sense_keys":["interest%1:09:00::"],"candidates":[{"index":1,"sense_key":"interest%1:09:00::","synset_id":"interest.n.01"},{"index":2,"sense_key":"interest%1:07:01::","synset_id":"sake.n.01"},{"index":3,"sense_key":"interest%1:07:02::","synset_id":"interest.n.03"},{"index":4,"sense_key":"interest%1:21:00::","synset_id":"interest.n.04"},{"index":5,"sense_key":"interest%1:21:03::","synset_id":"interest.n.05"},{"index":6,"sense_key":"interest%1:14:00::","synset_id":"interest.n.06"},{"index":7,"sense_key":"interest%1:04:01::","synset_id":"pastime.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"interest%1:09:00::","call_ids":["semeval2013.d005.s022.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"interest%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1098,"cached_input_tokens":0,"output_tokens":147,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003666,"input_uncached_usd":0.002196,"input_cached_usd":0.0,"output_usd":0.0014700000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.987857791944407}
{"item_id":"semeval2013.d005.s023.t000","gold_sense_keys":["challenge%1:10:01::"],"candidates":[{"index":1,"sense_key":"challenge%1:26:00::","synset_id":"challenge.n.01"},{"index":2,"sense_key":"challenge%1:10:00::","synset_id":"challenge.n.02"},{"index":3,"sense_key":"challenge%1:10:01::","synset_id":"challenge.n.03"},{"index":4,"sense_key":"challenge%1:10:02::","synset_id":"challenge.n.04"},{"index":5,"sense_key":"challenge%1:10:03::","synset_id":"challenge.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"challenge%1:10:01::","call_ids":["semeval2013.d005.s023.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"challenge%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":813,"cached_input_tokens":0,"output_tokens":117,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002796,"input_uncached_usd":0.0016259999999999998,"input_cached_usd":0.0,"output_usd":0.00117,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.111039042007178}
{"item_id":"semeval2013.d005.s023.t002","gold_sense_keys":["month%1:28:01::"],"candidates":[{"index":1,"sense_key":"month%1:28:01::","synset_id":"calendar_month.n.01"},{"index":2,"sense_key":"month%1:28:00::","synset_id":"month.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"month%1:28:01::","call_ids":["semeval2013.d005.s023.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"month%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":686,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002022,"input_uncached_usd":0.001372,"input_cached_usd":0.0,"output_usd":0.0006500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2362077089492232}
{"item_id":"semeval2013.d005.s023.t003","gold_sense_keys":["case%1:04:00::"],"candidates":[{"index":1,"sense_key":"case%1:11:00::","synset_id":"case.n.01"},{"index":2,"sense_key":"case%1:26:00::","synset_id":"event.n.02"},{"index":3,"sense_key":"case%1:04:00::","synset_id":"lawsuit.n.01"},{"index":4,"sense_key":"case%1:09:00::","synset_id":"case.n.04"},{"index":5,"sense_key":"case%1:06:00::","synset_id":"case.n.05"},{"index":6,"sense_key":"case%1:18:03::","synset_id":"case.n.06"},{"index":7,"sense_key":"case%1:18:02::","synset_id":"subject.n.06"},{"index":8,"sense_key":"case%1:10:02::","synset_id":"case.n.08"},{"index":9,"sense_key":"case%1:10:01::","synset_id":"case.n.09"},{"index":10,"sense_key":"case%1:23:00::","synset_id":"case.n.10"},{"index":11,"sense_key":"case%1:10:00::","synset_id":"case.n.11"},{"index":12,"sense_key":"case%1:26:02::","synset_id":"case.n.12"},{"index":13,"sense_key":"case%1:18:04::","synset_id":"character.n.05"},{"index":14,"sense_key":"case%1:10:03::","synset_id":"font.n.01"},{"index":15,"sense_key":"case%1:08:00::","synset_id":"sheath.n.02"},{"index":16,"sense_key":"case%1:06:06::","synset_id":"shell.n.08"},{"index":17,"sense_key":"case%1:06:07::","synset_id":"casing.n.03"},{"index":18,"sense_key":"case%1:06:03::","synset_id":"case.n.18"},{"index":19,"sense_key":"case%1:06:02::","synset_id":"case.n.19"},{"index":20,"sense_key":"case%1:06:01::","synset_id":"case.n.20"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"case%1:04:00::","call_ids":["semeval2013.d005.s023.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"case%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1834,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0037879999999999997,"input_uncached_usd":0.003668,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.7329462500056252}
{"item_id":"semeval2013.d005.s023.t004","gold_sense_keys":["news%1:10:01::","news%1:10:02::"],"candidates":[{"index":1,"sense_key":"news%1:10:00::","synset_id":"news.n.01"},{"index":2,"sense_key":"news%1:10:01::","synset_id":"news.n.02"},{"index":3,"sense_key":"news%1:10:02::","synset_id":"news_program.n.01"},{"index":4,"sense_key":"news%1:10:03::","synset_id":"news.n.04"},{"index":5,"sense_key":"news%1:07:00::","synset_id":"newsworthiness.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"news%1:10:00::","call_ids":["semeval2013.d005.s023.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"news%1:10:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":895,"cached_input_tokens":0,"output_tokens":156,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00335,"input_uncached_usd":0.00179,"input_cached_usd":0.0,"output_usd":0.0015600000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1344162499299273}
{"item_id":"semeval2013.d005.s023.t005","gold_sense_keys":["connecticut%1:15:00::"],"candidates":[{"index":1,"sense_key":"connecticut%1:15:00::","synset_id":"connecticut.n.01"},{"index":2,"sense_key":"connecticut%1:17:00::","synset_id":"connecticut.n.02"},{"index":3,"sense_key":"connecticut%1:15:01::","synset_id":"connecticut.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"connecticut%1:15:00::","call_ids":["semeval2013.d005.s023.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"connecticut%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":785,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0023,"input_uncached_usd":0.00157,"input_cached_usd":0.0,"output_usd":0.0007300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5077304159058258}
{"item_id":"semeval2013.d005.s023.t006","gold_sense_keys":["election%1:04:01::"],"candidates":[{"index":1,"sense_key":"election%1:04:01::","synset_id":"election.n.01"},{"index":2,"sense_key":"election%1:04:00::","synset_id":"election.n.02"},{"index":3,"sense_key":"election%1:26:00::","synset_id":"election.n.03"},{"index":4,"sense_key":"election%1:09:00::","synset_id":"election.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"election%1:04:01::","call_ids":["semeval2013.d005.s023.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"election%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":782,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001684,"input_uncached_usd":0.0015639999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.7752152920002118}
{"item_id":"semeval2013.d005.s023.t007","gold_sense_keys":["campaign%1:11:00::"],"candidates":[{"index":1,"sense_key":"campaign%1:11:00::","synset_id":"political_campaign.n.01"},{"index":2,"sense_key":"campaign%1:04:02::","synset_id":"campaign.n.02"},{"index":3,"sense_key":"campaign%1:04:00::","synset_id":"campaign.n.03"},{"index":4,"sense_key":"campaign%1:04:01::","synset_id":"campaign.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"campaign%1:11:00::","call_ids":["semeval2013.d005.s023.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"campaign%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":863,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001846,"input_uncached_usd":0.0017259999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1513020839774981}
{"item_id":"semeval2013.d005.s023.t008","gold_sense_keys":["candidate%1:18:01::"],"candidates":[{"index":1,"sense_key":"candidate%1:18:01::","synset_id":"campaigner.n.01"},{"index":2,"sense_key":"candidate%1:18:00::","synset_id":"candidate.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"candidate%1:18:01::","call_ids":["semeval2013.d005.s023.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"candidate%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":684,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001488,"input_uncached_usd":0.0013679999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3930122079327703}
{"item_id":"semeval2013.d005.s024.t002","gold_sense_keys":["case%1:04:00::"],"candidates":[{"index":1,"sense_key":"case%1:11:00::","synset_id":"case.n.01"},{"index":2,"sense_key":"case%1:26:00::","synset_id":"event.n.02"},{"index":3,"sense_key":"case%1:04:00::","synset_id":"lawsuit.n.01"},{"index":4,"sense_key":"case%1:09:00::","synset_id":"case.n.04"},{"index":5,"sense_key":"case%1:06:00::","synset_id":"case.n.05"},{"index":6,"sense_key":"case%1:18:03::","synset_id":"case.n.06"},{"index":7,"sense_key":"case%1:18:02::","synset_id":"subject.n.06"},{"index":8,"sense_key":"case%1:10:02::","synset_id":"case.n.08"},{"index":9,"sense_key":"case%1:10:01::","synset_id":"case.n.09"},{"index":10,"sense_key":"case%1:23:00::","synset_id":"case.n.10"},{"index":11,"sense_key":"case%1:10:00::","synset_id":"case.n.11"},{"index":12,"sense_key":"case%1:26:02::","synset_id":"case.n.12"},{"index":13,"sense_key":"case%1:18:04::","synset_id":"character.n.05"},{"index":14,"sense_key":"case%1:10:03::","synset_id":"font.n.01"},{"index":15,"sense_key":"case%1:08:00::","synset_id":"sheath.n.02"},{"index":16,"sense_key":"case%1:06:06::","synset_id":"shell.n.08"},{"index":17,"sense_key":"case%1:06:07::","synset_id":"casing.n.03"},{"index":18,"sense_key":"case%1:06:03::","synset_id":"case.n.18"},{"index":19,"sense_key":"case%1:06:02::","synset_id":"case.n.19"},{"index":20,"sense_key":"case%1:06:01::","synset_id":"case.n.20"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"case%1:04:00::","call_ids":["semeval2013.d005.s024.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"case%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1819,"cached_input_tokens":0,"output_tokens":116,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004798,"input_uncached_usd":0.0036379999999999997,"input_cached_usd":0.0,"output_usd":0.00116,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9123561249580234}
{"item_id":"semeval2013.d005.s025.t003","gold_sense_keys":["courthouse%1:06:00::"],"candidates":[{"index":1,"sense_key":"courthouse%1:06:01::","synset_id":"courthouse.n.01"},{"index":2,"sense_key":"courthouse%1:06:00::","synset_id":"courthouse.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"courthouse%1:06:00::","call_ids":["semeval2013.d005.s025.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"courthouse%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":596,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001312,"input_uncached_usd":0.001192,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9787668749922886}
{"item_id":"semeval2013.d005.s026.t000","gold_sense_keys":["focus%1:09:00::"],"candidates":[{"index":1,"sense_key":"focus%1:09:00::","synset_id":"focus.n.01"},{"index":2,"sense_key":"focus%1:07:01::","synset_id":"focus.n.02"},{"index":3,"sense_key":"focus%1:07:03::","synset_id":"focus.n.03"},{"index":4,"sense_key":"focus%1:26:00::","synset_id":"focus.n.04"},{"index":5,"sense_key":"focus%1:26:01::","synset_id":"stress.n.03"},{"index":6,"sense_key":"focus%1:19:00::","synset_id":"focus.n.06"},{"index":7,"sense_key":"focus%1:15:00::","synset_id":"focus.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"focus%1:09:00::","call_ids":["semeval2013.d005.s026.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"focus%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":899,"cached_input_tokens":0,"output_tokens":205,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003848,"input_uncached_usd":0.001798,"input_cached_usd":0.0,"output_usd":0.00205,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.135564250056632}
{"item_id":"semeval2013.d005.s026.t001","gold_sense_keys":["coverage%1:10:00::"],"candidates":[{"index":1,"sense_key":"coverage%1:21:00::","synset_id":"coverage.n.01"},{"index":2,"sense_key":"coverage%1:07:00::","synset_id":"coverage.n.02"},{"index":3,"sense_key":"coverage%1:10:00::","synset_id":"coverage.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"coverage%1:10:00::","call_ids":["semeval2013.d005.s026.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"coverage%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":664,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001448,"input_uncached_usd":0.001328,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.811498458031565}
{"item_id":"semeval2013.d005.s027.t000","gold_sense_keys":["interview%1:10:01::"],"candidates":[{"index":1,"sense_key":"interview%1:10:01::","synset_id":"interview.n.01"},{"index":2,"sense_key":"interview%1:10:00::","synset_id":"consultation.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"interview%1:10:01::","call_ids":["semeval2013.d005.s027.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"interview%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":661,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0014420000000000001,"input_uncached_usd":0.001322,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":14.18633666599635}
{"item_id":"semeval2013.d005.s027.t001","gold_sense_keys":["journalist%1:18:00::"],"candidates":[{"index":1,"sense_key":"journalist%1:18:00::","synset_id":"journalist.n.01"},{"index":2,"sense_key":"journalist%1:18:01::","synset_id":"diarist.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"journalist%1:18:00::","call_ids":["semeval2013.d005.s027.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"journalist%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":601,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001322,"input_uncached_usd":0.001202,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4304267090046778}
{"item_id":"semeval2013.d005.s027.t002","gold_sense_keys":["claim%1:10:02::"],"candidates":[{"index":1,"sense_key":"claim%1:10:00::","synset_id":"claim.n.01"},{"index":2,"sense_key":"claim%1:10:02::","synset_id":"claim.n.02"},{"index":3,"sense_key":"claim%1:04:00::","synset_id":"claim.n.03"},{"index":4,"sense_key":"claim%1:07:02::","synset_id":"claim.n.04"},{"index":5,"sense_key":"claim%1:07:01::","synset_id":"title.n.07"},{"index":6,"sense_key":"claim%1:10:01::","synset_id":"call.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"claim%1:10:02::","call_ids":["semeval2013.d005.s027.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"claim%1:10:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":841,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001802,"input_uncached_usd":0.001682,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9332160409539938}
{"item_id":"semeval2013.d005.s027.t003","gold_sense_keys":["news%1:10:00::","news%1:10:01::","news%1:10:03::"],"candidates":[{"index":1,"sense_key":"news%1:10:00::","synset_id":"news.n.01"},{"index":2,"sense_key":"news%1:10:01::","synset_id":"news.n.02"},{"index":3,"sense_key":"news%1:10:02::","synset_id":"news_program.n.01"},{"index":4,"sense_key":"news%1:10:03::","synset_id":"news.n.04"},{"index":5,"sense_key":"news%1:07:00::","synset_id":"newsworthiness.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"news%1:10:00::","call_ids":["semeval2013.d005.s027.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"news%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":827,"cached_input_tokens":0,"output_tokens":118,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0028339999999999997,"input_uncached_usd":0.0016539999999999999,"input_cached_usd":0.0,"output_usd":0.00118,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8084139580605552}
{"item_id":"semeval2013.d005.s027.t004","gold_sense_keys":["report%1:10:00::"],"candidates":[{"index":1,"sense_key":"report%1:10:03::","synset_id":"report.n.01"},{"index":2,"sense_key":"report%1:10:01::","synset_id":"report.n.02"},{"index":3,"sense_key":"report%1:10:00::","synset_id":"report.n.03"},{"index":4,"sense_key":"report%1:11:00::","synset_id":"report.n.04"},{"index":5,"sense_key":"report%1:10:04::","synset_id":"report_card.n.01"},{"index":6,"sense_key":"report%1:10:02::","synset_id":"composition.n.08"},{"index":7,"sense_key":"report%1:09:00::","synset_id":"reputation.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"report%1:10:00::","call_ids":["semeval2013.d005.s027.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"report%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":962,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0025139999999999997,"input_uncached_usd":0.001924,"input_cached_usd":0.0,"output_usd":0.00059,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":14.600775208091363}
{"item_id":"semeval2013.d005.s027.t005","gold_sense_keys":["view%1:09:02::","view%1:09:04::","view%1:10:00::"],"candidates":[{"index":1,"sense_key":"view%1:09:02::","synset_id":"position.n.03"},{"index":2,"sense_key":"view%1:09:01::","synset_id":"view.n.02"},{"index":3,"sense_key":"view%1:04:00::","synset_id":"view.n.03"},{"index":4,"sense_key":"view%1:15:00::","synset_id":"view.n.04"},{"index":5,"sense_key":"view%1:09:04::","synset_id":"opinion.n.01"},{"index":6,"sense_key":"view%1:10:00::","synset_id":"opinion.n.02"},{"index":7,"sense_key":"view%1:09:00::","synset_id":"view.n.07"},{"index":8,"sense_key":"view%1:06:00::","synset_id":"scene.n.08"},{"index":9,"sense_key":"view%1:07:00::","synset_id":"horizon.n.02"},{"index":10,"sense_key":"view%1:07:01::","synset_id":"view.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"view%1:09:02::","call_ids":["semeval2013.d005.s027.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"view%1:09:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1207,"cached_input_tokens":0,"output_tokens":133,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003744,"input_uncached_usd":0.002414,"input_cached_usd":0.0,"output_usd":0.00133,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":14.876002624980174}
{"item_id":"semeval2013.d005.s027.t006","gold_sense_keys":["court%1:14:00::"],"candidates":[{"index":1,"sense_key":"court%1:14:00::","synset_id":"court.n.01"},{"index":2,"sense_key":"court%1:06:02::","synset_id":"court.n.02"},{"index":3,"sense_key":"court%1:14:01::","synset_id":"court.n.03"},{"index":4,"sense_key":"court%1:06:01::","synset_id":"court.n.04"},{"index":5,"sense_key":"court%1:18:00::","synset_id":"court.n.05"},{"index":6,"sense_key":"court%1:14:02::","synset_id":"court.n.06"},{"index":7,"sense_key":"court%1:06:03::","synset_id":"motor_hotel.n.01"},{"index":8,"sense_key":"court%1:06:05::","synset_id":"court.n.08"},{"index":9,"sense_key":"court%1:06:04::","synset_id":"court.n.09"},{"index":10,"sense_key":"court%1:06:00::","synset_id":"court.n.10"},{"index":11,"sense_key":"court%1:04:00::","synset_id":"court.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"court%1:14:00::","call_ids":["semeval2013.d005.s027.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"court%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1131,"cached_input_tokens":0,"output_tokens":165,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003912,"input_uncached_usd":0.0022619999999999997,"input_cached_usd":0.0,"output_usd":0.0016500000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9613761670188978}
{"item_id":"semeval2013.d005.s027.t007","gold_sense_keys":["policy%1:09:00::"],"candidates":[{"index":1,"sense_key":"policy%1:09:00::","synset_id":"policy.n.01"},{"index":2,"sense_key":"policy%1:10:00::","synset_id":"policy.n.02"},{"index":3,"sense_key":"policy%1:10:02::","synset_id":"policy.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"policy%1:09:00::","call_ids":["semeval2013.d005.s027.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"policy%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":701,"cached_input_tokens":0,"output_tokens":113,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0025320000000000004,"input_uncached_usd":0.001402,"input_cached_usd":0.0,"output_usd":0.0011300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6710184999974445}
{"item_id":"semeval2013.d005.s027.t008","gold_sense_keys":["coverage%1:10:00::"],"candidates":[{"index":1,"sense_key":"coverage%1:21:00::","synset_id":"coverage.n.01"},{"index":2,"sense_key":"coverage%1:07:00::","synset_id":"coverage.n.02"},{"index":3,"sense_key":"coverage%1:10:00::","synset_id":"coverage.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"coverage%1:10:00::","call_ids":["semeval2013.d005.s027.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"coverage%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":691,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001502,"input_uncached_usd":0.001382,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7704044160200283}
{"item_id":"semeval2013.d005.s027.t009","gold_sense_keys":["trial%1:04:02::"],"candidates":[{"index":1,"sense_key":"trial%1:04:00::","synset_id":"test.n.05"},{"index":2,"sense_key":"trial%1:09:00::","synset_id":"trial.n.02"},{"index":3,"sense_key":"trial%1:04:03::","synset_id":"test.n.04"},{"index":4,"sense_key":"trial%1:04:02::","synset_id":"trial.n.04"},{"index":5,"sense_key":"trial%1:11:01::","synset_id":"trial.n.05"},{"index":6,"sense_key":"trial%1:11:00::","synset_id":"trial.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"trial%1:04:02::","call_ids":["semeval2013.d005.s027.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"trial%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":950,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00202,"input_uncached_usd":0.0019,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1265455420361832}
{"item_id":"semeval2013.d005.s028.t002","gold_sense_keys":["argument%1:09:01::","argument%1:10:02::"],"candidates":[{"index":1,"sense_key":"argument%1:10:02::","synset_id":"argument.n.01"},{"index":2,"sense_key":"argument%1:10:03::","synset_id":"controversy.n.01"},{"index":3,"sense_key":"argument%1:10:00::","synset_id":"argument.n.03"},{"index":4,"sense_key":"argument%1:10:01::","synset_id":"argument.n.04"},{"index":5,"sense_key":"argument%1:10:04::","synset_id":"argument.n.05"},{"index":6,"sense_key":"argument%1:09:00::","synset_id":"argument.n.06"},{"index":7,"sense_key":"argument%1:09:01::","synset_id":"argumentation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"argument%1:10:02::","call_ids":["semeval2013.d005.s028.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"argument%1:10:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1029,"cached_input_tokens":0,"output_tokens":195,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004008,"input_uncached_usd":0.002058,"input_cached_usd":0.0,"output_usd":0.0019500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.7815238329349086}
{"item_id":"semeval2013.d005.s028.t003","gold_sense_keys":["change%1:04:00::","change%1:11:00::","change%1:24:00::"],"candidates":[{"index":1,"sense_key":"change%1:11:00::","synset_id":"change.n.01"},{"index":2,"sense_key":"change%1:24:00::","synset_id":"change.n.02"},{"index":3,"sense_key":"change%1:04:00::","synset_id":"change.n.03"},{"index":4,"sense_key":"change%1:19:00::","synset_id":"change.n.04"},{"index":5,"sense_key":"change%1:21:02::","synset_id":"change.n.05"},{"index":6,"sense_key":"change%1:06:01::","synset_id":"change.n.06"},{"index":7,"sense_key":"change%1:06:00::","synset_id":"change.n.07"},{"index":8,"sense_key":"change%1:21:01::","synset_id":"change.n.08"},{"index":9,"sense_key":"change%1:21:03::","synset_id":"change.n.09"},{"index":10,"sense_key":"change%1:07:00::","synset_id":"variety.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"change%1:11:00::","call_ids":["semeval2013.d005.s028.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"change%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1090,"cached_input_tokens":0,"output_tokens":96,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0031400000000000004,"input_uncached_usd":0.00218,"input_cached_usd":0.0,"output_usd":0.0009600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.560860416968353}
{"item_id":"semeval2013.d005.s028.t004","gold_sense_keys":["adaptation%1:22:00::"],"candidates":[{"index":1,"sense_key":"adaptation%1:10:00::","synset_id":"adaptation.n.01"},{"index":2,"sense_key":"adaptation%1:22:00::","synset_id":"adaptation.n.02"},{"index":3,"sense_key":"adaptation%1:04:01::","synset_id":"adaptation.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"adaptation%1:22:00::","call_ids":["semeval2013.d005.s028.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"adaptation%1:22:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":693,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001506,"input_uncached_usd":0.0013859999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.159500833018683}
{"item_id":"semeval2013.d005.s028.t005","gold_sense_keys":["court%1:14:00::"],"candidates":[{"index":1,"sense_key":"court%1:14:00::","synset_id":"court.n.01"},{"index":2,"sense_key":"court%1:06:02::","synset_id":"court.n.02"},{"index":3,"sense_key":"court%1:14:01::","synset_id":"court.n.03"},{"index":4,"sense_key":"court%1:06:01::","synset_id":"court.n.04"},{"index":5,"sense_key":"court%1:18:00::","synset_id":"court.n.05"},{"index":6,"sense_key":"court%1:14:02::","synset_id":"court.n.06"},{"index":7,"sense_key":"court%1:06:03::","synset_id":"motor_hotel.n.01"},{"index":8,"sense_key":"court%1:06:05::","synset_id":"court.n.08"},{"index":9,"sense_key":"court%1:06:04::","synset_id":"court.n.09"},{"index":10,"sense_key":"court%1:06:00::","synset_id":"court.n.10"},{"index":11,"sense_key":"court%1:04:00::","synset_id":"court.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"court%1:14:00::","call_ids":["semeval2013.d005.s028.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"court%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1125,"cached_input_tokens":0,"output_tokens":192,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00417,"input_uncached_usd":0.00225,"input_cached_usd":0.0,"output_usd":0.0019200000000000003,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.792170249973424}
{"item_id":"semeval2013.d005.s029.t001","gold_sense_keys":["claim%1:10:02::"],"candidates":[{"index":1,"sense_key":"claim%1:10:00::","synset_id":"claim.n.01"},{"index":2,"sense_key":"claim%1:10:02::","synset_id":"claim.n.02"},{"index":3,"sense_key":"claim%1:04:00::","synset_id":"claim.n.03"},{"index":4,"sense_key":"claim%1:07:02::","synset_id":"claim.n.04"},{"index":5,"sense_key":"claim%1:07:01::","synset_id":"title.n.07"},{"index":6,"sense_key":"claim%1:10:01::","synset_id":"call.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"claim%1:10:02::","call_ids":["semeval2013.d005.s029.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"claim%1:10:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":842,"cached_input_tokens":0,"output_tokens":99,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002674,"input_uncached_usd":0.001684,"input_cached_usd":0.0,"output_usd":0.00099,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.95916304201819}
{"item_id":"semeval2013.d005.s030.t000","gold_sense_keys":["filing%1:10:00::"],"candidates":[{"index":1,"sense_key":"filing%1:10:00::","synset_id":"filing.n.01"},{"index":2,"sense_key":"filing%1:17:00::","synset_id":"filing.n.02"},{"index":3,"sense_key":"filing%1:04:01::","synset_id":"filing.n.03"},{"index":4,"sense_key":"filing%1:04:00::","synset_id":"filing.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"filing%1:10:00::","call_ids":["semeval2013.d005.s030.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"filing%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":710,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0015400000000000001,"input_uncached_usd":0.00142,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5159676659386605}
{"item_id":"semeval2013.d005.s030.t001","gold_sense_keys":["supreme_court%1:14:00::"],"candidates":[{"index":1,"sense_key":"supreme_court%1:14:00::","synset_id":"supreme_court.n.01"},{"index":2,"sense_key":"supreme_court%1:14:01::","synset_id":"supreme_court.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"supreme_court%1:14:00::","call_ids":["semeval2013.d005.s030.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"supreme_court%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":671,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002002,"input_uncached_usd":0.0013419999999999999,"input_cached_usd":0.0,"output_usd":0.0006600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4570612089009956}
{"item_id":"semeval2013.d005.s030.t002","gold_sense_keys":["decision%1:09:00::"],"candidates":[{"index":1,"sense_key":"decision%1:04:00::","synset_id":"decision.n.01"},{"index":2,"sense_key":"decision%1:09:00::","synset_id":"decision.n.02"},{"index":3,"sense_key":"decision%1:11:02::","synset_id":"decision.n.03"},{"index":4,"sense_key":"decision%1:11:01::","synset_id":"decision.n.04"},{"index":5,"sense_key":"decision%1:07:00::","synset_id":"decisiveness.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"decision%1:09:00::","call_ids":["semeval2013.d005.s030.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"decision%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":860,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00226,"input_uncached_usd":0.00172,"input_cached_usd":0.0,"output_usd":0.00054,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":14.153285749955103}
{"item_id":"semeval2013.d005.s030.t003","gold_sense_keys":["conviction%1:04:00::"],"candidates":[{"index":1,"sense_key":"conviction%1:09:00::","synset_id":"conviction.n.01"},{"index":2,"sense_key":"conviction%1:04:00::","synset_id":"conviction.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"conviction%1:04:00::","call_ids":["semeval2013.d005.s030.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"conviction%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":672,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001464,"input_uncached_usd":0.001344,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1638043749844655}
{"item_id":"semeval2013.d005.s030.t004","gold_sense_keys":["trial%1:04:02::"],"candidates":[{"index":1,"sense_key":"trial%1:04:00::","synset_id":"test.n.05"},{"index":2,"sense_key":"trial%1:09:00::","synset_id":"trial.n.02"},{"index":3,"sense_key":"trial%1:04:03::","synset_id":"test.n.04"},{"index":4,"sense_key":"trial%1:04:02::","synset_id":"trial.n.04"},{"index":5,"sense_key":"trial%1:11:01::","synset_id":"trial.n.05"},{"index":6,"sense_key":"trial%1:11:00::","synset_id":"trial.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"trial%1:04:02::","call_ids":["semeval2013.d005.s030.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"trial%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":960,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0020399999999999997,"input_uncached_usd":0.0019199999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1522555829724297}
{"item_id":"semeval2013.d005.s030.t005","gold_sense_keys":["coverage%1:10:00::"],"candidates":[{"index":1,"sense_key":"coverage%1:21:00::","synset_id":"coverage.n.01"},{"index":2,"sense_key":"coverage%1:07:00::","synset_id":"coverage.n.02"},{"index":3,"sense_key":"coverage%1:10:00::","synset_id":"coverage.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"coverage%1:10:00::","call_ids":["semeval2013.d005.s030.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"coverage%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":701,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0015220000000000001,"input_uncached_usd":0.001402,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0537369579542428}
{"item_id":"semeval2013.d005.s030.t006","gold_sense_keys":["rule%1:10:00::"],"candidates":[{"index":1,"sense_key":"rule%1:09:00::","synset_id":"rule.n.01"},{"index":2,"sense_key":"rule%1:09:01::","synset_id":"convention.n.02"},{"index":3,"sense_key":"rule%1:10:00::","synset_id":"rule.n.03"},{"index":4,"sense_key":"rule%1:10:01::","synset_id":"rule.n.04"},{"index":5,"sense_key":"rule%1:09:03::","synset_id":"principle.n.01"},{"index":6,"sense_key":"rule%1:28:00::","synset_id":"rule.n.06"},{"index":7,"sense_key":"rule%1:26:00::","synset_id":"dominion.n.01"},{"index":8,"sense_key":"rule%1:10:02::","synset_id":"rule.n.08"},{"index":9,"sense_key":"rule%1:10:03::","synset_id":"rule.n.09"},{"index":10,"sense_key":"rule%1:09:02::","synset_id":"principle.n.04"},{"index":11,"sense_key":"rule%1:09:04::","synset_id":"rule.n.11"},{"index":12,"sense_key":"rule%1:06:00::","synset_id":"rule.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"rule%1:10:00::","call_ids":["semeval2013.d005.s030.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"rule%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1339,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003288,"input_uncached_usd":0.002678,"input_cached_usd":0.0,"output_usd":0.0006100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4353468749905005}
{"item_id":"semeval2013.d005.s031.t000","gold_sense_keys":["case%1:04:00::"],"candidates":[{"index":1,"sense_key":"case%1:11:00::","synset_id":"case.n.01"},{"index":2,"sense_key":"case%1:26:00::","synset_id":"event.n.02"},{"index":3,"sense_key":"case%1:04:00::","synset_id":"lawsuit.n.01"},{"index":4,"sense_key":"case%1:09:00::","synset_id":"case.n.04"},{"index":5,"sense_key":"case%1:06:00::","synset_id":"case.n.05"},{"index":6,"sense_key":"case%1:18:03::","synset_id":"case.n.06"},{"index":7,"sense_key":"case%1:18:02::","synset_id":"subject.n.06"},{"index":8,"sense_key":"case%1:10:02::","synset_id":"case.n.08"},{"index":9,"sense_key":"case%1:10:01::","synset_id":"case.n.09"},{"index":10,"sense_key":"case%1:23:00::","synset_id":"case.n.10"},{"index":11,"sense_key":"case%1:10:00::","synset_id":"case.n.11"},{"index":12,"sense_key":"case%1:26:02::","synset_id":"case.n.12"},{"index":13,"sense_key":"case%1:18:04::","synset_id":"character.n.05"},{"index":14,"sense_key":"case%1:10:03::","synset_id":"font.n.01"},{"index":15,"sense_key":"case%1:08:00::","synset_id":"sheath.n.02"},{"index":16,"sense_key":"case%1:06:06::","synset_id":"shell.n.08"},{"index":17,"sense_key":"case%1:06:07::","synset_id":"casing.n.03"},{"index":18,"sense_key":"case%1:06:03::","synset_id":"case.n.18"},{"index":19,"sense_key":"case%1:06:02::","synset_id":"case.n.19"},{"index":20,"sense_key":"case%1:06:01::","synset_id":"case.n.20"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"case%1:04:00::","call_ids":["semeval2013.d005.s031.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"case%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1715,"cached_input_tokens":0,"output_tokens":69,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00412,"input_uncached_usd":0.00343,"input_cached_usd":0.0,"output_usd":0.0006900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5962003329768777}
{"item_id":"semeval2013.d005.s031.t001","gold_sense_keys":["century%1:28:00::"],"candidates":[{"index":1,"sense_key":"century%1:28:00::","synset_id":"century.n.01"},{"index":2,"sense_key":"century%1:23:00::","synset_id":"hundred.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"century%1:28:00::","call_ids":["semeval2013.d005.s031.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"century%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":532,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0011840000000000002,"input_uncached_usd":0.001064,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9807734589558095}
{"item_id":"semeval2013.d006.s000.t001","gold_sense_keys":["fraud%1:04:01::"],"candidates":[{"index":1,"sense_key":"fraud%1:04:00::","synset_id":"fraud.n.01"},{"index":2,"sense_key":"fraud%1:18:00::","synset_id":"imposter.n.01"},{"index":3,"sense_key":"fraud%1:04:01::","synset_id":"fraud.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"fraud%1:04:00::","call_ids":["semeval2013.d006.s000.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"fraud%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":460,"cached_input_tokens":0,"output_tokens":70,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00162,"input_uncached_usd":0.0009199999999999999,"input_cached_usd":0.0,"output_usd":0.0007000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8021201670635492}
{"item_id":"semeval2013.d006.s000.t002","gold_sense_keys":["law%1:14:00::"],"candidates":[{"index":1,"sense_key":"law%1:14:00::","synset_id":"law.n.01"},{"index":2,"sense_key":"law%1:10:00::","synset_id":"law.n.02"},{"index":3,"sense_key":"law%1:09:00::","synset_id":"law.n.03"},{"index":4,"sense_key":"law%1:09:01::","synset_id":"law.n.04"},{"index":5,"sense_key":"law%1:09:02::","synset_id":"jurisprudence.n.01"},{"index":6,"sense_key":"law%1:04:00::","synset_id":"law.n.06"},{"index":7,"sense_key":"law%1:14:01::","synset_id":"police.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"law%1:14:00::","call_ids":["semeval2013.d006.s000.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"law%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":745,"cached_input_tokens":0,"output_tokens":123,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00272,"input_uncached_usd":0.00149,"input_cached_usd":0.0,"output_usd":0.0012300000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9538317079422995}
{"item_id":"semeval2013.d006.s001.t000","gold_sense_keys":["trick%1:04:05::"],"candidates":[{"index":1,"sense_key":"trick%1:04:05::","synset_id":"trick.n.01"},{"index":2,"sense_key":"trick%1:28:00::","synset_id":"trick.n.02"},{"index":3,"sense_key":"trick%1:04:04::","synset_id":"trick.n.03"},{"index":4,"sense_key":"trick%1:04:00::","synset_id":"antic.n.01"},{"index":5,"sense_key":"trick%1:04:02::","synset_id":"magic_trick.n.01"},{"index":6,"sense_key":"trick%1:18:00::","synset_id":"whoremaster.n.01"},{"index":7,"sense_key":"trick%1:04:01::","synset_id":"trick.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"trick%1:04:05::","call_ids":["semeval2013.d006.s001.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"trick%1:04:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":788,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002216,"input_uncached_usd":0.001576,"input_cached_usd":0.0,"output_usd":0.00064,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4614064580528066}
{"item_id":"semeval2013.d006.s001.t002","gold_sense_keys":["will%1:09:01::"],"candidates":[{"index":1,"sense_key":"will%1:09:00::","synset_id":"volition.n.01"},{"index":2,"sense_key":"will%1:09:01::","synset_id":"will.n.02"},{"index":3,"sense_key":"will%1:10:00::","synset_id":"will.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"will%1:09:01::","call_ids":["semeval2013.d006.s001.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"will%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":566,"cached_input_tokens":0,"output_tokens":97,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002102,"input_uncached_usd":0.001132,"input_cached_usd":0.0,"output_usd":0.00097,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.6472252500243485}
{"item_id":"semeval2013.d006.s002.t000","gold_sense_keys":["expulsion%1:04:01::"],"candidates":[{"index":1,"sense_key":"expulsion%1:04:01::","synset_id":"ejection.n.02"},{"index":2,"sense_key":"expulsion%1:04:02::","synset_id":"extrusion.n.02"},{"index":3,"sense_key":"expulsion%1:04:00::","synset_id":"expulsion.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"expulsion%1:04:01::","call_ids":["semeval2013.d006.s002.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"expulsion%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":673,"cached_input_tokens":0,"output_tokens":71,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002056,"input_uncached_usd":0.001346,"input_cached_usd":0.0,"output_usd":0.00071,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":14.527543916949071}
{"item_id":"semeval2013.d006.s002.t001","gold_sense_keys":["player%1:18:01::"],"candidates":[{"index":1,"sense_key":"player%1:18:01::","synset_id":"player.n.01"},{"index":2,"sense_key":"player%1:18:02::","synset_id":"musician.n.01"},{"index":3,"sense_key":"player%1:18:00::","synset_id":"actor.n.01"},{"index":4,"sense_key":"player%1:18:04::","synset_id":"player.n.04"},{"index":5,"sense_key":"player%1:18:03::","synset_id":"player.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"player%1:18:01::","call_ids":["semeval2013.d006.s002.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"player%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":680,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00148,"input_uncached_usd":0.0013599999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2904717919882387}
{"item_id":"semeval2013.d006.s002.t002","gold_sense_keys":["game%1:04:03::"],"candidates":[{"index":1,"sense_key":"game%1:04:00::","synset_id":"game.n.01"},{"index":2,"sense_key":"game%1:04:03::","synset_id":"game.n.02"},{"index":3,"sense_key":"game%1:04:01::","synset_id":"game.n.03"},{"index":4,"sense_key":"game%1:05:00::","synset_id":"game.n.04"},{"index":5,"sense_key":"game%1:28:00::","synset_id":"game.n.05"},{"index":6,"sense_key":"game%1:23:00::","synset_id":"game.n.06"},{"index":7,"sense_key":"game%1:13:00::","synset_id":"game.n.07"},{"index":8,"sense_key":"game%1:09:00::","synset_id":"plot.n.01"},{"index":9,"sense_key":"game%1:06:00::","synset_id":"game.n.09"},{"index":10,"sense_key":"game%1:04:02::","synset_id":"game.n.10"},{"index":11,"sense_key":"game%1:04:04::","synset_id":"game.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"game%1:04:03::","call_ids":["semeval2013.d006.s002.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"game%1:04:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":998,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0027660000000000002,"input_uncached_usd":0.001996,"input_cached_usd":0.0,"output_usd":0.0007700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5954092079773545}
{"item_id":"semeval2013.d006.s002.t003","gold_sense_keys":["ink%1:27:01::"],"candidates":[{"index":1,"sense_key":"ink%1:27:01::","synset_id":"ink.n.01"},{"index":2,"sense_key":"ink%1:08:00::","synset_id":"ink.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ink%1:27:01::","call_ids":["semeval2013.d006.s002.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ink%1:27:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":532,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0011840000000000002,"input_uncached_usd":0.001064,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1627090419642627}
{"item_id":"semeval2013.d006.s002.t004","gold_sense_keys":["behavior%1:04:00::"],"candidates":[{"index":1,"sense_key":"behavior%1:04:00::","synset_id":"behavior.n.01"},{"index":2,"sense_key":"behavior%1:26:00::","synset_id":"behavior.n.02"},{"index":3,"sense_key":"behavior%1:07:00::","synset_id":"demeanor.n.01"},{"index":4,"sense_key":"behavior%1:04:01::","synset_id":"behavior.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"behavior%1:04:00::","call_ids":["semeval2013.d006.s002.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"behavior%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":711,"cached_input_tokens":0,"output_tokens":117,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0025919999999999997,"input_uncached_usd":0.0014219999999999999,"input_cached_usd":0.0,"output_usd":0.00117,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8948356249602512}
{"item_id":"semeval2013.d006.s002.t005","gold_sense_keys":["player%1:18:01::"],"candidates":[{"index":1,"sense_key":"player%1:18:01::","synset_id":"player.n.01"},{"index":2,"sense_key":"player%1:18:02::","synset_id":"musician.n.01"},{"index":3,"sense_key":"player%1:18:00::","synset_id":"actor.n.01"},{"index":4,"sense_key":"player%1:18:04::","synset_id":"player.n.04"},{"index":5,"sense_key":"player%1:18:03::","synset_id":"player.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"player%1:18:01::","call_ids":["semeval2013.d006.s002.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"player%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":680,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00148,"input_uncached_usd":0.0013599999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.124176749959588}
{"item_id":"semeval2013.d006.s003.t000","gold_sense_keys":["minute%1:28:00::"],"candidates":[{"index":1,"sense_key":"minute%1:28:00::","synset_id":"minute.n.01"},{"index":2,"sense_key":"minute%1:28:01::","synset_id":"moment.n.02"},{"index":3,"sense_key":"minute%1:28:02::","synset_id":"moment.n.01"},{"index":4,"sense_key":"minute%1:23:00::","synset_id":"minute.n.04"},{"index":5,"sense_key":"minute%1:10:00::","synset_id":"minute.n.05"},{"index":6,"sense_key":"minute%1:07:00::","synset_id":"hour.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"minute%1:28:00::","call_ids":["semeval2013.d006.s003.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"minute%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":866,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001852,"input_uncached_usd":0.0017319999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2616464999737218}
{"item_id":"semeval2013.d006.s003.t001","gold_sense_keys":["end%1:09:00::","end%1:11:00::","end%1:28:00::"],"candidates":[{"index":1,"sense_key":"end%1:15:00::","synset_id":"end.n.01"},{"index":2,"sense_key":"end%1:28:00::","synset_id":"end.n.02"},{"index":3,"sense_key":"end%1:11:00::","synset_id":"end.n.03"},{"index":4,"sense_key":"end%1:09:02::","synset_id":"goal.n.01"},{"index":5,"sense_key":"end%1:09:00::","synset_id":"end.n.05"},{"index":6,"sense_key":"end%1:26:00::","synset_id":"end.n.06"},{"index":7,"sense_key":"end%1:15:02::","synset_id":"end.n.07"},{"index":8,"sense_key":"end%1:18:00::","synset_id":"end.n.08"},{"index":9,"sense_key":"end%1:15:01::","synset_id":"end.n.09"},{"index":10,"sense_key":"end%1:15:03::","synset_id":"end.n.10"},{"index":11,"sense_key":"end%1:04:00::","synset_id":"end.n.11"},{"index":12,"sense_key":"end%1:10:00::","synset_id":"conclusion.n.08"},{"index":13,"sense_key":"end%1:06:02::","synset_id":"end.n.13"},{"index":14,"sense_key":"end%1:04:02::","synset_id":"end.n.14"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"end%1:28:00::","call_ids":["semeval2013.d006.s003.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"end%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1320,"cached_input_tokens":0,"output_tokens":219,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00483,"input_uncached_usd":0.00264,"input_cached_usd":0.0,"output_usd":0.00219,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.1021350419614464}
{"item_id":"semeval2013.d006.s003.t002","gold_sense_keys":["game%1:04:03::"],"candidates":[{"index":1,"sense_key":"game%1:04:00::","synset_id":"game.n.01"},{"index":2,"sense_key":"game%1:04:03::","synset_id":"game.n.02"},{"index":3,"sense_key":"game%1:04:01::","synset_id":"game.n.03"},{"index":4,"sense_key":"game%1:05:00::","synset_id":"game.n.04"},{"index":5,"sense_key":"game%1:28:00::","synset_id":"game.n.05"},{"index":6,"sense_key":"game%1:23:00::","synset_id":"game.n.06"},{"index":7,"sense_key":"game%1:13:00::","synset_id":"game.n.07"},{"index":8,"sense_key":"game%1:09:00::","synset_id":"plot.n.01"},{"index":9,"sense_key":"game%1:06:00::","synset_id":"game.n.09"},{"index":10,"sense_key":"game%1:04:02::","synset_id":"game.n.10"},{"index":11,"sense_key":"game%1:04:04::","synset_id":"game.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"game%1:04:03::","call_ids":["semeval2013.d006.s003.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"game%1:04:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1065,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00277,"input_uncached_usd":0.00213,"input_cached_usd":0.0,"output_usd":0.00064,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5924450000748038}
{"item_id":"semeval2013.d006.s003.t003","gold_sense_keys":["team%1:14:00::"],"candidates":[{"index":1,"sense_key":"team%1:14:00::","synset_id":"team.n.01"},{"index":2,"sense_key":"team%1:14:01::","synset_id":"team.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"team%1:14:00::","call_ids":["semeval2013.d006.s003.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"team%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":589,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0012980000000000001,"input_uncached_usd":0.001178,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8827043749624863}
{"item_id":"semeval2013.d006.s003.t004","gold_sense_keys":["player%1:18:01::"],"candidates":[{"index":1,"sense_key":"player%1:18:01::","synset_id":"player.n.01"},{"index":2,"sense_key":"player%1:18:02::","synset_id":"musician.n.01"},{"index":3,"sense_key":"player%1:18:00::","synset_id":"actor.n.01"},{"index":4,"sense_key":"player%1:18:04::","synset_id":"player.n.04"},{"index":5,"sense_key":"player%1:18:03::","synset_id":"player.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"player%1:18:01::","call_ids":["semeval2013.d006.s003.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"player%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":747,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001614,"input_uncached_usd":0.0014939999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1016603750176728}
{"item_id":"semeval2013.d006.s003.t005","gold_sense_keys":["time%1:28:05::"],"candidates":[{"index":1,"sense_key":"time%1:11:00::","synset_id":"time.n.01"},{"index":2,"sense_key":"time%1:28:05::","synset_id":"time.n.02"},{"index":3,"sense_key":"time%1:28:00::","synset_id":"time.n.03"},{"index":4,"sense_key":"time%1:28:06::","synset_id":"time.n.04"},{"index":5,"sense_key":"time%1:03:00::","synset_id":"time.n.05"},{"index":6,"sense_key":"time%1:11:01::","synset_id":"time.n.06"},{"index":7,"sense_key":"time%1:28:03::","synset_id":"clock_time.n.01"},{"index":8,"sense_key":"time%1:28:04::","synset_id":"fourth_dimension.n.01"},{"index":9,"sense_key":"time%1:07:01::","synset_id":"meter.n.04"},{"index":10,"sense_key":"time%1:28:01::","synset_id":"prison_term.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"time%1:28:05::","call_ids":["semeval2013.d006.s003.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"time%1:28:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1082,"cached_input_tokens":0,"output_tokens":90,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003064,"input_uncached_usd":0.0021639999999999997,"input_cached_usd":0.0,"output_usd":0.0009000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0366416659671813}
{"item_id":"semeval2013.d006.s003.t006","gold_sense_keys":["referee%1:18:00::"],"candidates":[{"index":1,"sense_key":"referee%1:18:00::","synset_id":"referee.n.01"},{"index":2,"sense_key":"referee%1:18:01::","synset_id":"reviewer.n.01"},{"index":3,"sense_key":"referee%1:18:02::","synset_id":"referee.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"referee%1:18:00::","call_ids":["semeval2013.d006.s003.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"referee%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":668,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001456,"input_uncached_usd":0.001336,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1491183750331402}
{"item_id":"semeval2013.d006.s004.t000","gold_sense_keys":["way%1:04:01::","way%1:07:01::"],"candidates":[{"index":1,"sense_key":"way%1:07:01::","synset_id":"manner.n.01"},{"index":2,"sense_key":"way%1:04:01::","synset_id":"means.n.01"},{"index":3,"sense_key":"way%1:15:00::","synset_id":"direction.n.01"},{"index":4,"sense_key":"way%1:26:00::","synset_id":"way.n.04"},{"index":5,"sense_key":"way%1:04:00::","synset_id":"way.n.05"},{"index":6,"sense_key":"way%1:06:00::","synset_id":"way.n.06"},{"index":7,"sense_key":"way%1:04:02::","synset_id":"way.n.07"},{"index":8,"sense_key":"way%1:23:00::","synset_id":"room.n.02"},{"index":9,"sense_key":"way%1:07:02::","synset_id":"way.n.09"},{"index":10,"sense_key":"way%1:09:00::","synset_id":"way.n.10"},{"index":11,"sense_key":"way%1:09:01::","synset_id":"way.n.11"},{"index":12,"sense_key":"way%1:21:00::","synset_id":"way.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"way%1:07:01::","call_ids":["semeval2013.d006.s004.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"way%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1186,"cached_input_tokens":0,"output_tokens":98,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003352,"input_uncached_usd":0.002372,"input_cached_usd":0.0,"output_usd":0.0009800000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8131598749896511}
{"item_id":"semeval2013.d006.s004.t002","gold_sense_keys":["group%1:03:00::"],"candidates":[{"index":1,"sense_key":"group%1:03:00::","synset_id":"group.n.01"},{"index":2,"sense_key":"group%1:27:00::","synset_id":"group.n.02"},{"index":3,"sense_key":"group%1:09:00::","synset_id":"group.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"group%1:03:00::","call_ids":["semeval2013.d006.s004.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"group%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":729,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001578,"input_uncached_usd":0.0014579999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8177630000282079}
{"item_id":"semeval2013.d006.s004.t003","gold_sense_keys":["player%1:18:01::"],"candidates":[{"index":1,"sense_key":"player%1:18:01::","synset_id":"player.n.01"},{"index":2,"sense_key":"player%1:18:02::","synset_id":"musician.n.01"},{"index":3,"sense_key":"player%1:18:00::","synset_id":"actor.n.01"},{"index":4,"sense_key":"player%1:18:04::","synset_id":"player.n.04"},{"index":5,"sense_key":"player%1:18:03::","synset_id":"player.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"player%1:18:01::","call_ids":["semeval2013.d006.s004.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"player%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":803,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001726,"input_uncached_usd":0.001606,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6057683340040967}
{"item_id":"semeval2013.d006.s004.t005","gold_sense_keys":["game%1:04:03::"],"candidates":[{"index":1,"sense_key":"game%1:04:00::","synset_id":"game.n.01"},{"index":2,"sense_key":"game%1:04:03::","synset_id":"game.n.02"},{"index":3,"sense_key":"game%1:04:01::","synset_id":"game.n.03"},{"index":4,"sense_key":"game%1:05:00::","synset_id":"game.n.04"},{"index":5,"sense_key":"game%1:28:00::","synset_id":"game.n.05"},{"index":6,"sense_key":"game%1:23:00::","synset_id":"game.n.06"},{"index":7,"sense_key":"game%1:13:00::","synset_id":"game.n.07"},{"index":8,"sense_key":"game%1:09:00::","synset_id":"plot.n.01"},{"index":9,"sense_key":"game%1:06:00::","synset_id":"game.n.09"},{"index":10,"sense_key":"game%1:04:02::","synset_id":"game.n.10"},{"index":11,"sense_key":"game%1:04:04::","synset_id":"game.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"game%1:04:03::","call_ids":["semeval2013.d006.s004.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"game%1:04:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1121,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002862,"input_uncached_usd":0.002242,"input_cached_usd":0.0,"output_usd":0.00062,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.220201458898373}
{"item_id":"semeval2013.d006.s004.t006","gold_sense_keys":["league%1:14:01::"],"candidates":[{"index":1,"sense_key":"league%1:14:01::","synset_id":"league.n.01"},{"index":2,"sense_key":"league%1:14:00::","synset_id":"league.n.02"},{"index":3,"sense_key":"league%1:23:00::","synset_id":"league.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"league%1:14:01::","call_ids":["semeval2013.d006.s004.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"league%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":692,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0015040000000000001,"input_uncached_usd":0.001384,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0789316670270637}
{"item_id":"semeval2013.d006.s004.t008","gold_sense_keys":["phase%1:28:00::"],"candidates":[{"index":1,"sense_key":"phase%1:28:00::","synset_id":"phase.n.01"},{"index":2,"sense_key":"phase%1:26:00::","synset_id":"phase.n.02"},{"index":3,"sense_key":"phase%1:28:01::","synset_id":"phase.n.03"},{"index":4,"sense_key":"phase%1:07:00::","synset_id":"phase.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"phase%1:28:00::","call_ids":["semeval2013.d006.s004.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"phase%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":891,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002352,"input_uncached_usd":0.001782,"input_cached_usd":0.0,"output_usd":0.0005700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3907375830458477}
{"item_id":"semeval2013.d006.s005.t000","gold_sense_keys":["committee%1:14:00::"],"candidates":[{"index":1,"sense_key":"committee%1:14:00::","synset_id":"committee.n.01"},{"index":2,"sense_key":"committee%1:14:01::","synset_id":"committee.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"committee%1:14:00::","call_ids":["semeval2013.d006.s005.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"committee%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":726,"cached_input_tokens":0,"output_tokens":71,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002162,"input_uncached_usd":0.001452,"input_cached_usd":0.0,"output_usd":0.00071,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.55717295804061}
{"item_id":"semeval2013.d006.s005.t002","gold_sense_keys":["game%1:04:03::"],"candidates":[{"index":1,"sense_key":"game%1:04:00::","synset_id":"game.n.01"},{"index":2,"sense_key":"game%1:04:03::","synset_id":"game.n.02"},{"index":3,"sense_key":"game%1:04:01::","synset_id":"game.n.03"},{"index":4,"sense_key":"game%1:05:00::","synset_id":"game.n.04"},{"index":5,"sense_key":"game%1:28:00::","synset_id":"game.n.05"},{"index":6,"sense_key":"game%1:23:00::","synset_id":"game.n.06"},{"index":7,"sense_key":"game%1:13:00::","synset_id":"game.n.07"},{"index":8,"sense_key":"game%1:09:00::","synset_id":"plot.n.01"},{"index":9,"sense_key":"game%1:06:00::","synset_id":"game.n.09"},{"index":10,"sense_key":"game%1:04:02::","synset_id":"game.n.10"},{"index":11,"sense_key":"game%1:04:04::","synset_id":"game.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"game%1:04:03::","call_ids":["semeval2013.d006.s005.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"game%1:04:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1160,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00299,"input_uncached_usd":0.00232,"input_cached_usd":0.0,"output_usd":0.00067,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":4.7702209999552}
{"item_id":"semeval2013.d006.s005.t003","gold_sense_keys":["trainer%1:18:00::"],"candidates":[{"index":1,"sense_key":"trainer%1:18:00::","synset_id":"trainer.n.01"},{"index":2,"sense_key":"trainer%1:06:00::","synset_id":"flight_simulator.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"trainer%1:18:00::","call_ids":["semeval2013.d006.s005.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"trainer%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":705,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0015300000000000001,"input_uncached_usd":0.00141,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9526600838871673}
{"item_id":"semeval2013.d006.s005.t004","gold_sense_keys":["team%1:14:00::"],"candidates":[{"index":1,"sense_key":"team%1:14:00::","synset_id":"team.n.01"},{"index":2,"sense_key":"team%1:14:01::","synset_id":"team.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"team%1:14:00::","call_ids":["semeval2013.d006.s005.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"team%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":684,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001488,"input_uncached_usd":0.0013679999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.174008832895197}
{"item_id":"semeval2013.d006.s005.t005","gold_sense_keys":["amount%1:21:00::"],"candidates":[{"index":1,"sense_key":"amount%1:21:00::","synset_id":"sum.n.01"},{"index":2,"sense_key":"amount%1:07:00::","synset_id":"amount.n.02"},{"index":3,"sense_key":"amount%1:03:00::","synset_id":"measure.n.02"},{"index":4,"sense_key":"amount%1:09:00::","synset_id":"sum.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"amount%1:21:00::","call_ids":["semeval2013.d006.s005.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"amount%1:21:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":837,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001794,"input_uncached_usd":0.001674,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.050489166053012}
{"item_id":"semeval2013.d006.s005.t006","gold_sense_keys":["player%1:18:01::"],"candidates":[{"index":1,"sense_key":"player%1:18:01::","synset_id":"player.n.01"},{"index":2,"sense_key":"player%1:18:02::","synset_id":"musician.n.01"},{"index":3,"sense_key":"player%1:18:00::","synset_id":"actor.n.01"},{"index":4,"sense_key":"player%1:18:04::","synset_id":"player.n.04"},{"index":5,"sense_key":"player%1:18:03::","synset_id":"player.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"player%1:18:01::","call_ids":["semeval2013.d006.s005.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"player%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":842,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001804,"input_uncached_usd":0.001684,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9569972499739379}
{"item_id":"semeval2013.d006.s006.t000","gold_sense_keys":["provocation%1:16:00::"],"candidates":[{"index":1,"sense_key":"provocation%1:04:00::","synset_id":"aggravation.n.02"},{"index":2,"sense_key":"provocation%1:16:00::","synset_id":"incitement.n.03"},{"index":3,"sense_key":"provocation%1:10:00::","synset_id":"provocation.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"provocation%1:10:00::","call_ids":["semeval2013.d006.s006.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"provocation%1:10:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":806,"cached_input_tokens":0,"output_tokens":330,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004912000000000001,"input_uncached_usd":0.001612,"input_cached_usd":0.0,"output_usd":0.0033000000000000004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.7630810840055346}
{"item_id":"semeval2013.d006.s006.t002","gold_sense_keys":["circumstance%1:26:01::","circumstance%1:26:02::"],"candidates":[{"index":1,"sense_key":"circumstance%1:26:01::","synset_id":"circumstance.n.01"},{"index":2,"sense_key":"circumstance%1:26:02::","synset_id":"context.n.02"},{"index":3,"sense_key":"circumstance%1:09:00::","synset_id":"circumstance.n.03"},{"index":4,"sense_key":"circumstance%1:11:00::","synset_id":"circumstance.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"circumstance%1:26:02::","call_ids":["semeval2013.d006.s006.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"circumstance%1:26:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":851,"cached_input_tokens":0,"output_tokens":149,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0031920000000000004,"input_uncached_usd":0.001702,"input_cached_usd":0.0,"output_usd":0.0014900000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9021383749786764}
{"item_id":"semeval2013.d006.s006.t003","gold_sense_keys":["practice%1:04:00::"],"candidates":[{"index":1,"sense_key":"practice%1:04:00::","synset_id":"practice.n.01"},{"index":2,"sense_key":"practice%1:04:02::","synset_id":"exercise.n.03"},{"index":3,"sense_key":"practice%1:04:04::","synset_id":"practice.n.03"},{"index":4,"sense_key":"practice%1:04:03::","synset_id":"practice.n.04"},{"index":5,"sense_key":"practice%1:09:00::","synset_id":"practice.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"practice%1:04:00::","call_ids":["semeval2013.d006.s006.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"practice%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":939,"cached_input_tokens":0,"output_tokens":79,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002668,"input_uncached_usd":0.001878,"input_cached_usd":0.0,"output_usd":0.00079,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.84536079200916}
{"item_id":"semeval2013.d006.s006.t004","gold_sense_keys":["world%1:14:01::"],"candidates":[{"index":1,"sense_key":"world%1:17:01::","synset_id":"universe.n.01"},{"index":2,"sense_key":"world%1:14:01::","synset_id":"world.n.02"},{"index":3,"sense_key":"world%1:09:00::","synset_id":"world.n.03"},{"index":4,"sense_key":"world%1:17:00::","synset_id":"earth.n.01"},{"index":5,"sense_key":"world%1:14:02::","synset_id":"populace.n.01"},{"index":6,"sense_key":"world%1:17:02::","synset_id":"world.n.06"},{"index":7,"sense_key":"world%1:09:01::","synset_id":"worldly_concern.n.01"},{"index":8,"sense_key":"world%1:05:00::","synset_id":"world.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"world%1:14:01::","call_ids":["semeval2013.d006.s006.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"world%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1188,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002926,"input_uncached_usd":0.002376,"input_cached_usd":0.0,"output_usd":0.00055,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":4.83035862504039}
{"item_id":"semeval2013.d006.s006.t005","gold_sense_keys":["football%1:04:00::"],"candidates":[{"index":1,"sense_key":"football%1:04:00::","synset_id":"football.n.01"},{"index":2,"sense_key":"football%1:06:00::","synset_id":"football.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"football%1:04:00::","call_ids":["semeval2013.d006.s006.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"football%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":738,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001596,"input_uncached_usd":0.0014759999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0685295420698822}
{"item_id":"semeval2013.d006.s007.t001","gold_sense_keys":["rule%1:10:02::"],"candidates":[{"index":1,"sense_key":"rule%1:09:00::","synset_id":"rule.n.01"},{"index":2,"sense_key":"rule%1:09:01::","synset_id":"convention.n.02"},{"index":3,"sense_key":"rule%1:10:00::","synset_id":"rule.n.03"},{"index":4,"sense_key":"rule%1:10:01::","synset_id":"rule.n.04"},{"index":5,"sense_key":"rule%1:09:03::","synset_id":"principle.n.01"},{"index":6,"sense_key":"rule%1:28:00::","synset_id":"rule.n.06"},{"index":7,"sense_key":"rule%1:26:00::","synset_id":"dominion.n.01"},{"index":8,"sense_key":"rule%1:10:02::","synset_id":"rule.n.08"},{"index":9,"sense_key":"rule%1:10:03::","synset_id":"rule.n.09"},{"index":10,"sense_key":"rule%1:09:02::","synset_id":"principle.n.04"},{"index":11,"sense_key":"rule%1:09:04::","synset_id":"rule.n.11"},{"index":12,"sense_key":"rule%1:06:00::","synset_id":"rule.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"rule%1:09:00::","call_ids":["semeval2013.d006.s007.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"rule%1:09:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1522,"cached_input_tokens":0,"output_tokens":110,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004144,"input_uncached_usd":0.003044,"input_cached_usd":0.0,"output_usd":0.0011,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1478627918986604}
{"item_id":"semeval2013.d006.s008.t000","gold_sense_keys":["part%1:24:00::"],"candidates":[{"index":1,"sense_key":"part%1:24:00::","synset_id":"part.n.01"},{"index":2,"sense_key":"part%1:06:00::","synset_id":"part.n.02"},{"index":3,"sense_key":"part%1:17:00::","synset_id":"part.n.03"},{"index":4,"sense_key":"part%1:09:02::","synset_id":"part.n.04"},{"index":5,"sense_key":"part%1:15:00::","synset_id":"region.n.01"},{"index":6,"sense_key":"part%1:04:01::","synset_id":"function.n.03"},{"index":7,"sense_key":"part%1:09:01::","synset_id":"character.n.04"},{"index":8,"sense_key":"part%1:21:00::","synset_id":"share.n.01"},{"index":9,"sense_key":"part%1:09:00::","synset_id":"part.n.09"},{"index":10,"sense_key":"part%1:08:01::","synset_id":"part.n.10"},{"index":11,"sense_key":"part%1:10:00::","synset_id":"part.n.11"},{"index":12,"sense_key":"part%1:04:03::","synset_id":"contribution.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"part%1:24:00::","call_ids":["semeval2013.d006.s008.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"part%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1499,"cached_input_tokens":0,"output_tokens":130,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004298,"input_uncached_usd":0.002998,"input_cached_usd":0.0,"output_usd":0.0013000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7641598749905825}
{"item_id":"semeval2013.d006.s008.t001","gold_sense_keys":["debate%1:10:01::"],"candidates":[{"index":1,"sense_key":"debate%1:10:01::","synset_id":"argument.n.03"},{"index":2,"sense_key":"debate%1:10:00::","synset_id":"debate.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"debate%1:10:01::","call_ids":["semeval2013.d006.s008.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"debate%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":831,"cached_input_tokens":0,"output_tokens":87,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002532,"input_uncached_usd":0.0016619999999999998,"input_cached_usd":0.0,"output_usd":0.0008700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4785353749757633}
{"item_id":"semeval2013.d006.s008.t002","gold_sense_keys":["case%1:11:00::"],"candidates":[{"index":1,"sense_key":"case%1:11:00::","synset_id":"case.n.01"},{"index":2,"sense_key":"case%1:26:00::","synset_id":"event.n.02"},{"index":3,"sense_key":"case%1:04:00::","synset_id":"lawsuit.n.01"},{"index":4,"sense_key":"case%1:09:00::","synset_id":"case.n.04"},{"index":5,"sense_key":"case%1:06:00::","synset_id":"case.n.05"},{"index":6,"sense_key":"case%1:18:03::","synset_id":"case.n.06"},{"index":7,"sense_key":"case%1:18:02::","synset_id":"subject.n.06"},{"index":8,"sense_key":"case%1:10:02::","synset_id":"case.n.08"},{"index":9,"sense_key":"case%1:10:01::","synset_id":"case.n.09"},{"index":10,"sense_key":"case%1:23:00::","synset_id":"case.n.10"},{"index":11,"sense_key":"case%1:10:00::","synset_id":"case.n.11"},{"index":12,"sense_key":"case%1:26:02::","synset_id":"case.n.12"},{"index":13,"sense_key":"case%1:18:04::","synset_id":"character.n.05"},{"index":14,"sense_key":"case%1:10:03::","synset_id":"font.n.01"},{"index":15,"sense_key":"case%1:08:00::","synset_id":"sheath.n.02"},{"index":16,"sense_key":"case%1:06:06::","synset_id":"shell.n.08"},{"index":17,"sense_key":"case%1:06:07::","synset_id":"casing.n.03"},{"index":18,"sense_key":"case%1:06:03::","synset_id":"case.n.18"},{"index":19,"sense_key":"case%1:06:02::","synset_id":"case.n.19"},{"index":20,"sense_key":"case%1:06:01::","synset_id":"case.n.20"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"case%1:11:00::","call_ids":["semeval2013.d006.s008.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"case%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1941,"cached_input_tokens":0,"output_tokens":91,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004792,"input_uncached_usd":0.003882,"input_cached_usd":0.0,"output_usd":0.0009100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0107976669678465}
{"item_id":"semeval2013.d006.s008.t003","gold_sense_keys":["fact%1:09:01::"],"candidates":[{"index":1,"sense_key":"fact%1:09:01::","synset_id":"fact.n.01"},{"index":2,"sense_key":"fact%1:10:01::","synset_id":"fact.n.02"},{"index":3,"sense_key":"fact%1:26:01::","synset_id":"fact.n.03"},{"index":4,"sense_key":"fact%1:09:02::","synset_id":"fact.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"fact%1:26:01::","call_ids":["semeval2013.d006.s008.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"fact%1:26:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":929,"cached_input_tokens":0,"output_tokens":108,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0029379999999999996,"input_uncached_usd":0.0018579999999999998,"input_cached_usd":0.0,"output_usd":0.00108,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6386581669794396}
{"item_id":"semeval2013.d006.s008.t004","gold_sense_keys":["team%1:14:00::"],"candidates":[{"index":1,"sense_key":"team%1:14:00::","synset_id":"team.n.01"},{"index":2,"sense_key":"team%1:14:01::","synset_id":"team.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"team%1:14:00::","call_ids":["semeval2013.d006.s008.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"team%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":765,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00165,"input_uncached_usd":0.00153,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1349745830520988}
{"item_id":"semeval2013.d006.s008.t005","gold_sense_keys":["organization%1:14:00::","organization%1:14:01::"],"candidates":[{"index":1,"sense_key":"organization%1:14:00::","synset_id":"organization.n.01"},{"index":2,"sense_key":"organization%1:09:00::","synset_id":"arrangement.n.03"},{"index":3,"sense_key":"organization%1:14:01::","synset_id":"administration.n.02"},{"index":4,"sense_key":"organization%1:04:00::","synset_id":"organization.n.04"},{"index":5,"sense_key":"organization%1:07:00::","synset_id":"organization.n.05"},{"index":6,"sense_key":"organization%1:04:01::","synset_id":"organization.n.06"},{"index":7,"sense_key":"organization%1:04:02::","synset_id":"constitution.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"organization%1:14:00::","call_ids":["semeval2013.d006.s008.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"organization%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1230,"cached_input_tokens":0,"output_tokens":147,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00393,"input_uncached_usd":0.00246,"input_cached_usd":0.0,"output_usd":0.0014700000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":5.12189883296378}
{"item_id":"semeval2013.d006.s008.t007","gold_sense_keys":["nobility%1:07:00::"],"candidates":[{"index":1,"sense_key":"nobility%1:14:00::","synset_id":"nobility.n.01"},{"index":2,"sense_key":"nobility%1:07:00::","synset_id":"nobility.n.02"},{"index":3,"sense_key":"nobility%1:26:00::","synset_id":"nobility.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"nobility%1:07:00::","call_ids":["semeval2013.d006.s008.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"nobility%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":850,"cached_input_tokens":0,"output_tokens":70,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0024000000000000002,"input_uncached_usd":0.0017,"input_cached_usd":0.0,"output_usd":0.0007000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2806367919547483}
{"item_id":"semeval2013.d006.s008.t008","gold_sense_keys":["honor%1:07:00::"],"candidates":[{"index":1,"sense_key":"honor%1:10:00::","synset_id":"award.n.02"},{"index":2,"sense_key":"honor%1:26:00::","synset_id":"honor.n.02"},{"index":3,"sense_key":"honor%1:07:00::","synset_id":"honor.n.03"},{"index":4,"sense_key":"honor%1:07:01::","synset_id":"honor.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"honor%1:07:00::","call_ids":["semeval2013.d006.s008.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"honor%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":895,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00235,"input_uncached_usd":0.00179,"input_cached_usd":0.0,"output_usd":0.0005600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2985258750850335}
{"item_id":"semeval2013.d006.s008.t009","gold_sense_keys":["player%1:18:01::"],"candidates":[{"index":1,"sense_key":"player%1:18:01::","synset_id":"player.n.01"},{"index":2,"sense_key":"player%1:18:02::","synset_id":"musician.n.01"},{"index":3,"sense_key":"player%1:18:00::","synset_id":"actor.n.01"},{"index":4,"sense_key":"player%1:18:04::","synset_id":"player.n.04"},{"index":5,"sense_key":"player%1:18:03::","synset_id":"player.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"player%1:18:01::","call_ids":["semeval2013.d006.s008.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"player%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":923,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001966,"input_uncached_usd":0.001846,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4115677500376478}
{"item_id":"semeval2013.d006.s008.t010","gold_sense_keys":["act%1:03:00::"],"candidates":[{"index":1,"sense_key":"act%1:10:01::","synset_id":"act.n.01"},{"index":2,"sense_key":"act%1:03:00::","synset_id":"act.n.02"},{"index":3,"sense_key":"act%1:10:00::","synset_id":"act.n.03"},{"index":4,"sense_key":"act%1:10:02::","synset_id":"act.n.04"},{"index":5,"sense_key":"act%1:10:03::","synset_id":"act.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"act%1:03:00::","call_ids":["semeval2013.d006.s008.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"act%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":948,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002486,"input_uncached_usd":0.001896,"input_cached_usd":0.0,"output_usd":0.00059,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4972681250656024}
{"item_id":"semeval2013.d006.s008.t012","gold_sense_keys":["history%1:28:00::"],"candidates":[{"index":1,"sense_key":"history%1:28:00::","synset_id":"history.n.01"},{"index":2,"sense_key":"history%1:10:00::","synset_id":"history.n.02"},{"index":3,"sense_key":"history%1:09:00::","synset_id":"history.n.03"},{"index":4,"sense_key":"history%1:28:02::","synset_id":"history.n.04"},{"index":5,"sense_key":"history%1:09:01::","synset_id":"history.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"history%1:28:00::","call_ids":["semeval2013.d006.s008.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"history%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":980,"cached_input_tokens":0,"output_tokens":79,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00275,"input_uncached_usd":0.00196,"input_cached_usd":0.0,"output_usd":0.00079,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.58588254195638}
{"item_id":"semeval2013.d006.s009.t000","gold_sense_keys":["debate%1:10:01::"],"candidates":[{"index":1,"sense_key":"debate%1:10:01::","synset_id":"argument.n.03"},{"index":2,"sense_key":"debate%1:10:00::","synset_id":"debate.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"debate%1:10:01::","call_ids":["semeval2013.d006.s009.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"debate%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":803,"cached_input_tokens":0,"output_tokens":82,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002426,"input_uncached_usd":0.001606,"input_cached_usd":0.0,"output_usd":0.0008200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5087815000442788}
{"item_id":"semeval2013.d006.s009.t001","gold_sense_keys":["type%1:09:00::"],"candidates":[{"index":1,"sense_key":"type%1:09:00::","synset_id":"type.n.01"},{"index":2,"sense_key":"type%1:18:00::","synset_id":"character.n.05"},{"index":3,"sense_key":"type%1:14:01::","synset_id":"type.n.03"},{"index":4,"sense_key":"type%1:10:01::","synset_id":"type.n.04"},{"index":5,"sense_key":"type%1:10:00::","synset_id":"type.n.05"},{"index":6,"sense_key":"type%1:06:00::","synset_id":"type.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"type%1:09:00::","call_ids":["semeval2013.d006.s009.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"type%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":985,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00209,"input_uncached_usd":0.00197,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":4.981774708023295}
{"item_id":"semeval2013.d006.s009.t002","gold_sense_keys":["behavior%1:04:00::"],"candidates":[{"index":1,"sense_key":"behavior%1:04:00::","synset_id":"behavior.n.01"},{"index":2,"sense_key":"behavior%1:26:00::","synset_id":"behavior.n.02"},{"index":3,"sense_key":"behavior%1:07:00::","synset_id":"demeanor.n.01"},{"index":4,"sense_key":"behavior%1:04:01::","synset_id":"behavior.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"behavior%1:04:00::","call_ids":["semeval2013.d006.s009.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"behavior%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":926,"cached_input_tokens":0,"output_tokens":110,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0029519999999999998,"input_uncached_usd":0.001852,"input_cached_usd":0.0,"output_usd":0.0011,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.360994042013772}
{"item_id":"semeval2013.d006.s009.t003","gold_sense_keys":["principle%1:09:01::"],"candidates":[{"index":1,"sense_key":"principle%1:09:03::","synset_id":"principle.n.01"},{"index":2,"sense_key":"principle%1:09:01::","synset_id":"principle.n.02"},{"index":3,"sense_key":"principle%1:09:00::","synset_id":"principle.n.03"},{"index":4,"sense_key":"principle%1:09:04::","synset_id":"principle.n.04"},{"index":5,"sense_key":"principle%1:10:00::","synset_id":"principle.n.05"},{"index":6,"sense_key":"principle%1:09:02::","synset_id":"rationale.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"principle%1:09:00::","call_ids":["semeval2013.d006.s009.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"principle%1:09:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1050,"cached_input_tokens":0,"output_tokens":114,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00324,"input_uncached_usd":0.0021,"input_cached_usd":0.0,"output_usd":0.0011400000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.073811624897644}
{"item_id":"semeval2013.d006.s009.t005","gold_sense_keys":["action%1:04:02::"],"candidates":[{"index":1,"sense_key":"action%1:04:02::","synset_id":"action.n.01"},{"index":2,"sense_key":"action%1:26:00::","synset_id":"action.n.02"},{"index":3,"sense_key":"action%1:04:03::","synset_id":"military_action.n.01"},{"index":4,"sense_key":"action%1:22:00::","synset_id":"natural_process.n.01"},{"index":5,"sense_key":"action%1:10:00::","synset_id":"action.n.05"},{"index":6,"sense_key":"action%1:07:00::","synset_id":"action.n.06"},{"index":7,"sense_key":"action%1:06:00::","synset_id":"action.n.07"},{"index":8,"sense_key":"action%1:04:01::","synset_id":"legal_action.n.01"},{"index":9,"sense_key":"action%1:04:04::","synset_id":"action.n.09"},{"index":10,"sense_key":"action%1:04:00::","synset_id":"action.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"action%1:04:02::","call_ids":["semeval2013.d006.s009.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"action%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1284,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003158,"input_uncached_usd":0.002568,"input_cached_usd":0.0,"output_usd":0.00059,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":4.136093125096522}
{"item_id":"semeval2013.d006.s009.t006","gold_sense_keys":["fraud%1:04:01::"],"candidates":[{"index":1,"sense_key":"fraud%1:04:00::","synset_id":"fraud.n.01"},{"index":2,"sense_key":"fraud%1:18:00::","synset_id":"imposter.n.01"},{"index":3,"sense_key":"fraud%1:04:01::","synset_id":"fraud.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"fraud%1:04:00::","call_ids":["semeval2013.d006.s009.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"fraud%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":836,"cached_input_tokens":0,"output_tokens":228,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003952000000000001,"input_uncached_usd":0.0016719999999999999,"input_cached_usd":0.0,"output_usd":0.0022800000000000003,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":5.989773624925874}
{"item_id":"semeval2013.d006.s009.t007","gold_sense_keys":["action%1:04:02::"],"candidates":[{"index":1,"sense_key":"action%1:04:02::","synset_id":"action.n.01"},{"index":2,"sense_key":"action%1:26:00::","synset_id":"action.n.02"},{"index":3,"sense_key":"action%1:04:03::","synset_id":"military_action.n.01"},{"index":4,"sense_key":"action%1:22:00::","synset_id":"natural_process.n.01"},{"index":5,"sense_key":"action%1:10:00::","synset_id":"action.n.05"},{"index":6,"sense_key":"action%1:07:00::","synset_id":"action.n.06"},{"index":7,"sense_key":"action%1:06:00::","synset_id":"action.n.07"},{"index":8,"sense_key":"action%1:04:01::","synset_id":"legal_action.n.01"},{"index":9,"sense_key":"action%1:04:04::","synset_id":"action.n.09"},{"index":10,"sense_key":"action%1:04:00::","synset_id":"action.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"action%1:04:02::","call_ids":["semeval2013.d006.s009.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"action%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1284,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002688,"input_uncached_usd":0.002568,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0642319159815088}
{"item_id":"semeval2013.d006.s009.t009","gold_sense_keys":["regulation%1:10:00::"],"candidates":[{"index":1,"sense_key":"regulation%1:10:00::","synset_id":"regulation.n.01"},{"index":2,"sense_key":"regulation%1:09:00::","synset_id":"rule.n.01"},{"index":3,"sense_key":"regulation%1:26:00::","synset_id":"regulation.n.03"},{"index":4,"sense_key":"regulation%1:22:00::","synset_id":"regulation.n.04"},{"index":5,"sense_key":"regulation%1:04:01::","synset_id":"regulation.n.05"},{"index":6,"sense_key":"regulation%1:04:00::","synset_id":"regulation.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"regulation%1:10:00::","call_ids":["semeval2013.d006.s009.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"regulation%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":987,"cached_input_tokens":0,"output_tokens":110,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0030740000000000003,"input_uncached_usd":0.001974,"input_cached_usd":0.0,"output_usd":0.0011,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":5.619770125020295}
{"item_id":"semeval2013.d006.s010.t000","gold_sense_keys":["matter%1:09:00::"],"candidates":[{"index":1,"sense_key":"matter%1:09:01::","synset_id":"matter.n.01"},{"index":2,"sense_key":"matter%1:09:00::","synset_id":"topic.n.02"},{"index":3,"sense_key":"matter%1:03:00::","synset_id":"matter.n.03"},{"index":4,"sense_key":"matter%1:09:02::","synset_id":"matter.n.04"},{"index":5,"sense_key":"matter%1:07:00::","synset_id":"matter.n.05"},{"index":6,"sense_key":"matter%1:10:00::","synset_id":"matter.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"matter%1:09:00::","call_ids":["semeval2013.d006.s010.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"matter%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":979,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002548,"input_uncached_usd":0.001958,"input_cached_usd":0.0,"output_usd":0.00059,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3202833749819547}
{"item_id":"semeval2013.d006.s010.t001","gold_sense_keys":["type%1:09:00::"],"candidates":[{"index":1,"sense_key":"type%1:09:00::","synset_id":"type.n.01"},{"index":2,"sense_key":"type%1:18:00::","synset_id":"character.n.05"},{"index":3,"sense_key":"type%1:14:01::","synset_id":"type.n.03"},{"index":4,"sense_key":"type%1:10:01::","synset_id":"type.n.04"},{"index":5,"sense_key":"type%1:10:00::","synset_id":"type.n.05"},{"index":6,"sense_key":"type%1:06:00::","synset_id":"type.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"type%1:09:00::","call_ids":["semeval2013.d006.s010.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"type%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":975,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00207,"input_uncached_usd":0.00195,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1335053340299055}
{"item_id":"semeval2013.d006.s010.t002","gold_sense_keys":["damage%1:11:00::"],"candidates":[{"index":1,"sense_key":"damage%1:11:00::","synset_id":"damage.n.01"},{"index":2,"sense_key":"damage%1:11:01::","synset_id":"damage.n.02"},{"index":3,"sense_key":"damage%1:04:00::","synset_id":"damage.n.03"},{"index":4,"sense_key":"damage%1:21:00::","synset_id":"price.n.02"},{"index":5,"sense_key":"damage%1:04:01::","synset_id":"wrong.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"damage%1:11:00::","call_ids":["semeval2013.d006.s010.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"damage%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":907,"cached_input_tokens":0,"output_tokens":119,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0030039999999999997,"input_uncached_usd":0.0018139999999999999,"input_cached_usd":0.0,"output_usd":0.00119,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.682287915959023}
{"item_id":"semeval2013.d006.s011.t000","gold_sense_keys":["advantage%1:07:01::"],"candidates":[{"index":1,"sense_key":"advantage%1:07:00::","synset_id":"advantage.n.01"},{"index":2,"sense_key":"advantage%1:23:00::","synset_id":"advantage.n.02"},{"index":3,"sense_key":"advantage%1:07:01::","synset_id":"advantage.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"advantage%1:07:01::","call_ids":["semeval2013.d006.s011.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"advantage%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":823,"cached_input_tokens":0,"output_tokens":97,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002616,"input_uncached_usd":0.001646,"input_cached_usd":0.0,"output_usd":0.00097,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9122346659423783}
{"item_id":"semeval2013.d006.s011.t003","gold_sense_keys":["integrity%1:26:00::"],"candidates":[{"index":1,"sense_key":"integrity%1:26:00::","synset_id":"integrity.n.01"},{"index":2,"sense_key":"integrity%1:07:00::","synset_id":"integrity.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"integrity%1:26:00::","call_ids":["semeval2013.d006.s011.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"integrity%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":822,"cached_input_tokens":0,"output_tokens":71,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0023539999999999998,"input_uncached_usd":0.0016439999999999998,"input_cached_usd":0.0,"output_usd":0.00071,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8917827919358388}
{"item_id":"semeval2013.d006.s011.t004","gold_sense_keys":["player%1:18:01::"],"candidates":[{"index":1,"sense_key":"player%1:18:01::","synset_id":"player.n.01"},{"index":2,"sense_key":"player%1:18:02::","synset_id":"musician.n.01"},{"index":3,"sense_key":"player%1:18:00::","synset_id":"actor.n.01"},{"index":4,"sense_key":"player%1:18:04::","synset_id":"player.n.04"},{"index":5,"sense_key":"player%1:18:03::","synset_id":"player.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"player%1:18:01::","call_ids":["semeval2013.d006.s011.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"player%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":890,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0019,"input_uncached_usd":0.00178,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.394812708022073}
{"item_id":"semeval2013.d006.s011.t005","gold_sense_keys":["aspect%1:07:02::"],"candidates":[{"index":1,"sense_key":"aspect%1:09:00::","synset_id":"aspect.n.01"},{"index":2,"sense_key":"aspect%1:07:02::","synset_id":"aspect.n.02"},{"index":3,"sense_key":"aspect%1:09:01::","synset_id":"view.n.02"},{"index":4,"sense_key":"aspect%1:24:00::","synset_id":"aspect.n.04"},{"index":5,"sense_key":"aspect%1:07:01::","synset_id":"expression.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"aspect%1:09:00::","call_ids":["semeval2013.d006.s011.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"aspect%1:09:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":931,"cached_input_tokens":0,"output_tokens":216,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0040219999999999995,"input_uncached_usd":0.001862,"input_cached_usd":0.0,"output_usd":0.00216,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.849786209058948}
{"item_id":"semeval2013.d006.s011.t006","gold_sense_keys":["action%1:04:02::"],"candidates":[{"index":1,"sense_key":"action%1:04:02::","synset_id":"action.n.01"},{"index":2,"sense_key":"action%1:26:00::","synset_id":"action.n.02"},{"index":3,"sense_key":"action%1:04:03::","synset_id":"military_action.n.01"},{"index":4,"sense_key":"action%1:22:00::","synset_id":"natural_process.n.01"},{"index":5,"sense_key":"action%1:10:00::","synset_id":"action.n.05"},{"index":6,"sense_key":"action%1:07:00::","synset_id":"action.n.06"},{"index":7,"sense_key":"action%1:06:00::","synset_id":"action.n.07"},{"index":8,"sense_key":"action%1:04:01::","synset_id":"legal_action.n.01"},{"index":9,"sense_key":"action%1:04:04::","synset_id":"action.n.09"},{"index":10,"sense_key":"action%1:04:00::","synset_id":"action.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"action%1:04:02::","call_ids":["semeval2013.d006.s011.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"action%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1279,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003088,"input_uncached_usd":0.002558,"input_cached_usd":0.0,"output_usd":0.0005300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":9.498620999976993}
{"item_id":"semeval2013.d006.s011.t007","gold_sense_keys":["danger%1:26:00::"],"candidates":[{"index":1,"sense_key":"danger%1:26:00::","synset_id":"danger.n.01"},{"index":2,"sense_key":"danger%1:04:00::","synset_id":"risk.n.02"},{"index":3,"sense_key":"danger%1:26:01::","synset_id":"danger.n.03"},{"index":4,"sense_key":"danger%1:15:00::","synset_id":"danger.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"danger%1:26:00::","call_ids":["semeval2013.d006.s011.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"danger%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":889,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002498,"input_uncached_usd":0.0017779999999999998,"input_cached_usd":0.0,"output_usd":0.00072,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.311327000032179}
{"item_id":"semeval2013.d006.s012.t000","gold_sense_keys":["advantage%1:07:01::"],"candidates":[{"index":1,"sense_key":"advantage%1:07:00::","synset_id":"advantage.n.01"},{"index":2,"sense_key":"advantage%1:23:00::","synset_id":"advantage.n.02"},{"index":3,"sense_key":"advantage%1:07:01::","synset_id":"advantage.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"advantage%1:07:01::","call_ids":["semeval2013.d006.s012.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"advantage%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":875,"cached_input_tokens":0,"output_tokens":126,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0030099999999999997,"input_uncached_usd":0.0017499999999999998,"input_cached_usd":0.0,"output_usd":0.00126,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1087202089838684}
{"item_id":"semeval2013.d006.s012.t001","gold_sense_keys":["team%1:14:00::"],"candidates":[{"index":1,"sense_key":"team%1:14:00::","synset_id":"team.n.01"},{"index":2,"sense_key":"team%1:14:01::","synset_id":"team.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"team%1:14:00::","call_ids":["semeval2013.d006.s012.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"team%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":784,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001688,"input_uncached_usd":0.001568,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.050959166022949}
{"item_id":"semeval2013.d006.s012.t003","gold_sense_keys":["player%1:18:01::"],"candidates":[{"index":1,"sense_key":"player%1:18:01::","synset_id":"player.n.01"},{"index":2,"sense_key":"player%1:18:02::","synset_id":"musician.n.01"},{"index":3,"sense_key":"player%1:18:00::","synset_id":"actor.n.01"},{"index":4,"sense_key":"player%1:18:04::","synset_id":"player.n.04"},{"index":5,"sense_key":"player%1:18:03::","synset_id":"player.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"player%1:18:01::","call_ids":["semeval2013.d006.s012.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"player%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":942,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0020039999999999997,"input_uncached_usd":0.0018839999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3581006249878556}
{"item_id":"semeval2013.d006.s012.t004","gold_sense_keys":["take_advantage%2:40:00::"],"candidates":[{"index":1,"sense_key":"take_advantage%2:40:00::","synset_id":"capitalize.v.01"},{"index":2,"sense_key":"take_advantage%2:41:12::","synset_id":"trespass.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"take_advantage%2:41:12::","call_ids":["semeval2013.d006.s012.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"take_advantage%2:41:12::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":861,"cached_input_tokens":0,"output_tokens":140,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003122,"input_uncached_usd":0.001722,"input_cached_usd":0.0,"output_usd":0.0014000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":9.468513042083941}
{"item_id":"semeval2013.d006.s012.t005","gold_sense_keys":["team%1:14:00::"],"candidates":[{"index":1,"sense_key":"team%1:14:00::","synset_id":"team.n.01"},{"index":2,"sense_key":"team%1:14:01::","synset_id":"team.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"team%1:14:00::","call_ids":["semeval2013.d006.s012.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"team%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":784,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001688,"input_uncached_usd":0.001568,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8453557910397649}
{"item_id":"semeval2013.d006.s013.t000","gold_sense_keys":["thing%1:07:00::"],"candidates":[{"index":1,"sense_key":"thing%1:26:00::","synset_id":"thing.n.01"},{"index":2,"sense_key":"thing%1:04:00::","synset_id":"thing.n.02"},{"index":3,"sense_key":"thing%1:09:00::","synset_id":"thing.n.03"},{"index":4,"sense_key":"thing%1:06:00::","synset_id":"thing.n.04"},{"index":5,"sense_key":"thing%1:11:00::","synset_id":"thing.n.05"},{"index":6,"sense_key":"thing%1:09:02::","synset_id":"matter.n.01"},{"index":7,"sense_key":"thing%1:10:00::","synset_id":"thing.n.07"},{"index":8,"sense_key":"thing%1:06:01::","synset_id":"thing.n.08"},{"index":9,"sense_key":"thing%1:07:00::","synset_id":"thing.n.09"},{"index":10,"sense_key":"thing%1:09:01::","synset_id":"thing.n.10"},{"index":11,"sense_key":"thing%1:12:00::","synset_id":"thing.n.11"},{"index":12,"sense_key":"thing%1:03:00::","synset_id":"thing.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"thing%1:07:00::","call_ids":["semeval2013.d006.s013.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"thing%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1246,"cached_input_tokens":0,"output_tokens":155,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0040420000000000005,"input_uncached_usd":0.002492,"input_cached_usd":0.0,"output_usd":0.0015500000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.044355084071867}
{"item_id":"semeval2013.d006.s013.t001","gold_sense_keys":["action%1:04:02::"],"candidates":[{"index":1,"sense_key":"action%1:04:02::","synset_id":"action.n.01"},{"index":2,"sense_key":"action%1:26:00::","synset_id":"action.n.02"},{"index":3,"sense_key":"action%1:04:03::","synset_id":"military_action.n.01"},{"index":4,"sense_key":"action%1:22:00::","synset_id":"natural_process.n.01"},{"index":5,"sense_key":"action%1:10:00::","synset_id":"action.n.05"},{"index":6,"sense_key":"action%1:07:00::","synset_id":"action.n.06"},{"index":7,"sense_key":"action%1:06:00::","synset_id":"action.n.07"},{"index":8,"sense_key":"action%1:04:01::","synset_id":"legal_action.n.01"},{"index":9,"sense_key":"action%1:04:04::","synset_id":"action.n.09"},{"index":10,"sense_key":"action%1:04:00::","synset_id":"action.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"action%1:04:02::","call_ids":["semeval2013.d006.s013.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"action%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1312,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002744,"input_uncached_usd":0.002624,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0141900840681046}
{"item_id":"semeval2013.d006.s013.t002","gold_sense_keys":["player%1:18:01::"],"candidates":[{"index":1,"sense_key":"player%1:18:01::","synset_id":"player.n.01"},{"index":2,"sense_key":"player%1:18:02::","synset_id":"musician.n.01"},{"index":3,"sense_key":"player%1:18:00::","synset_id":"actor.n.01"},{"index":4,"sense_key":"player%1:18:04::","synset_id":"player.n.04"},{"index":5,"sense_key":"player%1:18:03::","synset_id":"player.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"player%1:18:01::","call_ids":["semeval2013.d006.s013.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"player%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":924,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0019679999999999997,"input_uncached_usd":0.0018479999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1151183330221102}
{"item_id":"semeval2013.d006.s013.t005","gold_sense_keys":["rule%1:10:02::"],"candidates":[{"index":1,"sense_key":"rule%1:09:00::","synset_id":"rule.n.01"},{"index":2,"sense_key":"rule%1:09:01::","synset_id":"convention.n.02"},{"index":3,"sense_key":"rule%1:10:00::","synset_id":"rule.n.03"},{"index":4,"sense_key":"rule%1:10:01::","synset_id":"rule.n.04"},{"index":5,"sense_key":"rule%1:09:03::","synset_id":"principle.n.01"},{"index":6,"sense_key":"rule%1:28:00::","synset_id":"rule.n.06"},{"index":7,"sense_key":"rule%1:26:00::","synset_id":"dominion.n.01"},{"index":8,"sense_key":"rule%1:10:02::","synset_id":"rule.n.08"},{"index":9,"sense_key":"rule%1:10:03::","synset_id":"rule.n.09"},{"index":10,"sense_key":"rule%1:09:02::","synset_id":"principle.n.04"},{"index":11,"sense_key":"rule%1:09:04::","synset_id":"rule.n.11"},{"index":12,"sense_key":"rule%1:06:00::","synset_id":"rule.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"rule%1:10:02::","call_ids":["semeval2013.d006.s013.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"rule%1:10:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1505,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00373,"input_uncached_usd":0.0030099999999999997,"input_cached_usd":0.0,"output_usd":0.00072,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.7014141670661047}
{"item_id":"semeval2013.d006.s013.t007","gold_sense_keys":["action%1:04:02::"],"candidates":[{"index":1,"sense_key":"action%1:04:02::","synset_id":"action.n.01"},{"index":2,"sense_key":"action%1:26:00::","synset_id":"action.n.02"},{"index":3,"sense_key":"action%1:04:03::","synset_id":"military_action.n.01"},{"index":4,"sense_key":"action%1:22:00::","synset_id":"natural_process.n.01"},{"index":5,"sense_key":"action%1:10:00::","synset_id":"action.n.05"},{"index":6,"sense_key":"action%1:07:00::","synset_id":"action.n.06"},{"index":7,"sense_key":"action%1:06:00::","synset_id":"action.n.07"},{"index":8,"sense_key":"action%1:04:01::","synset_id":"legal_action.n.01"},{"index":9,"sense_key":"action%1:04:04::","synset_id":"action.n.09"},{"index":10,"sense_key":"action%1:04:00::","synset_id":"action.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"action%1:04:02::","call_ids":["semeval2013.d006.s013.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"action%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1312,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002744,"input_uncached_usd":0.002624,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":8.325840750010684}
{"item_id":"semeval2013.d006.s013.t009","gold_sense_keys":["referee%1:18:00::"],"candidates":[{"index":1,"sense_key":"referee%1:18:00::","synset_id":"referee.n.01"},{"index":2,"sense_key":"referee%1:18:01::","synset_id":"reviewer.n.01"},{"index":3,"sense_key":"referee%1:18:02::","synset_id":"referee.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"referee%1:18:00::","call_ids":["semeval2013.d006.s013.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"referee%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":845,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00181,"input_uncached_usd":0.0016899999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":4.704593708040193}
{"item_id":"semeval2013.d006.s014.t001","gold_sense_keys":["complaint%1:10:00::"],"candidates":[{"index":1,"sense_key":"complaint%1:26:00::","synset_id":"ailment.n.01"},{"index":2,"sense_key":"complaint%1:10:02::","synset_id":"complaint.n.02"},{"index":3,"sense_key":"complaint%1:10:00::","synset_id":"complaint.n.03"},{"index":4,"sense_key":"complaint%1:10:03::","synset_id":"complaint.n.04"},{"index":5,"sense_key":"complaint%1:10:04::","synset_id":"charge.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"complaint%1:10:00::","call_ids":["semeval2013.d006.s014.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"complaint%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":869,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001858,"input_uncached_usd":0.001738,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.057007999974303}
{"item_id":"semeval2013.d006.s015.t000","gold_sense_keys":["question%1:10:00::","question%1:10:01::","question%1:10:03::"],"candidates":[{"index":1,"sense_key":"question%1:10:00::","synset_id":"question.n.01"},{"index":2,"sense_key":"question%1:10:01::","synset_id":"question.n.02"},{"index":3,"sense_key":"question%1:10:03::","synset_id":"question.n.03"},{"index":4,"sense_key":"question%1:07:00::","synset_id":"doubt.n.02"},{"index":5,"sense_key":"question%1:10:02::","synset_id":"motion.n.05"},{"index":6,"sense_key":"question%1:10:04::","synset_id":"question.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"question%1:10:01::","call_ids":["semeval2013.d006.s015.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"question%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1006,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002622,"input_uncached_usd":0.002012,"input_cached_usd":0.0,"output_usd":0.0006100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1976573340361938}
{"item_id":"semeval2013.d006.s015.t001","gold_sense_keys":["type%1:09:00::"],"candidates":[{"index":1,"sense_key":"type%1:09:00::","synset_id":"type.n.01"},{"index":2,"sense_key":"type%1:18:00::","synset_id":"character.n.05"},{"index":3,"sense_key":"type%1:14:01::","synset_id":"type.n.03"},{"index":4,"sense_key":"type%1:10:01::","synset_id":"type.n.04"},{"index":5,"sense_key":"type%1:10:00::","synset_id":"type.n.05"},{"index":6,"sense_key":"type%1:06:00::","synset_id":"type.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"type%1:09:00::","call_ids":["semeval2013.d006.s015.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"type%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":897,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0019140000000000001,"input_uncached_usd":0.001794,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1464453750522807}
{"item_id":"semeval2013.d006.s015.t002","gold_sense_keys":["advantage%1:07:01::"],"candidates":[{"index":1,"sense_key":"advantage%1:07:00::","synset_id":"advantage.n.01"},{"index":2,"sense_key":"advantage%1:23:00::","synset_id":"advantage.n.02"},{"index":3,"sense_key":"advantage%1:07:01::","synset_id":"advantage.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"advantage%1:07:01::","call_ids":["semeval2013.d006.s015.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"advantage%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":740,"cached_input_tokens":0,"output_tokens":162,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0031000000000000003,"input_uncached_usd":0.00148,"input_cached_usd":0.0,"output_usd":0.0016200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0430869580013677}
{"item_id":"semeval2013.d006.s015.t004","gold_sense_keys":["action%1:04:02::"],"candidates":[{"index":1,"sense_key":"action%1:04:02::","synset_id":"action.n.01"},{"index":2,"sense_key":"action%1:26:00::","synset_id":"action.n.02"},{"index":3,"sense_key":"action%1:04:03::","synset_id":"military_action.n.01"},{"index":4,"sense_key":"action%1:22:00::","synset_id":"natural_process.n.01"},{"index":5,"sense_key":"action%1:10:00::","synset_id":"action.n.05"},{"index":6,"sense_key":"action%1:07:00::","synset_id":"action.n.06"},{"index":7,"sense_key":"action%1:06:00::","synset_id":"action.n.07"},{"index":8,"sense_key":"action%1:04:01::","synset_id":"legal_action.n.01"},{"index":9,"sense_key":"action%1:04:04::","synset_id":"action.n.09"},{"index":10,"sense_key":"action%1:04:00::","synset_id":"action.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"action%1:04:02::","call_ids":["semeval2013.d006.s015.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"action%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1196,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002512,"input_uncached_usd":0.002392,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":4.9063236670335755}
{"item_id":"semeval2013.d006.s016.t000","gold_sense_keys":["situation%1:26:00::","situation%1:26:01::"],"candidates":[{"index":1,"sense_key":"situation%1:26:00::","synset_id":"situation.n.01"},{"index":2,"sense_key":"situation%1:26:01::","synset_id":"situation.n.02"},{"index":3,"sense_key":"situation%1:26:02::","synset_id":"situation.n.03"},{"index":4,"sense_key":"situation%1:15:00::","synset_id":"site.n.02"},{"index":5,"sense_key":"situation%1:04:00::","synset_id":"position.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"situation%1:26:01::","call_ids":["semeval2013.d006.s016.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"situation%1:26:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":945,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00254,"input_uncached_usd":0.00189,"input_cached_usd":0.0,"output_usd":0.0006500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9014373749960214}
{"item_id":"semeval2013.d006.s016.t002","gold_sense_keys":["benefit%1:07:00::"],"candidates":[{"index":1,"sense_key":"benefit%1:21:00::","synset_id":"benefit.n.01"},{"index":2,"sense_key":"benefit%1:07:00::","synset_id":"benefit.n.02"},{"index":3,"sense_key":"benefit%1:10:00::","synset_id":"benefit.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"benefit%1:07:00::","call_ids":["semeval2013.d006.s016.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"benefit%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":704,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001528,"input_uncached_usd":0.001408,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":8.905323167098686}
{"item_id":"semeval2013.d006.s016.t003","gold_sense_keys":["game%1:04:03::"],"candidates":[{"index":1,"sense_key":"game%1:04:00::","synset_id":"game.n.01"},{"index":2,"sense_key":"game%1:04:03::","synset_id":"game.n.02"},{"index":3,"sense_key":"game%1:04:01::","synset_id":"game.n.03"},{"index":4,"sense_key":"game%1:05:00::","synset_id":"game.n.04"},{"index":5,"sense_key":"game%1:28:00::","synset_id":"game.n.05"},{"index":6,"sense_key":"game%1:23:00::","synset_id":"game.n.06"},{"index":7,"sense_key":"game%1:13:00::","synset_id":"game.n.07"},{"index":8,"sense_key":"game%1:09:00::","synset_id":"plot.n.01"},{"index":9,"sense_key":"game%1:06:00::","synset_id":"game.n.09"},{"index":10,"sense_key":"game%1:04:02::","synset_id":"game.n.10"},{"index":11,"sense_key":"game%1:04:04::","synset_id":"game.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"game%1:04:03::","call_ids":["semeval2013.d006.s016.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"game%1:04:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1131,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0028519999999999995,"input_uncached_usd":0.0022619999999999997,"input_cached_usd":0.0,"output_usd":0.00059,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1519745420664549}
{"item_id":"semeval2013.d006.s016.t004","gold_sense_keys":["result%1:19:00::"],"candidates":[{"index":1,"sense_key":"result%1:19:00::","synset_id":"consequence.n.01"},{"index":2,"sense_key":"result%1:10:00::","synset_id":"solution.n.02"},{"index":3,"sense_key":"result%1:11:00::","synset_id":"result.n.03"},{"index":4,"sense_key":"result%1:10:01::","synset_id":"resultant_role.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"result%1:19:00::","call_ids":["semeval2013.d006.s016.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"result%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":895,"cached_input_tokens":0,"output_tokens":85,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00264,"input_uncached_usd":0.00179,"input_cached_usd":0.0,"output_usd":0.0008500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2970122499391437}
{"item_id":"semeval2013.d006.s016.t005","gold_sense_keys":["action%1:04:02::"],"candidates":[{"index":1,"sense_key":"action%1:04:02::","synset_id":"action.n.01"},{"index":2,"sense_key":"action%1:26:00::","synset_id":"action.n.02"},{"index":3,"sense_key":"action%1:04:03::","synset_id":"military_action.n.01"},{"index":4,"sense_key":"action%1:22:00::","synset_id":"natural_process.n.01"},{"index":5,"sense_key":"action%1:10:00::","synset_id":"action.n.05"},{"index":6,"sense_key":"action%1:07:00::","synset_id":"action.n.06"},{"index":7,"sense_key":"action%1:06:00::","synset_id":"action.n.07"},{"index":8,"sense_key":"action%1:04:01::","synset_id":"legal_action.n.01"},{"index":9,"sense_key":"action%1:04:04::","synset_id":"action.n.09"},{"index":10,"sense_key":"action%1:04:00::","synset_id":"action.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"action%1:04:02::","call_ids":["semeval2013.d006.s016.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"action%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1201,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002522,"input_uncached_usd":0.002402,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":4.666800666018389}
{"item_id":"semeval2013.d006.s016.t006","gold_sense_keys":["team%1:14:00::"],"candidates":[{"index":1,"sense_key":"team%1:14:00::","synset_id":"team.n.01"},{"index":2,"sense_key":"team%1:14:01::","synset_id":"team.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"team%1:14:00::","call_ids":["semeval2013.d006.s016.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"team%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":655,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00143,"input_uncached_usd":0.00131,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":8.773782249889337}
{"item_id":"semeval2013.d006.s016.t007","gold_sense_keys":["player%1:18:01::"],"candidates":[{"index":1,"sense_key":"player%1:18:01::","synset_id":"player.n.01"},{"index":2,"sense_key":"player%1:18:02::","synset_id":"musician.n.01"},{"index":3,"sense_key":"player%1:18:00::","synset_id":"actor.n.01"},{"index":4,"sense_key":"player%1:18:04::","synset_id":"player.n.04"},{"index":5,"sense_key":"player%1:18:03::","synset_id":"player.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"player%1:18:01::","call_ids":["semeval2013.d006.s016.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"player%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":813,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001746,"input_uncached_usd":0.0016259999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1078458749689162}
{"item_id":"semeval2013.d006.s017.t000","gold_sense_keys":["strategy%1:09:00::"],"candidates":[{"index":1,"sense_key":"strategy%1:09:00::","synset_id":"scheme.n.01"},{"index":2,"sense_key":"strategy%1:09:01::","synset_id":"strategy.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"strategy%1:09:00::","call_ids":["semeval2013.d006.s017.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"strategy%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":661,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0014420000000000001,"input_uncached_usd":0.001322,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.023546499898657}
{"item_id":"semeval2013.d006.s017.t001","gold_sense_keys":["game%1:04:03::"],"candidates":[{"index":1,"sense_key":"game%1:04:00::","synset_id":"game.n.01"},{"index":2,"sense_key":"game%1:04:03::","synset_id":"game.n.02"},{"index":3,"sense_key":"game%1:04:01::","synset_id":"game.n.03"},{"index":4,"sense_key":"game%1:05:00::","synset_id":"game.n.04"},{"index":5,"sense_key":"game%1:28:00::","synset_id":"game.n.05"},{"index":6,"sense_key":"game%1:23:00::","synset_id":"game.n.06"},{"index":7,"sense_key":"game%1:13:00::","synset_id":"game.n.07"},{"index":8,"sense_key":"game%1:09:00::","synset_id":"plot.n.01"},{"index":9,"sense_key":"game%1:06:00::","synset_id":"game.n.09"},{"index":10,"sense_key":"game%1:04:02::","synset_id":"game.n.10"},{"index":11,"sense_key":"game%1:04:04::","synset_id":"game.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"game%1:04:03::","call_ids":["semeval2013.d006.s017.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"game%1:04:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1120,"cached_input_tokens":0,"output_tokens":47,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0027099999999999997,"input_uncached_usd":0.00224,"input_cached_usd":0.0,"output_usd":0.00047000000000000004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":11.825468790950254}
{"item_id":"semeval2013.d006.s017.t003","gold_sense_keys":["player%1:18:01::"],"candidates":[{"index":1,"sense_key":"player%1:18:01::","synset_id":"player.n.01"},{"index":2,"sense_key":"player%1:18:02::","synset_id":"musician.n.01"},{"index":3,"sense_key":"player%1:18:00::","synset_id":"actor.n.01"},{"index":4,"sense_key":"player%1:18:04::","synset_id":"player.n.04"},{"index":5,"sense_key":"player%1:18:03::","synset_id":"player.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"player%1:18:01::","call_ids":["semeval2013.d006.s017.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"player%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":802,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001724,"input_uncached_usd":0.001604,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":8.92012466606684}
{"item_id":"semeval2013.d006.s018.t000","gold_sense_keys":["advantage%1:07:00::","advantage%1:07:01::"],"candidates":[{"index":1,"sense_key":"advantage%1:07:00::","synset_id":"advantage.n.01"},{"index":2,"sense_key":"advantage%1:23:00::","synset_id":"advantage.n.02"},{"index":3,"sense_key":"advantage%1:07:01::","synset_id":"advantage.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"advantage%1:07:01::","call_ids":["semeval2013.d006.s018.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"advantage%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":766,"cached_input_tokens":0,"output_tokens":91,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002442,"input_uncached_usd":0.001532,"input_cached_usd":0.0,"output_usd":0.0009100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":5.416143042035401}
{"item_id":"semeval2013.d006.s018.t001","gold_sense_keys":["fact%1:09:01::"],"candidates":[{"index":1,"sense_key":"fact%1:09:01::","synset_id":"fact.n.01"},{"index":2,"sense_key":"fact%1:10:01::","synset_id":"fact.n.02"},{"index":3,"sense_key":"fact%1:26:01::","synset_id":"fact.n.03"},{"index":4,"sense_key":"fact%1:09:02::","synset_id":"fact.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"fact%1:09:01::","call_ids":["semeval2013.d006.s018.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"fact%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":839,"cached_input_tokens":0,"output_tokens":213,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0038080000000000006,"input_uncached_usd":0.001678,"input_cached_usd":0.0,"output_usd":0.0021300000000000004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.9210144999669865}
{"item_id":"semeval2013.d006.s018.t002","gold_sense_keys":["player%1:18:01::"],"candidates":[{"index":1,"sense_key":"player%1:18:01::","synset_id":"player.n.01"},{"index":2,"sense_key":"player%1:18:02::","synset_id":"musician.n.01"},{"index":3,"sense_key":"player%1:18:00::","synset_id":"actor.n.01"},{"index":4,"sense_key":"player%1:18:04::","synset_id":"player.n.04"},{"index":5,"sense_key":"player%1:18:03::","synset_id":"player.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"player%1:18:01::","call_ids":["semeval2013.d006.s018.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"player%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":833,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001786,"input_uncached_usd":0.001666,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3826782499672845}
{"item_id":"semeval2013.d006.s018.t003","gold_sense_keys":["phase%1:28:00::"],"candidates":[{"index":1,"sense_key":"phase%1:28:00::","synset_id":"phase.n.01"},{"index":2,"sense_key":"phase%1:26:00::","synset_id":"phase.n.02"},{"index":3,"sense_key":"phase%1:28:01::","synset_id":"phase.n.03"},{"index":4,"sense_key":"phase%1:07:00::","synset_id":"phase.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"phase%1:28:00::","call_ids":["semeval2013.d006.s018.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"phase%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":921,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0019619999999999998,"input_uncached_usd":0.0018419999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1436641670297831}
{"item_id":"semeval2013.d006.s018.t004","gold_sense_keys":["championship%1:11:00::"],"candidates":[{"index":1,"sense_key":"championship%1:26:00::","synset_id":"championship.n.01"},{"index":2,"sense_key":"championship%1:11:00::","synset_id":"championship.n.02"},{"index":3,"sense_key":"championship%1:04:00::","synset_id":"backing.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"championship%1:11:00::","call_ids":["semeval2013.d006.s018.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"championship%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":754,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002158,"input_uncached_usd":0.001508,"input_cached_usd":0.0,"output_usd":0.0006500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.403531082905829}
{"item_id":"semeval2013.d006.s018.t006","gold_sense_keys":["participation%1:04:00::"],"candidates":[{"index":1,"sense_key":"participation%1:04:00::","synset_id":"engagement.n.07"},{"index":2,"sense_key":"participation%1:26:00::","synset_id":"participation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"participation%1:04:00::","call_ids":["semeval2013.d006.s018.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"participation%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":727,"cached_input_tokens":0,"output_tokens":82,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002274,"input_uncached_usd":0.001454,"input_cached_usd":0.0,"output_usd":0.0008200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5128763749962673}
{"item_id":"semeval2013.d006.s018.t007","gold_sense_keys":["game%1:04:03::"],"candidates":[{"index":1,"sense_key":"game%1:04:00::","synset_id":"game.n.01"},{"index":2,"sense_key":"game%1:04:03::","synset_id":"game.n.02"},{"index":3,"sense_key":"game%1:04:01::","synset_id":"game.n.03"},{"index":4,"sense_key":"game%1:05:00::","synset_id":"game.n.04"},{"index":5,"sense_key":"game%1:28:00::","synset_id":"game.n.05"},{"index":6,"sense_key":"game%1:23:00::","synset_id":"game.n.06"},{"index":7,"sense_key":"game%1:13:00::","synset_id":"game.n.07"},{"index":8,"sense_key":"game%1:09:00::","synset_id":"plot.n.01"},{"index":9,"sense_key":"game%1:06:00::","synset_id":"game.n.09"},{"index":10,"sense_key":"game%1:04:02::","synset_id":"game.n.10"},{"index":11,"sense_key":"game%1:04:04::","synset_id":"game.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"game%1:04:03::","call_ids":["semeval2013.d006.s018.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"game%1:04:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1151,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003022,"input_uncached_usd":0.002302,"input_cached_usd":0.0,"output_usd":0.00072,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.599042540998198}
{"item_id":"semeval2013.d006.s019.t000","gold_sense_keys":["term%1:28:00::"],"candidates":[{"index":1,"sense_key":"term%1:10:00::","synset_id":"term.n.01"},{"index":2,"sense_key":"term%1:28:00::","synset_id":"term.n.02"},{"index":3,"sense_key":"term%1:10:02::","synset_id":"condition.n.07"},{"index":4,"sense_key":"term%1:09:00::","synset_id":"term.n.04"},{"index":5,"sense_key":"term%1:10:01::","synset_id":"term.n.05"},{"index":6,"sense_key":"term%1:28:01::","synset_id":"term.n.06"},{"index":7,"sense_key":"term%1:06:00::","synset_id":"terminus.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"term%1:28:00::","call_ids":["semeval2013.d006.s019.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"term%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":952,"cached_input_tokens":0,"output_tokens":41,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002314,"input_uncached_usd":0.0019039999999999999,"input_cached_usd":0.0,"output_usd":0.00041000000000000005,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4247530410066247}
{"item_id":"semeval2013.d006.s019.t001","gold_sense_keys":["advantage%1:07:00::","advantage%1:07:01::"],"candidates":[{"index":1,"sense_key":"advantage%1:07:00::","synset_id":"advantage.n.01"},{"index":2,"sense_key":"advantage%1:23:00::","synset_id":"advantage.n.02"},{"index":3,"sense_key":"advantage%1:07:01::","synset_id":"advantage.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"advantage%1:07:01::","call_ids":["semeval2013.d006.s019.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"advantage%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":699,"cached_input_tokens":0,"output_tokens":154,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002938,"input_uncached_usd":0.001398,"input_cached_usd":0.0,"output_usd":0.0015400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8539385420735925}
{"item_id":"semeval2013.d006.s019.t002","gold_sense_keys":["degree%1:07:00::","degree%1:26:01::"],"candidates":[{"index":1,"sense_key":"degree%1:07:00::","synset_id":"degree.n.01"},{"index":2,"sense_key":"degree%1:26:01::","synset_id":"degree.n.02"},{"index":3,"sense_key":"degree%1:10:00::","synset_id":"academic_degree.n.01"},{"index":4,"sense_key":"degree%1:23:00::","synset_id":"degree.n.04"},{"index":5,"sense_key":"degree%1:09:00::","synset_id":"degree.n.05"},{"index":6,"sense_key":"degree%1:23:03::","synset_id":"degree.n.06"},{"index":7,"sense_key":"degree%1:07:01::","synset_id":"degree.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"degree%1:07:00::","call_ids":["semeval2013.d006.s019.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"degree%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":975,"cached_input_tokens":0,"output_tokens":197,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00392,"input_uncached_usd":0.00195,"input_cached_usd":0.0,"output_usd":0.00197,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.06613687495701}
{"item_id":"semeval2013.d006.s019.t004","gold_sense_keys":["team%1:14:00::"],"candidates":[{"index":1,"sense_key":"team%1:14:00::","synset_id":"team.n.01"},{"index":2,"sense_key":"team%1:14:01::","synset_id":"team.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"team%1:14:00::","call_ids":["semeval2013.d006.s019.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"team%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":608,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001336,"input_uncached_usd":0.0012159999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.977553749922663}
{"item_id":"semeval2013.d006.s019.t005","gold_sense_keys":["change%1:11:00::"],"candidates":[{"index":1,"sense_key":"change%1:11:00::","synset_id":"change.n.01"},{"index":2,"sense_key":"change%1:24:00::","synset_id":"change.n.02"},{"index":3,"sense_key":"change%1:04:00::","synset_id":"change.n.03"},{"index":4,"sense_key":"change%1:19:00::","synset_id":"change.n.04"},{"index":5,"sense_key":"change%1:21:02::","synset_id":"change.n.05"},{"index":6,"sense_key":"change%1:06:01::","synset_id":"change.n.06"},{"index":7,"sense_key":"change%1:06:00::","synset_id":"change.n.07"},{"index":8,"sense_key":"change%1:21:01::","synset_id":"change.n.08"},{"index":9,"sense_key":"change%1:21:03::","synset_id":"change.n.09"},{"index":10,"sense_key":"change%1:07:00::","synset_id":"variety.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"change%1:11:00::","call_ids":["semeval2013.d006.s019.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"change%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1114,"cached_input_tokens":0,"output_tokens":180,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004028,"input_uncached_usd":0.002228,"input_cached_usd":0.0,"output_usd":0.0018000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.6258448750013486}
{"item_id":"semeval2013.d006.s019.t007","gold_sense_keys":["action%1:04:02::"],"candidates":[{"index":1,"sense_key":"action%1:04:02::","synset_id":"action.n.01"},{"index":2,"sense_key":"action%1:26:00::","synset_id":"action.n.02"},{"index":3,"sense_key":"action%1:04:03::","synset_id":"military_action.n.01"},{"index":4,"sense_key":"action%1:22:00::","synset_id":"natural_process.n.01"},{"index":5,"sense_key":"action%1:10:00::","synset_id":"action.n.05"},{"index":6,"sense_key":"action%1:07:00::","synset_id":"action.n.06"},{"index":7,"sense_key":"action%1:06:00::","synset_id":"action.n.07"},{"index":8,"sense_key":"action%1:04:01::","synset_id":"legal_action.n.01"},{"index":9,"sense_key":"action%1:04:04::","synset_id":"action.n.09"},{"index":10,"sense_key":"action%1:04:00::","synset_id":"action.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"action%1:04:02::","call_ids":["semeval2013.d006.s019.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"action%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1154,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002908,"input_uncached_usd":0.0023079999999999997,"input_cached_usd":0.0,"output_usd":0.0006000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4559699170058593}
{"item_id":"semeval2013.d006.s019.t008","gold_sense_keys":["player%1:18:01::"],"candidates":[{"index":1,"sense_key":"player%1:18:01::","synset_id":"player.n.01"},{"index":2,"sense_key":"player%1:18:02::","synset_id":"musician.n.01"},{"index":3,"sense_key":"player%1:18:00::","synset_id":"actor.n.01"},{"index":4,"sense_key":"player%1:18:04::","synset_id":"player.n.04"},{"index":5,"sense_key":"player%1:18:03::","synset_id":"player.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"player%1:18:01::","call_ids":["semeval2013.d006.s019.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"player%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":766,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001652,"input_uncached_usd":0.001532,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1052874999586493}
{"item_id":"semeval2013.d006.s019.t010","gold_sense_keys":["game%1:04:03::"],"candidates":[{"index":1,"sense_key":"game%1:04:00::","synset_id":"game.n.01"},{"index":2,"sense_key":"game%1:04:03::","synset_id":"game.n.02"},{"index":3,"sense_key":"game%1:04:01::","synset_id":"game.n.03"},{"index":4,"sense_key":"game%1:05:00::","synset_id":"game.n.04"},{"index":5,"sense_key":"game%1:28:00::","synset_id":"game.n.05"},{"index":6,"sense_key":"game%1:23:00::","synset_id":"game.n.06"},{"index":7,"sense_key":"game%1:13:00::","synset_id":"game.n.07"},{"index":8,"sense_key":"game%1:09:00::","synset_id":"plot.n.01"},{"index":9,"sense_key":"game%1:06:00::","synset_id":"game.n.09"},{"index":10,"sense_key":"game%1:04:02::","synset_id":"game.n.10"},{"index":11,"sense_key":"game%1:04:04::","synset_id":"game.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"game%1:04:03::","call_ids":["semeval2013.d006.s019.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"game%1:04:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1084,"cached_input_tokens":0,"output_tokens":88,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003048,"input_uncached_usd":0.002168,"input_cached_usd":0.0,"output_usd":0.00088,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":5.209522875025868}
{"item_id":"semeval2013.d006.s019.t013","gold_sense_keys":["suspension%1:04:01::"],"candidates":[{"index":1,"sense_key":"suspension%1:27:00::","synset_id":"suspension.n.01"},{"index":2,"sense_key":"suspension%1:28:00::","synset_id":"pause.n.01"},{"index":3,"sense_key":"suspension%1:26:00::","synset_id":"abeyance.n.01"},{"index":4,"sense_key":"suspension%1:11:00::","synset_id":"suspension.n.04"},{"index":5,"sense_key":"suspension%1:06:00::","synset_id":"suspension.n.05"},{"index":6,"sense_key":"suspension%1:04:00::","synset_id":"suspension.n.06"},{"index":7,"sense_key":"suspension%1:04:01::","synset_id":"suspension.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"suspension%1:04:01::","call_ids":["semeval2013.d006.s019.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"suspension%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":952,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002584,"input_uncached_usd":0.0019039999999999999,"input_cached_usd":0.0,"output_usd":0.00068,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9761133749270812}
{"item_id":"semeval2013.d006.s019.t015","gold_sense_keys":["accumulation%1:04:00::"],"candidates":[{"index":1,"sense_key":"accumulation%1:22:00::","synset_id":"accretion.n.01"},{"index":2,"sense_key":"accumulation%1:14:00::","synset_id":"collection.n.01"},{"index":3,"sense_key":"accumulation%1:04:00::","synset_id":"accumulation.n.03"},{"index":4,"sense_key":"accumulation%1:21:00::","synset_id":"accumulation.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"accumulation%1:04:00::","call_ids":["semeval2013.d006.s019.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"accumulation%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":732,"cached_input_tokens":0,"output_tokens":104,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002504,"input_uncached_usd":0.001464,"input_cached_usd":0.0,"output_usd":0.0010400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8681404159869999}
{"item_id":"semeval2013.d006.s020.t000","gold_sense_keys":["round%1:28:01::"],"candidates":[{"index":1,"sense_key":"round%1:06:01::","synset_id":"round.n.01"},{"index":2,"sense_key":"round%1:28:01::","synset_id":"cycle.n.01"},{"index":3,"sense_key":"round%1:15:00::","synset_id":"beat.n.01"},{"index":4,"sense_key":"round%1:04:00::","synset_id":"round.n.04"},{"index":5,"sense_key":"round%1:04:03::","synset_id":"round_of_golf.n.01"},{"index":6,"sense_key":"round%1:04:01::","synset_id":"round.n.06"},{"index":7,"sense_key":"round%1:28:00::","synset_id":"turn.n.09"},{"index":8,"sense_key":"round%1:17:00::","synset_id":"round.n.08"},{"index":9,"sense_key":"round%1:13:01::","synset_id":"round.n.09"},{"index":10,"sense_key":"round%1:13:00::","synset_id":"round.n.10"},{"index":11,"sense_key":"round%1:10:00::","synset_id":"round.n.11"},{"index":12,"sense_key":"round%1:10:01::","synset_id":"round.n.12"},{"index":13,"sense_key":"round%1:06:03::","synset_id":"rung.n.01"},{"index":14,"sense_key":"round%1:06:00::","synset_id":"circle.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"round%1:28:01::","call_ids":["semeval2013.d006.s020.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"round%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1339,"cached_input_tokens":0,"output_tokens":178,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004458,"input_uncached_usd":0.002678,"input_cached_usd":0.0,"output_usd":0.0017800000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1110414159484208}
{"item_id":"semeval2013.d006.s020.t001","gold_sense_keys":["advantage%1:07:01::"],"candidates":[{"index":1,"sense_key":"advantage%1:07:00::","synset_id":"advantage.n.01"},{"index":2,"sense_key":"advantage%1:23:00::","synset_id":"advantage.n.02"},{"index":3,"sense_key":"advantage%1:07:01::","synset_id":"advantage.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"advantage%1:07:01::","call_ids":["semeval2013.d006.s020.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"advantage%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":723,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0021060000000000002,"input_uncached_usd":0.001446,"input_cached_usd":0.0,"output_usd":0.0006600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2394315829733387}
{"item_id":"semeval2013.d006.s021.t000","gold_sense_keys":["action%1:04:02::"],"candidates":[{"index":1,"sense_key":"action%1:04:02::","synset_id":"action.n.01"},{"index":2,"sense_key":"action%1:26:00::","synset_id":"action.n.02"},{"index":3,"sense_key":"action%1:04:03::","synset_id":"military_action.n.01"},{"index":4,"sense_key":"action%1:22:00::","synset_id":"natural_process.n.01"},{"index":5,"sense_key":"action%1:10:00::","synset_id":"action.n.05"},{"index":6,"sense_key":"action%1:07:00::","synset_id":"action.n.06"},{"index":7,"sense_key":"action%1:06:00::","synset_id":"action.n.07"},{"index":8,"sense_key":"action%1:04:01::","synset_id":"legal_action.n.01"},{"index":9,"sense_key":"action%1:04:04::","synset_id":"action.n.09"},{"index":10,"sense_key":"action%1:04:00::","synset_id":"action.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"action%1:04:02::","call_ids":["semeval2013.d006.s021.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"action%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1206,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002942,"input_uncached_usd":0.002412,"input_cached_usd":0.0,"output_usd":0.0005300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4570877919904888}
{"item_id":"semeval2013.d006.s021.t002","gold_sense_keys":["part%1:24:00::"],"candidates":[{"index":1,"sense_key":"part%1:24:00::","synset_id":"part.n.01"},{"index":2,"sense_key":"part%1:06:00::","synset_id":"part.n.02"},{"index":3,"sense_key":"part%1:17:00::","synset_id":"part.n.03"},{"index":4,"sense_key":"part%1:09:02::","synset_id":"part.n.04"},{"index":5,"sense_key":"part%1:15:00::","synset_id":"region.n.01"},{"index":6,"sense_key":"part%1:04:01::","synset_id":"function.n.03"},{"index":7,"sense_key":"part%1:09:01::","synset_id":"character.n.04"},{"index":8,"sense_key":"part%1:21:00::","synset_id":"share.n.01"},{"index":9,"sense_key":"part%1:09:00::","synset_id":"part.n.09"},{"index":10,"sense_key":"part%1:08:01::","synset_id":"part.n.10"},{"index":11,"sense_key":"part%1:10:00::","synset_id":"part.n.11"},{"index":12,"sense_key":"part%1:04:03::","synset_id":"contribution.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"part%1:24:00::","call_ids":["semeval2013.d006.s021.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"part%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1392,"cached_input_tokens":0,"output_tokens":115,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003934,"input_uncached_usd":0.002784,"input_cached_usd":0.0,"output_usd":0.0011500000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1436864589340985}
{"item_id":"semeval2013.d006.s021.t003","gold_sense_keys":["decision%1:04:00::"],"candidates":[{"index":1,"sense_key":"decision%1:04:00::","synset_id":"decision.n.01"},{"index":2,"sense_key":"decision%1:09:00::","synset_id":"decision.n.02"},{"index":3,"sense_key":"decision%1:11:02::","synset_id":"decision.n.03"},{"index":4,"sense_key":"decision%1:11:01::","synset_id":"decision.n.04"},{"index":5,"sense_key":"decision%1:07:00::","synset_id":"decisiveness.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"decision%1:04:00::","call_ids":["semeval2013.d006.s021.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"decision%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":920,"cached_input_tokens":0,"output_tokens":126,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0031,"input_uncached_usd":0.0018399999999999998,"input_cached_usd":0.0,"output_usd":0.00126,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.956362667027861}
{"item_id":"semeval2013.d006.s021.t005","gold_sense_keys":["rule%1:10:02::"],"candidates":[{"index":1,"sense_key":"rule%1:09:00::","synset_id":"rule.n.01"},{"index":2,"sense_key":"rule%1:09:01::","synset_id":"convention.n.02"},{"index":3,"sense_key":"rule%1:10:00::","synset_id":"rule.n.03"},{"index":4,"sense_key":"rule%1:10:01::","synset_id":"rule.n.04"},{"index":5,"sense_key":"rule%1:09:03::","synset_id":"principle.n.01"},{"index":6,"sense_key":"rule%1:28:00::","synset_id":"rule.n.06"},{"index":7,"sense_key":"rule%1:26:00::","synset_id":"dominion.n.01"},{"index":8,"sense_key":"rule%1:10:02::","synset_id":"rule.n.08"},{"index":9,"sense_key":"rule%1:10:03::","synset_id":"rule.n.09"},{"index":10,"sense_key":"rule%1:09:02::","synset_id":"principle.n.04"},{"index":11,"sense_key":"rule%1:09:04::","synset_id":"rule.n.11"},{"index":12,"sense_key":"rule%1:06:00::","synset_id":"rule.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"rule%1:10:02::","call_ids":["semeval2013.d006.s021.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"rule%1:10:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1398,"cached_input_tokens":0,"output_tokens":82,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003616,"input_uncached_usd":0.002796,"input_cached_usd":0.0,"output_usd":0.0008200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.53615933295805}
{"item_id":"semeval2013.d006.s022.t000","gold_sense_keys":["assumption%1:09:00::"],"candidates":[{"index":1,"sense_key":"assumption%1:10:00::","synset_id":"premise.n.01"},{"index":2,"sense_key":"assumption%1:09:00::","synset_id":"assumption.n.02"},{"index":3,"sense_key":"assumption%1:04:00::","synset_id":"assumption.n.03"},{"index":4,"sense_key":"assumption%1:28:00::","synset_id":"assumption.n.04"},{"index":5,"sense_key":"assumption%1:11:00::","synset_id":"assumption.n.05"},{"index":6,"sense_key":"assumption%1:07:00::","synset_id":"presumption.n.03"},{"index":7,"sense_key":"assumption%1:04:01::","synset_id":"assumption.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"assumption%1:09:00::","call_ids":["semeval2013.d006.s022.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"assumption%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1066,"cached_input_tokens":0,"output_tokens":169,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003822,"input_uncached_usd":0.0021319999999999998,"input_cached_usd":0.0,"output_usd":0.00169,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9700993340229616}
{"item_id":"semeval2013.d006.s022.t001","gold_sense_keys":["player%1:18:01::"],"candidates":[{"index":1,"sense_key":"player%1:18:01::","synset_id":"player.n.01"},{"index":2,"sense_key":"player%1:18:02::","synset_id":"musician.n.01"},{"index":3,"sense_key":"player%1:18:00::","synset_id":"actor.n.01"},{"index":4,"sense_key":"player%1:18:04::","synset_id":"player.n.04"},{"index":5,"sense_key":"player%1:18:03::","synset_id":"player.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"player%1:18:01::","call_ids":["semeval2013.d006.s022.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"player%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":771,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001662,"input_uncached_usd":0.001542,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1825462090782821}
{"item_id":"semeval2013.d006.s022.t002","gold_sense_keys":["application%1:04:00::"],"candidates":[{"index":1,"sense_key":"application%1:04:02::","synset_id":"application.n.01"},{"index":2,"sense_key":"application%1:10:00::","synset_id":"application.n.02"},{"index":3,"sense_key":"application%1:04:03::","synset_id":"application.n.03"},{"index":4,"sense_key":"application%1:10:01::","synset_id":"application.n.04"},{"index":5,"sense_key":"application%1:06:00::","synset_id":"lotion.n.02"},{"index":6,"sense_key":"application%1:04:01::","synset_id":"application.n.06"},{"index":7,"sense_key":"application%1:04:00::","synset_id":"application.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"application%1:04:00::","call_ids":["semeval2013.d006.s022.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"application%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1001,"cached_input_tokens":0,"output_tokens":88,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002882,"input_uncached_usd":0.002002,"input_cached_usd":0.0,"output_usd":0.00088,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8769935000455007}
{"item_id":"semeval2013.d006.s022.t004","gold_sense_keys":["case%1:11:00::"],"candidates":[{"index":1,"sense_key":"case%1:11:00::","synset_id":"case.n.01"},{"index":2,"sense_key":"case%1:26:00::","synset_id":"event.n.02"},{"index":3,"sense_key":"case%1:04:00::","synset_id":"lawsuit.n.01"},{"index":4,"sense_key":"case%1:09:00::","synset_id":"case.n.04"},{"index":5,"sense_key":"case%1:06:00::","synset_id":"case.n.05"},{"index":6,"sense_key":"case%1:18:03::","synset_id":"case.n.06"},{"index":7,"sense_key":"case%1:18:02::","synset_id":"subject.n.06"},{"index":8,"sense_key":"case%1:10:02::","synset_id":"case.n.08"},{"index":9,"sense_key":"case%1:10:01::","synset_id":"case.n.09"},{"index":10,"sense_key":"case%1:23:00::","synset_id":"case.n.10"},{"index":11,"sense_key":"case%1:10:00::","synset_id":"case.n.11"},{"index":12,"sense_key":"case%1:26:02::","synset_id":"case.n.12"},{"index":13,"sense_key":"case%1:18:04::","synset_id":"character.n.05"},{"index":14,"sense_key":"case%1:10:03::","synset_id":"font.n.01"},{"index":15,"sense_key":"case%1:08:00::","synset_id":"sheath.n.02"},{"index":16,"sense_key":"case%1:06:06::","synset_id":"shell.n.08"},{"index":17,"sense_key":"case%1:06:07::","synset_id":"casing.n.03"},{"index":18,"sense_key":"case%1:06:03::","synset_id":"case.n.18"},{"index":19,"sense_key":"case%1:06:02::","synset_id":"case.n.19"},{"index":20,"sense_key":"case%1:06:01::","synset_id":"case.n.20"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"case%1:11:00::","call_ids":["semeval2013.d006.s022.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"case%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1789,"cached_input_tokens":0,"output_tokens":41,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003988,"input_uncached_usd":0.003578,"input_cached_usd":0.0,"output_usd":0.00041000000000000005,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7970855419989675}
{"item_id":"semeval2013.d006.s023.t000","gold_sense_keys":["decision%1:04:00::"],"candidates":[{"index":1,"sense_key":"decision%1:04:00::","synset_id":"decision.n.01"},{"index":2,"sense_key":"decision%1:09:00::","synset_id":"decision.n.02"},{"index":3,"sense_key":"decision%1:11:02::","synset_id":"decision.n.03"},{"index":4,"sense_key":"decision%1:11:01::","synset_id":"decision.n.04"},{"index":5,"sense_key":"decision%1:07:00::","synset_id":"decisiveness.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"decision%1:04:00::","call_ids":["semeval2013.d006.s023.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"decision%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":886,"cached_input_tokens":0,"output_tokens":103,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002802,"input_uncached_usd":0.001772,"input_cached_usd":0.0,"output_usd":0.00103,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.461364458897151}
{"item_id":"semeval2013.d006.s024.t002","gold_sense_keys":["law%1:14:00::"],"candidates":[{"index":1,"sense_key":"law%1:14:00::","synset_id":"law.n.01"},{"index":2,"sense_key":"law%1:10:00::","synset_id":"law.n.02"},{"index":3,"sense_key":"law%1:09:00::","synset_id":"law.n.03"},{"index":4,"sense_key":"law%1:09:01::","synset_id":"law.n.04"},{"index":5,"sense_key":"law%1:09:02::","synset_id":"jurisprudence.n.01"},{"index":6,"sense_key":"law%1:04:00::","synset_id":"law.n.06"},{"index":7,"sense_key":"law%1:14:01::","synset_id":"police.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"law%1:14:00::","call_ids":["semeval2013.d006.s024.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"law%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1009,"cached_input_tokens":0,"output_tokens":69,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002708,"input_uncached_usd":0.002018,"input_cached_usd":0.0,"output_usd":0.0006900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9684674999443814}
{"item_id":"semeval2013.d006.s024.t003","gold_sense_keys":["fraud%1:04:01::"],"candidates":[{"index":1,"sense_key":"fraud%1:04:00::","synset_id":"fraud.n.01"},{"index":2,"sense_key":"fraud%1:18:00::","synset_id":"imposter.n.01"},{"index":3,"sense_key":"fraud%1:04:01::","synset_id":"fraud.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"fraud%1:04:00::","call_ids":["semeval2013.d006.s024.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"fraud%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":724,"cached_input_tokens":0,"output_tokens":177,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003218,"input_uncached_usd":0.0014479999999999999,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.093462249962613}
{"item_id":"semeval2013.d006.s024.t004","gold_sense_keys":["occasion%1:11:00::"],"candidates":[{"index":1,"sense_key":"occasion%1:11:00::","synset_id":"juncture.n.01"},{"index":2,"sense_key":"occasion%1:11:01::","synset_id":"affair.n.03"},{"index":3,"sense_key":"occasion%1:16:00::","synset_id":"occasion.n.03"},{"index":4,"sense_key":"occasion%1:28:00::","synset_id":"occasion.n.04"},{"index":5,"sense_key":"occasion%1:26:00::","synset_id":"occasion.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"occasion%1:28:00::","call_ids":["semeval2013.d006.s024.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"occasion%1:28:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":838,"cached_input_tokens":0,"output_tokens":70,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002376,"input_uncached_usd":0.001676,"input_cached_usd":0.0,"output_usd":0.0007000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.374442166998051}
{"item_id":"semeval2013.d006.s024.t005","gold_sense_keys":["case%1:11:00::"],"candidates":[{"index":1,"sense_key":"case%1:11:00::","synset_id":"case.n.01"},{"index":2,"sense_key":"case%1:26:00::","synset_id":"event.n.02"},{"index":3,"sense_key":"case%1:04:00::","synset_id":"lawsuit.n.01"},{"index":4,"sense_key":"case%1:09:00::","synset_id":"case.n.04"},{"index":5,"sense_key":"case%1:06:00::","synset_id":"case.n.05"},{"index":6,"sense_key":"case%1:18:03::","synset_id":"case.n.06"},{"index":7,"sense_key":"case%1:18:02::","synset_id":"subject.n.06"},{"index":8,"sense_key":"case%1:10:02::","synset_id":"case.n.08"},{"index":9,"sense_key":"case%1:10:01::","synset_id":"case.n.09"},{"index":10,"sense_key":"case%1:23:00::","synset_id":"case.n.10"},{"index":11,"sense_key":"case%1:10:00::","synset_id":"case.n.11"},{"index":12,"sense_key":"case%1:26:02::","synset_id":"case.n.12"},{"index":13,"sense_key":"case%1:18:04::","synset_id":"character.n.05"},{"index":14,"sense_key":"case%1:10:03::","synset_id":"font.n.01"},{"index":15,"sense_key":"case%1:08:00::","synset_id":"sheath.n.02"},{"index":16,"sense_key":"case%1:06:06::","synset_id":"shell.n.08"},{"index":17,"sense_key":"case%1:06:07::","synset_id":"casing.n.03"},{"index":18,"sense_key":"case%1:06:03::","synset_id":"case.n.18"},{"index":19,"sense_key":"case%1:06:02::","synset_id":"case.n.19"},{"index":20,"sense_key":"case%1:06:01::","synset_id":"case.n.20"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"case%1:11:00::","call_ids":["semeval2013.d006.s024.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"case%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1801,"cached_input_tokens":0,"output_tokens":49,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004092,"input_uncached_usd":0.0036019999999999997,"input_cached_usd":0.0,"output_usd":0.0004900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":14.374682084075175}
{"item_id":"semeval2013.d006.s025.t001","gold_sense_keys":["factor%1:11:00::"],"candidates":[{"index":1,"sense_key":"factor%1:11:00::","synset_id":"factor.n.01"},{"index":2,"sense_key":"factor%1:09:00::","synset_id":"component.n.01"},{"index":3,"sense_key":"factor%1:23:00::","synset_id":"divisor.n.01"},{"index":4,"sense_key":"factor%1:18:00::","synset_id":"agent.n.04"},{"index":5,"sense_key":"factor%1:23:01::","synset_id":"factor.n.05"},{"index":6,"sense_key":"factor%1:09:01::","synset_id":"factor.n.06"},{"index":7,"sense_key":"factor%1:08:00::","synset_id":"gene.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"factor%1:11:00::","call_ids":["semeval2013.d006.s025.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"factor%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":922,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001964,"input_uncached_usd":0.001844,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":14.00565262499731}
{"item_id":"semeval2013.d006.s025.t002","gold_sense_keys":["expulsion%1:04:01::"],"candidates":[{"index":1,"sense_key":"expulsion%1:04:01::","synset_id":"ejection.n.02"},{"index":2,"sense_key":"expulsion%1:04:02::","synset_id":"extrusion.n.02"},{"index":3,"sense_key":"expulsion%1:04:00::","synset_id":"expulsion.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"expulsion%1:04:01::","call_ids":["semeval2013.d006.s025.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"expulsion%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":716,"cached_input_tokens":0,"output_tokens":84,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0022719999999999997,"input_uncached_usd":0.001432,"input_cached_usd":0.0,"output_usd":0.00084,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0154167918954045}
{"item_id":"semeval2013.d006.s025.t003","gold_sense_keys":["way%1:04:01::","way%1:07:01::"],"candidates":[{"index":1,"sense_key":"way%1:07:01::","synset_id":"manner.n.01"},{"index":2,"sense_key":"way%1:04:01::","synset_id":"means.n.01"},{"index":3,"sense_key":"way%1:15:00::","synset_id":"direction.n.01"},{"index":4,"sense_key":"way%1:26:00::","synset_id":"way.n.04"},{"index":5,"sense_key":"way%1:04:00::","synset_id":"way.n.05"},{"index":6,"sense_key":"way%1:06:00::","synset_id":"way.n.06"},{"index":7,"sense_key":"way%1:04:02::","synset_id":"way.n.07"},{"index":8,"sense_key":"way%1:23:00::","synset_id":"room.n.02"},{"index":9,"sense_key":"way%1:07:02::","synset_id":"way.n.09"},{"index":10,"sense_key":"way%1:09:00::","synset_id":"way.n.10"},{"index":11,"sense_key":"way%1:09:01::","synset_id":"way.n.11"},{"index":12,"sense_key":"way%1:21:00::","synset_id":"way.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"way%1:07:01::","call_ids":["semeval2013.d006.s025.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"way%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1106,"cached_input_tokens":0,"output_tokens":49,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002702,"input_uncached_usd":0.002212,"input_cached_usd":0.0,"output_usd":0.0004900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3542863330803812}
{"item_id":"semeval2013.d006.s025.t004","gold_sense_keys":["transmission%1:04:01::"],"candidates":[{"index":1,"sense_key":"transmission%1:04:01::","synset_id":"transmission.n.01"},{"index":2,"sense_key":"transmission%1:10:00::","synset_id":"transmission.n.02"},{"index":3,"sense_key":"transmission%1:23:00::","synset_id":"transmittance.n.01"},{"index":4,"sense_key":"transmission%1:11:00::","synset_id":"infection.n.04"},{"index":5,"sense_key":"transmission%1:06:01::","synset_id":"transmission.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"transmission%1:04:01::","call_ids":["semeval2013.d006.s025.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"transmission%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":742,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002014,"input_uncached_usd":0.0014839999999999999,"input_cached_usd":0.0,"output_usd":0.0005300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6192685409914702}
{"item_id":"semeval2013.d006.s026.t000","gold_sense_keys":["damage%1:11:00::"],"candidates":[{"index":1,"sense_key":"damage%1:11:00::","synset_id":"damage.n.01"},{"index":2,"sense_key":"damage%1:11:01::","synset_id":"damage.n.02"},{"index":3,"sense_key":"damage%1:04:00::","synset_id":"damage.n.03"},{"index":4,"sense_key":"damage%1:21:00::","synset_id":"price.n.02"},{"index":5,"sense_key":"damage%1:04:01::","synset_id":"wrong.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"damage%1:11:00::","call_ids":["semeval2013.d006.s026.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"damage%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":721,"cached_input_tokens":0,"output_tokens":95,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002392,"input_uncached_usd":0.001442,"input_cached_usd":0.0,"output_usd":0.0009500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6785696670413017}
{"item_id":"semeval2013.d006.s026.t001","gold_sense_keys":["principle%1:09:00::"],"candidates":[{"index":1,"sense_key":"principle%1:09:03::","synset_id":"principle.n.01"},{"index":2,"sense_key":"principle%1:09:01::","synset_id":"principle.n.02"},{"index":3,"sense_key":"principle%1:09:00::","synset_id":"principle.n.03"},{"index":4,"sense_key":"principle%1:09:04::","synset_id":"principle.n.04"},{"index":5,"sense_key":"principle%1:10:00::","synset_id":"principle.n.05"},{"index":6,"sense_key":"principle%1:09:02::","synset_id":"rationale.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"principle%1:09:00::","call_ids":["semeval2013.d006.s026.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"principle%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":854,"cached_input_tokens":0,"output_tokens":186,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003568,"input_uncached_usd":0.0017079999999999999,"input_cached_usd":0.0,"output_usd":0.00186,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":7.28125295904465}
{"item_id":"semeval2013.d006.s026.t003","gold_sense_keys":["action%1:04:02::"],"candidates":[{"index":1,"sense_key":"action%1:04:02::","synset_id":"action.n.01"},{"index":2,"sense_key":"action%1:26:00::","synset_id":"action.n.02"},{"index":3,"sense_key":"action%1:04:03::","synset_id":"military_action.n.01"},{"index":4,"sense_key":"action%1:22:00::","synset_id":"natural_process.n.01"},{"index":5,"sense_key":"action%1:10:00::","synset_id":"action.n.05"},{"index":6,"sense_key":"action%1:07:00::","synset_id":"action.n.06"},{"index":7,"sense_key":"action%1:06:00::","synset_id":"action.n.07"},{"index":8,"sense_key":"action%1:04:01::","synset_id":"legal_action.n.01"},{"index":9,"sense_key":"action%1:04:04::","synset_id":"action.n.09"},{"index":10,"sense_key":"action%1:04:00::","synset_id":"action.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"action%1:04:02::","call_ids":["semeval2013.d006.s026.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"action%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1088,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002296,"input_uncached_usd":0.002176,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":16.768017833004706}
{"item_id":"semeval2013.d007.s000.t001","gold_sense_keys":["possibility%1:26:00::"],"candidates":[{"index":1,"sense_key":"possibility%1:09:03::","synset_id":"possibility.n.01"},{"index":2,"sense_key":"possibility%1:26:00::","synset_id":"possibility.n.02"},{"index":3,"sense_key":"possibility%1:09:00::","synset_id":"hypothesis.n.02"},{"index":4,"sense_key":"possibility%1:09:01::","synset_id":"possibility.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"possibility%1:26:00::","call_ids":["semeval2013.d007.s000.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"possibility%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":618,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0018059999999999999,"input_uncached_usd":0.001236,"input_cached_usd":0.0,"output_usd":0.0005700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":8.487459875061177}
{"item_id":"semeval2013.d007.s000.t002","gold_sense_keys":["type%1:09:00::"],"candidates":[{"index":1,"sense_key":"type%1:09:00::","synset_id":"type.n.01"},{"index":2,"sense_key":"type%1:18:00::","synset_id":"character.n.05"},{"index":3,"sense_key":"type%1:14:01::","synset_id":"type.n.03"},{"index":4,"sense_key":"type%1:10:01::","synset_id":"type.n.04"},{"index":5,"sense_key":"type%1:10:00::","synset_id":"type.n.05"},{"index":6,"sense_key":"type%1:06:00::","synset_id":"type.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"type%1:09:00::","call_ids":["semeval2013.d007.s000.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"type%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":641,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001402,"input_uncached_usd":0.001282,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0636700419709086}
{"item_id":"semeval2013.d007.s001.t000","gold_sense_keys":["discovery%1:04:00::"],"candidates":[{"index":1,"sense_key":"discovery%1:04:00::","synset_id":"discovery.n.01"},{"index":2,"sense_key":"discovery%1:10:00::","synset_id":"discovery.n.02"},{"index":3,"sense_key":"discovery%1:09:00::","synset_id":"discovery.n.03"},{"index":4,"sense_key":"discovery%1:10:01::","synset_id":"discovery.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"discovery%1:04:00::","call_ids":["semeval2013.d007.s001.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"discovery%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":554,"cached_input_tokens":0,"output_tokens":102,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002128,"input_uncached_usd":0.001108,"input_cached_usd":0.0,"output_usd":0.00102,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.21738120901864}
{"item_id":"semeval2013.d007.s001.t002","gold_sense_keys":["arsenic%1:27:00::"],"candidates":[{"index":1,"sense_key":"arsenic%1:27:01::","synset_id":"arsenic.n.01"},{"index":2,"sense_key":"arsenic%1:27:00::","synset_id":"arsenic.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"arsenic%1:27:00::","call_ids":["semeval2013.d007.s001.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"arsenic%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":588,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001726,"input_uncached_usd":0.001176,"input_cached_usd":0.0,"output_usd":0.00055,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.8645760839572176}
{"item_id":"semeval2013.d007.s001.t003","gold_sense_keys":["nutrient%1:03:00::"],"candidates":[{"index":1,"sense_key":"nutrient%1:03:00::","synset_id":"food.n.01"},{"index":2,"sense_key":"nutrient%1:03:01::","synset_id":"nutrient.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"nutrient%1:03:01::","call_ids":["semeval2013.d007.s001.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"nutrient%1:03:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":486,"cached_input_tokens":0,"output_tokens":183,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0028020000000000002,"input_uncached_usd":0.000972,"input_cached_usd":0.0,"output_usd":0.0018300000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.518725667032413}
{"item_id":"semeval2013.d007.s001.t004","gold_sense_keys":["scope%1:07:00::"],"candidates":[{"index":1,"sense_key":"scope%1:07:00::","synset_id":"scope.n.01"},{"index":2,"sense_key":"scope%1:26:00::","synset_id":"setting.n.02"},{"index":3,"sense_key":"scope%1:06:02::","synset_id":"telescope.n.01"},{"index":4,"sense_key":"scope%1:06:01::","synset_id":"oscilloscope.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"scope%1:07:00::","call_ids":["semeval2013.d007.s001.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"scope%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":626,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001372,"input_uncached_usd":0.0012519999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2333404159871861}
{"item_id":"semeval2013.d007.s001.t005","gold_sense_keys":["form%1:09:01::"],"candidates":[{"index":1,"sense_key":"form%1:10:00::","synset_id":"form.n.01"},{"index":2,"sense_key":"form%1:09:01::","synset_id":"kind.n.01"},{"index":3,"sense_key":"form%1:09:00::","synset_id":"form.n.03"},{"index":4,"sense_key":"form%1:07:01::","synset_id":"shape.n.01"},{"index":5,"sense_key":"form%1:08:00::","synset_id":"human_body.n.01"},{"index":6,"sense_key":"form%1:03:00::","synset_id":"shape.n.02"},{"index":7,"sense_key":"form%1:07:00::","synset_id":"form.n.07"},{"index":8,"sense_key":"form%1:10:01::","synset_id":"form.n.08"},{"index":9,"sense_key":"form%1:14:02::","synset_id":"form.n.09"},{"index":10,"sense_key":"form%1:10:02::","synset_id":"form.n.10"},{"index":11,"sense_key":"form%1:07:03::","synset_id":"form.n.11"},{"index":12,"sense_key":"form%1:26:00::","synset_id":"phase.n.02"},{"index":13,"sense_key":"form%1:14:00::","synset_id":"class.n.02"},{"index":14,"sense_key":"form%1:07:02::","synset_id":"form.n.14"},{"index":15,"sense_key":"form%1:06:00::","synset_id":"mannequin.n.02"},{"index":16,"sense_key":"form%1:06:01::","synset_id":"form.n.16"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"form%1:09:01::","call_ids":["semeval2013.d007.s001.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"form%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1437,"cached_input_tokens":0,"output_tokens":81,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0036839999999999998,"input_uncached_usd":0.002874,"input_cached_usd":0.0,"output_usd":0.0008100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.569272166932933}
{"item_id":"semeval2013.d007.s002.t002","gold_sense_keys":["earth%1:17:00::"],"candidates":[{"index":1,"sense_key":"earth%1:17:00::","synset_id":"earth.n.01"},{"index":2,"sense_key":"earth%1:27:00::","synset_id":"earth.n.02"},{"index":3,"sense_key":"earth%1:17:01::","synset_id":"land.n.04"},{"index":4,"sense_key":"earth%1:15:00::","synset_id":"earth.n.04"},{"index":5,"sense_key":"earth%1:27:01::","synset_id":"earth.n.05"},{"index":6,"sense_key":"earth%1:09:00::","synset_id":"worldly_concern.n.01"},{"index":7,"sense_key":"earth%1:06:00::","synset_id":"ground.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"earth%1:17:00::","call_ids":["semeval2013.d007.s002.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"earth%1:17:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":844,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001808,"input_uncached_usd":0.0016879999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2886426250915974}
{"item_id":"semeval2013.d007.s002.t003","gold_sense_keys":["life%1:19:00::"],"candidates":[{"index":1,"sense_key":"life%1:26:01::","synset_id":"life.n.01"},{"index":2,"sense_key":"life%1:09:00::","synset_id":"life.n.02"},{"index":3,"sense_key":"life%1:26:02::","synset_id":"life.n.03"},{"index":4,"sense_key":"life%1:26:00::","synset_id":"animation.n.01"},{"index":5,"sense_key":"life%1:28:00::","synset_id":"life.n.05"},{"index":6,"sense_key":"life%1:28:01::","synset_id":"life.n.06"},{"index":7,"sense_key":"life%1:28:02::","synset_id":"life.n.07"},{"index":8,"sense_key":"life%1:18:00::","synset_id":"life.n.08"},{"index":9,"sense_key":"life%1:07:01::","synset_id":"liveliness.n.02"},{"index":10,"sense_key":"life%1:03:00::","synset_id":"life.n.10"},{"index":11,"sense_key":"life%1:19:00::","synset_id":"life.n.11"},{"index":12,"sense_key":"life%1:10:00::","synset_id":"biography.n.01"},{"index":13,"sense_key":"life%1:16:00::","synset_id":"life.n.13"},{"index":14,"sense_key":"life%1:28:03::","synset_id":"life_sentence.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"life%1:19:00::","call_ids":["semeval2013.d007.s002.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"life%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1134,"cached_input_tokens":0,"output_tokens":194,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004208,"input_uncached_usd":0.002268,"input_cached_usd":0.0,"output_usd":0.00194,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.7239916670368984}
{"item_id":"semeval2013.d007.s002.t004","gold_sense_keys":["possibility%1:09:03::"],"candidates":[{"index":1,"sense_key":"possibility%1:09:03::","synset_id":"possibility.n.01"},{"index":2,"sense_key":"possibility%1:26:00::","synset_id":"possibility.n.02"},{"index":3,"sense_key":"possibility%1:09:00::","synset_id":"hypothesis.n.02"},{"index":4,"sense_key":"possibility%1:09:01::","synset_id":"possibility.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"possibility%1:09:03::","call_ids":["semeval2013.d007.s002.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"possibility%1:09:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":677,"cached_input_tokens":0,"output_tokens":144,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002794,"input_uncached_usd":0.001354,"input_cached_usd":0.0,"output_usd":0.00144,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1413365829503164}
{"item_id":"semeval2013.d007.s002.t005","gold_sense_keys":["element%1:27:00::"],"candidates":[{"index":1,"sense_key":"element%1:09:00::","synset_id":"component.n.01"},{"index":2,"sense_key":"element%1:06:00::","synset_id":"component.n.03"},{"index":3,"sense_key":"element%1:27:00::","synset_id":"chemical_element.n.01"},{"index":4,"sense_key":"element%1:15:00::","synset_id":"element.n.04"},{"index":5,"sense_key":"element%1:27:01::","synset_id":"element.n.05"},{"index":6,"sense_key":"element%1:26:00::","synset_id":"element.n.06"},{"index":7,"sense_key":"element%1:25:00::","synset_id":"element.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"element%1:27:00::","call_ids":["semeval2013.d007.s002.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"element%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":818,"cached_input_tokens":0,"output_tokens":79,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0024259999999999998,"input_uncached_usd":0.0016359999999999999,"input_cached_usd":0.0,"output_usd":0.00079,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7181287090061232}
{"item_id":"semeval2013.d007.s003.t000","gold_sense_keys":["organism%1:03:00::"],"candidates":[{"index":1,"sense_key":"organism%1:03:00::","synset_id":"organism.n.01"},{"index":2,"sense_key":"organism%1:14:00::","synset_id":"organism.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"organism%1:03:00::","call_ids":["semeval2013.d007.s003.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"organism%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":489,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001098,"input_uncached_usd":0.000978,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.7595211669104174}
{"item_id":"semeval2013.d007.s003.t001","gold_sense_keys":["capability%1:07:00::"],"candidates":[{"index":1,"sense_key":"capability%1:07:00::","synset_id":"capability.n.01"},{"index":2,"sense_key":"capability%1:26:00::","synset_id":"capability.n.02"},{"index":3,"sense_key":"capability%1:09:00::","synset_id":"capability.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"capability%1:07:00::","call_ids":["semeval2013.d007.s003.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"capability%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":565,"cached_input_tokens":0,"output_tokens":107,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0021999999999999997,"input_uncached_usd":0.00113,"input_cached_usd":0.0,"output_usd":0.00107,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7856972500449046}
{"item_id":"semeval2013.d007.s004.t000","gold_sense_keys":["phosphorus%1:27:00::"],"candidates":[{"index":1,"sense_key":"phosphorus%1:27:00::","synset_id":"phosphorus.n.01"},{"index":2,"sense_key":"phosphorus%1:17:00::","synset_id":"morning_star.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"phosphorus%1:27:00::","call_ids":["semeval2013.d007.s004.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"phosphorus%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":654,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001428,"input_uncached_usd":0.001308,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.064750042045489}
{"item_id":"semeval2013.d007.s004.t001","gold_sense_keys":["arsenic%1:27:00::"],"candidates":[{"index":1,"sense_key":"arsenic%1:27:01::","synset_id":"arsenic.n.01"},{"index":2,"sense_key":"arsenic%1:27:00::","synset_id":"arsenic.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"arsenic%1:27:00::","call_ids":["semeval2013.d007.s004.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"arsenic%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":692,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0015040000000000001,"input_uncached_usd":0.001384,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.202182749984786}
{"item_id":"semeval2013.d007.s005.t000","gold_sense_keys":["form%1:09:01::"],"candidates":[{"index":1,"sense_key":"form%1:10:00::","synset_id":"form.n.01"},{"index":2,"sense_key":"form%1:09:01::","synset_id":"kind.n.01"},{"index":3,"sense_key":"form%1:09:00::","synset_id":"form.n.03"},{"index":4,"sense_key":"form%1:07:01::","synset_id":"shape.n.01"},{"index":5,"sense_key":"form%1:08:00::","synset_id":"human_body.n.01"},{"index":6,"sense_key":"form%1:03:00::","synset_id":"shape.n.02"},{"index":7,"sense_key":"form%1:07:00::","synset_id":"form.n.07"},{"index":8,"sense_key":"form%1:10:01::","synset_id":"form.n.08"},{"index":9,"sense_key":"form%1:14:02::","synset_id":"form.n.09"},{"index":10,"sense_key":"form%1:10:02::","synset_id":"form.n.10"},{"index":11,"sense_key":"form%1:07:03::","synset_id":"form.n.11"},{"index":12,"sense_key":"form%1:26:00::","synset_id":"phase.n.02"},{"index":13,"sense_key":"form%1:14:00::","synset_id":"class.n.02"},{"index":14,"sense_key":"form%1:07:02::","synset_id":"form.n.14"},{"index":15,"sense_key":"form%1:06:00::","synset_id":"mannequin.n.02"},{"index":16,"sense_key":"form%1:06:01::","synset_id":"form.n.16"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"form%1:09:01::","call_ids":["semeval2013.d007.s005.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"form%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1587,"cached_input_tokens":0,"output_tokens":107,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0042439999999999995,"input_uncached_usd":0.0031739999999999997,"input_cached_usd":0.0,"output_usd":0.00107,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1378984580514953}
{"item_id":"semeval2013.d007.s005.t001","gold_sense_keys":["life%1:19:00::"],"candidates":[{"index":1,"sense_key":"life%1:26:01::","synset_id":"life.n.01"},{"index":2,"sense_key":"life%1:09:00::","synset_id":"life.n.02"},{"index":3,"sense_key":"life%1:26:02::","synset_id":"life.n.03"},{"index":4,"sense_key":"life%1:26:00::","synset_id":"animation.n.01"},{"index":5,"sense_key":"life%1:28:00::","synset_id":"life.n.05"},{"index":6,"sense_key":"life%1:28:01::","synset_id":"life.n.06"},{"index":7,"sense_key":"life%1:28:02::","synset_id":"life.n.07"},{"index":8,"sense_key":"life%1:18:00::","synset_id":"life.n.08"},{"index":9,"sense_key":"life%1:07:01::","synset_id":"liveliness.n.02"},{"index":10,"sense_key":"life%1:03:00::","synset_id":"life.n.10"},{"index":11,"sense_key":"life%1:19:00::","synset_id":"life.n.11"},{"index":12,"sense_key":"life%1:10:00::","synset_id":"biography.n.01"},{"index":13,"sense_key":"life%1:16:00::","synset_id":"life.n.13"},{"index":14,"sense_key":"life%1:28:03::","synset_id":"life_sentence.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"life%1:03:00::","call_ids":["semeval2013.d007.s005.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"life%1:03:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1275,"cached_input_tokens":0,"output_tokens":230,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00485,"input_uncached_usd":0.0025499999999999997,"input_cached_usd":0.0,"output_usd":0.0023000000000000004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.5463487500092015}
{"item_id":"semeval2013.d007.s005.t002","gold_sense_keys":["report%1:10:03::"],"candidates":[{"index":1,"sense_key":"report%1:10:03::","synset_id":"report.n.01"},{"index":2,"sense_key":"report%1:10:01::","synset_id":"report.n.02"},{"index":3,"sense_key":"report%1:10:00::","synset_id":"report.n.03"},{"index":4,"sense_key":"report%1:11:00::","synset_id":"report.n.04"},{"index":5,"sense_key":"report%1:10:04::","synset_id":"report_card.n.01"},{"index":6,"sense_key":"report%1:10:02::","synset_id":"composition.n.08"},{"index":7,"sense_key":"report%1:09:00::","synset_id":"reputation.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"report%1:10:03::","call_ids":["semeval2013.d007.s005.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"report%1:10:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":963,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002576,"input_uncached_usd":0.001926,"input_cached_usd":0.0,"output_usd":0.0006500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":10.632122875074856}
{"item_id":"semeval2013.d007.s005.t004","gold_sense_keys":["edition%1:10:02::"],"candidates":[{"index":1,"sense_key":"edition%1:10:02::","synset_id":"edition.n.01"},{"index":2,"sense_key":"edition%1:14:00::","synset_id":"edition.n.02"},{"index":3,"sense_key":"edition%1:10:00::","synset_id":"edition.n.03"},{"index":4,"sense_key":"edition%1:09:00::","synset_id":"version.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"edition%1:10:00::","call_ids":["semeval2013.d007.s005.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"edition%1:10:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":738,"cached_input_tokens":0,"output_tokens":136,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002836,"input_uncached_usd":0.0014759999999999999,"input_cached_usd":0.0,"output_usd":0.00136,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1697591250995174}
{"item_id":"semeval2013.d007.s005.t005","gold_sense_keys":["journal%1:10:00::"],"candidates":[{"index":1,"sense_key":"journal%1:10:01::","synset_id":"diary.n.01"},{"index":2,"sense_key":"journal%1:10:00::","synset_id":"journal.n.02"},{"index":3,"sense_key":"journal%1:21:00::","synset_id":"daybook.n.01"},{"index":4,"sense_key":"journal%1:06:01::","synset_id":"journal.n.04"},{"index":5,"sense_key":"journal%1:06:00::","synset_id":"journal.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"journal%1:10:00::","call_ids":["semeval2013.d007.s005.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"journal%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":727,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001574,"input_uncached_usd":0.001454,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5388055839575827}
{"item_id":"semeval2013.d007.s006.t000","gold_sense_keys":["element%1:27:00::"],"candidates":[{"index":1,"sense_key":"element%1:09:00::","synset_id":"component.n.01"},{"index":2,"sense_key":"element%1:06:00::","synset_id":"component.n.03"},{"index":3,"sense_key":"element%1:27:00::","synset_id":"chemical_element.n.01"},{"index":4,"sense_key":"element%1:15:00::","synset_id":"element.n.04"},{"index":5,"sense_key":"element%1:27:01::","synset_id":"element.n.05"},{"index":6,"sense_key":"element%1:26:00::","synset_id":"element.n.06"},{"index":7,"sense_key":"element%1:25:00::","synset_id":"element.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"element%1:27:00::","call_ids":["semeval2013.d007.s006.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"element%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":976,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002072,"input_uncached_usd":0.001952,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0680167500395328}
{"item_id":"semeval2013.d007.s006.t001","gold_sense_keys":["life%1:19:00::"],"candidates":[{"index":1,"sense_key":"life%1:26:01::","synset_id":"life.n.01"},{"index":2,"sense_key":"life%1:09:00::","synset_id":"life.n.02"},{"index":3,"sense_key":"life%1:26:02::","synset_id":"life.n.03"},{"index":4,"sense_key":"life%1:26:00::","synset_id":"animation.n.01"},{"index":5,"sense_key":"life%1:28:00::","synset_id":"life.n.05"},{"index":6,"sense_key":"life%1:28:01::","synset_id":"life.n.06"},{"index":7,"sense_key":"life%1:28:02::","synset_id":"life.n.07"},{"index":8,"sense_key":"life%1:18:00::","synset_id":"life.n.08"},{"index":9,"sense_key":"life%1:07:01::","synset_id":"liveliness.n.02"},{"index":10,"sense_key":"life%1:03:00::","synset_id":"life.n.10"},{"index":11,"sense_key":"life%1:19:00::","synset_id":"life.n.11"},{"index":12,"sense_key":"life%1:10:00::","synset_id":"biography.n.01"},{"index":13,"sense_key":"life%1:16:00::","synset_id":"life.n.13"},{"index":14,"sense_key":"life%1:28:03::","synset_id":"life_sentence.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"life%1:19:00::","call_ids":["semeval2013.d007.s006.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"life%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1292,"cached_input_tokens":0,"output_tokens":117,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003754,"input_uncached_usd":0.002584,"input_cached_usd":0.0,"output_usd":0.00117,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":5.248913084040396}
{"item_id":"semeval2013.d007.s006.t002","gold_sense_keys":["carbon%1:27:00::"],"candidates":[{"index":1,"sense_key":"carbon%1:27:00::","synset_id":"carbon.n.01"},{"index":2,"sense_key":"carbon%1:27:01::","synset_id":"carbon_paper.n.01"},{"index":3,"sense_key":"carbon%1:06:00::","synset_id":"carbon.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"carbon%1:27:00::","call_ids":["semeval2013.d007.s006.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"carbon%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":750,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0016200000000000001,"input_uncached_usd":0.0015,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9902421250008047}
{"item_id":"semeval2013.d007.s006.t006","gold_sense_keys":["phosphorus%1:27:00::"],"candidates":[{"index":1,"sense_key":"phosphorus%1:27:00::","synset_id":"phosphorus.n.01"},{"index":2,"sense_key":"phosphorus%1:17:00::","synset_id":"morning_star.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"phosphorus%1:27:00::","call_ids":["semeval2013.d007.s006.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"phosphorus%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":718,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001556,"input_uncached_usd":0.001436,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1248875410528854}
{"item_id":"semeval2013.d007.s007.t003","gold_sense_keys":["arsenic%1:27:00::"],"candidates":[{"index":1,"sense_key":"arsenic%1:27:01::","synset_id":"arsenic.n.01"},{"index":2,"sense_key":"arsenic%1:27:00::","synset_id":"arsenic.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"arsenic%1:27:00::","call_ids":["semeval2013.d007.s007.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"arsenic%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":745,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00161,"input_uncached_usd":0.00149,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0148204170400277}
{"item_id":"semeval2013.d007.s007.t004","gold_sense_keys":["phosphorus%1:27:00::"],"candidates":[{"index":1,"sense_key":"phosphorus%1:27:00::","synset_id":"phosphorus.n.01"},{"index":2,"sense_key":"phosphorus%1:17:00::","synset_id":"morning_star.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"phosphorus%1:27:00::","call_ids":["semeval2013.d007.s007.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"phosphorus%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":707,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001534,"input_uncached_usd":0.001414,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0385920000262558}
{"item_id":"semeval2013.d007.s008.t000","gold_sense_keys":["report%1:10:03::"],"candidates":[{"index":1,"sense_key":"report%1:10:03::","synset_id":"report.n.01"},{"index":2,"sense_key":"report%1:10:01::","synset_id":"report.n.02"},{"index":3,"sense_key":"report%1:10:00::","synset_id":"report.n.03"},{"index":4,"sense_key":"report%1:11:00::","synset_id":"report.n.04"},{"index":5,"sense_key":"report%1:10:04::","synset_id":"report_card.n.01"},{"index":6,"sense_key":"report%1:10:02::","synset_id":"composition.n.08"},{"index":7,"sense_key":"report%1:09:00::","synset_id":"reputation.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"report%1:10:03::","call_ids":["semeval2013.d007.s008.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"report%1:10:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":964,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002558,"input_uncached_usd":0.0019279999999999998,"input_cached_usd":0.0,"output_usd":0.00063,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.07534766709432}
{"item_id":"semeval2013.d007.s009.t000","gold_sense_keys":["find%1:09:00::"],"candidates":[{"index":1,"sense_key":"find%1:09:00::","synset_id":"discovery.n.03"},{"index":2,"sense_key":"find%1:04:00::","synset_id":"discovery.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"find%1:04:00::","call_ids":["semeval2013.d007.s009.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"find%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":604,"cached_input_tokens":0,"output_tokens":227,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003478,"input_uncached_usd":0.0012079999999999999,"input_cached_usd":0.0,"output_usd":0.0022700000000000003,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.840807499946095}
{"item_id":"semeval2013.d007.s009.t001","gold_sense_keys":["search%1:04:00::","search%1:04:02::"],"candidates":[{"index":1,"sense_key":"search%1:04:00::","synset_id":"search.n.01"},{"index":2,"sense_key":"search%1:04:02::","synset_id":"search.n.02"},{"index":3,"sense_key":"search%1:22:00::","synset_id":"search.n.03"},{"index":4,"sense_key":"search%1:09:00::","synset_id":"search.n.04"},{"index":5,"sense_key":"search%1:04:01::","synset_id":"search.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"search%1:04:00::","call_ids":["semeval2013.d007.s009.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"search%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":804,"cached_input_tokens":0,"output_tokens":130,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002908,"input_uncached_usd":0.0016079999999999998,"input_cached_usd":0.0,"output_usd":0.0013000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8278210000135005}
{"item_id":"semeval2013.d007.s009.t002","gold_sense_keys":["life%1:19:00::"],"candidates":[{"index":1,"sense_key":"life%1:26:01::","synset_id":"life.n.01"},{"index":2,"sense_key":"life%1:09:00::","synset_id":"life.n.02"},{"index":3,"sense_key":"life%1:26:02::","synset_id":"life.n.03"},{"index":4,"sense_key":"life%1:26:00::","synset_id":"animation.n.01"},{"index":5,"sense_key":"life%1:28:00::","synset_id":"life.n.05"},{"index":6,"sense_key":"life%1:28:01::","synset_id":"life.n.06"},{"index":7,"sense_key":"life%1:28:02::","synset_id":"life.n.07"},{"index":8,"sense_key":"life%1:18:00::","synset_id":"life.n.08"},{"index":9,"sense_key":"life%1:07:01::","synset_id":"liveliness.n.02"},{"index":10,"sense_key":"life%1:03:00::","synset_id":"life.n.10"},{"index":11,"sense_key":"life%1:19:00::","synset_id":"life.n.11"},{"index":12,"sense_key":"life%1:10:00::","synset_id":"biography.n.01"},{"index":13,"sense_key":"life%1:16:00::","synset_id":"life.n.13"},{"index":14,"sense_key":"life%1:28:03::","synset_id":"life_sentence.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"life%1:19:00::","call_ids":["semeval2013.d007.s009.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"life%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1282,"cached_input_tokens":0,"output_tokens":125,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0038139999999999997,"input_uncached_usd":0.002564,"input_cached_usd":0.0,"output_usd":0.00125,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1562098748981953}
{"item_id":"semeval2013.d007.s009.t003","gold_sense_keys":["earth%1:17:00::"],"candidates":[{"index":1,"sense_key":"earth%1:17:00::","synset_id":"earth.n.01"},{"index":2,"sense_key":"earth%1:27:00::","synset_id":"earth.n.02"},{"index":3,"sense_key":"earth%1:17:01::","synset_id":"land.n.04"},{"index":4,"sense_key":"earth%1:15:00::","synset_id":"earth.n.04"},{"index":5,"sense_key":"earth%1:27:01::","synset_id":"earth.n.05"},{"index":6,"sense_key":"earth%1:09:00::","synset_id":"worldly_concern.n.01"},{"index":7,"sense_key":"earth%1:06:00::","synset_id":"ground.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"earth%1:17:00::","call_ids":["semeval2013.d007.s009.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"earth%1:17:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":992,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002104,"input_uncached_usd":0.001984,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3666270829271525}
{"item_id":"semeval2013.d007.s009.t005","gold_sense_keys":["life%1:19:00::"],"candidates":[{"index":1,"sense_key":"life%1:26:01::","synset_id":"life.n.01"},{"index":2,"sense_key":"life%1:09:00::","synset_id":"life.n.02"},{"index":3,"sense_key":"life%1:26:02::","synset_id":"life.n.03"},{"index":4,"sense_key":"life%1:26:00::","synset_id":"animation.n.01"},{"index":5,"sense_key":"life%1:28:00::","synset_id":"life.n.05"},{"index":6,"sense_key":"life%1:28:01::","synset_id":"life.n.06"},{"index":7,"sense_key":"life%1:28:02::","synset_id":"life.n.07"},{"index":8,"sense_key":"life%1:18:00::","synset_id":"life.n.08"},{"index":9,"sense_key":"life%1:07:01::","synset_id":"liveliness.n.02"},{"index":10,"sense_key":"life%1:03:00::","synset_id":"life.n.10"},{"index":11,"sense_key":"life%1:19:00::","synset_id":"life.n.11"},{"index":12,"sense_key":"life%1:10:00::","synset_id":"biography.n.01"},{"index":13,"sense_key":"life%1:16:00::","synset_id":"life.n.13"},{"index":14,"sense_key":"life%1:28:03::","synset_id":"life_sentence.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"life%1:19:00::","call_ids":["semeval2013.d007.s009.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"life%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1282,"cached_input_tokens":0,"output_tokens":158,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004144,"input_uncached_usd":0.002564,"input_cached_usd":0.0,"output_usd":0.00158,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0935550410067663}
{"item_id":"semeval2013.d007.s009.t006","gold_sense_keys":["life%1:19:00::"],"candidates":[{"index":1,"sense_key":"life%1:26:01::","synset_id":"life.n.01"},{"index":2,"sense_key":"life%1:09:00::","synset_id":"life.n.02"},{"index":3,"sense_key":"life%1:26:02::","synset_id":"life.n.03"},{"index":4,"sense_key":"life%1:26:00::","synset_id":"animation.n.01"},{"index":5,"sense_key":"life%1:28:00::","synset_id":"life.n.05"},{"index":6,"sense_key":"life%1:28:01::","synset_id":"life.n.06"},{"index":7,"sense_key":"life%1:28:02::","synset_id":"life.n.07"},{"index":8,"sense_key":"life%1:18:00::","synset_id":"life.n.08"},{"index":9,"sense_key":"life%1:07:01::","synset_id":"liveliness.n.02"},{"index":10,"sense_key":"life%1:03:00::","synset_id":"life.n.10"},{"index":11,"sense_key":"life%1:19:00::","synset_id":"life.n.11"},{"index":12,"sense_key":"life%1:10:00::","synset_id":"biography.n.01"},{"index":13,"sense_key":"life%1:16:00::","synset_id":"life.n.13"},{"index":14,"sense_key":"life%1:28:03::","synset_id":"life_sentence.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"life%1:19:00::","call_ids":["semeval2013.d007.s009.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"life%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1282,"cached_input_tokens":0,"output_tokens":134,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003904,"input_uncached_usd":0.002564,"input_cached_usd":0.0,"output_usd":0.00134,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2652449999004602}
{"item_id":"semeval2013.d007.s010.t000","gold_sense_keys":["study%1:10:00::"],"candidates":[{"index":1,"sense_key":"study%1:04:00::","synset_id":"survey.n.01"},{"index":2,"sense_key":"study%1:09:03::","synset_id":"study.n.02"},{"index":3,"sense_key":"study%1:10:00::","synset_id":"report.n.01"},{"index":4,"sense_key":"study%1:09:04::","synset_id":"study.n.04"},{"index":5,"sense_key":"study%1:06:00::","synset_id":"study.n.05"},{"index":6,"sense_key":"study%1:09:02::","synset_id":"discipline.n.01"},{"index":7,"sense_key":"study%1:06:01::","synset_id":"sketch.n.01"},{"index":8,"sense_key":"study%1:09:01::","synset_id":"cogitation.n.02"},{"index":9,"sense_key":"study%1:18:00::","synset_id":"study.n.09"},{"index":10,"sense_key":"study%1:10:01::","synset_id":"study.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"study%1:10:00::","call_ids":["semeval2013.d007.s010.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"study%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1153,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0024259999999999998,"input_uncached_usd":0.002306,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4329871659865603}
{"item_id":"semeval2013.d007.s010.t002","gold_sense_keys":["earth%1:17:00::"],"candidates":[{"index":1,"sense_key":"earth%1:17:00::","synset_id":"earth.n.01"},{"index":2,"sense_key":"earth%1:27:00::","synset_id":"earth.n.02"},{"index":3,"sense_key":"earth%1:17:01::","synset_id":"land.n.04"},{"index":4,"sense_key":"earth%1:15:00::","synset_id":"earth.n.04"},{"index":5,"sense_key":"earth%1:27:01::","synset_id":"earth.n.05"},{"index":6,"sense_key":"earth%1:09:00::","synset_id":"worldly_concern.n.01"},{"index":7,"sense_key":"earth%1:06:00::","synset_id":"ground.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"earth%1:17:00::","call_ids":["semeval2013.d007.s010.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"earth%1:17:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1036,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002192,"input_uncached_usd":0.002072,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.134347333922051}
{"item_id":"semeval2013.d007.s011.t002","gold_sense_keys":["detail%1:09:00::"],"candidates":[{"index":1,"sense_key":"detail%1:09:00::","synset_id":"detail.n.01"},{"index":2,"sense_key":"detail%1:24:00::","synset_id":"detail.n.02"},{"index":3,"sense_key":"detail%1:10:00::","synset_id":"detail.n.03"},{"index":4,"sense_key":"detail%1:14:01::","synset_id":"detail.n.04"},{"index":5,"sense_key":"detail%1:14:00::","synset_id":"contingent.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"detail%1:09:00::","call_ids":["semeval2013.d007.s011.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"detail%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":771,"cached_input_tokens":0,"output_tokens":210,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0036420000000000003,"input_uncached_usd":0.001542,"input_cached_usd":0.0,"output_usd":0.0021000000000000003,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.4408245419617742}
{"item_id":"semeval2013.d007.s011.t003","gold_sense_keys":["find%1:09:00::"],"candidates":[{"index":1,"sense_key":"find%1:09:00::","synset_id":"discovery.n.03"},{"index":2,"sense_key":"find%1:04:00::","synset_id":"discovery.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"find%1:04:00::","call_ids":["semeval2013.d007.s011.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"find%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":577,"cached_input_tokens":0,"output_tokens":150,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002654,"input_uncached_usd":0.0011539999999999999,"input_cached_usd":0.0,"output_usd":0.0015,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.3898260419955477}
{"item_id":"semeval2013.d007.s011.t004","gold_sense_keys":["speculation%1:09:00::"],"candidates":[{"index":1,"sense_key":"speculation%1:10:03::","synset_id":"guess.n.01"},{"index":2,"sense_key":"speculation%1:09:00::","synset_id":"speculation.n.02"},{"index":3,"sense_key":"speculation%1:21:00::","synset_id":"speculation.n.03"},{"index":4,"sense_key":"speculation%1:09:01::","synset_id":"meditation.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"speculation%1:09:00::","call_ids":["semeval2013.d007.s011.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"speculation%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":819,"cached_input_tokens":0,"output_tokens":177,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003408,"input_uncached_usd":0.001638,"input_cached_usd":0.0,"output_usd":0.00177,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2724162079393864}
{"item_id":"semeval2013.d007.s011.t006","gold_sense_keys":["report%1:10:03::"],"candidates":[{"index":1,"sense_key":"report%1:10:03::","synset_id":"report.n.01"},{"index":2,"sense_key":"report%1:10:01::","synset_id":"report.n.02"},{"index":3,"sense_key":"report%1:10:00::","synset_id":"report.n.03"},{"index":4,"sense_key":"report%1:11:00::","synset_id":"report.n.04"},{"index":5,"sense_key":"report%1:10:04::","synset_id":"report_card.n.01"},{"index":6,"sense_key":"report%1:10:02::","synset_id":"composition.n.08"},{"index":7,"sense_key":"report%1:09:00::","synset_id":"reputation.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"report%1:10:03::","call_ids":["semeval2013.d007.s011.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"report%1:10:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":943,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002496,"input_uncached_usd":0.0018859999999999999,"input_cached_usd":0.0,"output_usd":0.0006100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6343265830073506}
{"item_id":"semeval2013.d007.s011.t007","gold_sense_keys":["discovery%1:04:00::"],"candidates":[{"index":1,"sense_key":"discovery%1:04:00::","synset_id":"discovery.n.01"},{"index":2,"sense_key":"discovery%1:10:00::","synset_id":"discovery.n.02"},{"index":3,"sense_key":"discovery%1:09:00::","synset_id":"discovery.n.03"},{"index":4,"sense_key":"discovery%1:10:01::","synset_id":"discovery.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"discovery%1:04:00::","call_ids":["semeval2013.d007.s011.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"discovery%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":685,"cached_input_tokens":0,"output_tokens":109,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00246,"input_uncached_usd":0.00137,"input_cached_usd":0.0,"output_usd":0.00109,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.69022450002376}
{"item_id":"semeval2013.d007.s013.t000","gold_sense_keys":["discovery%1:09:00::","discovery%1:10:00::"],"candidates":[{"index":1,"sense_key":"discovery%1:04:00::","synset_id":"discovery.n.01"},{"index":2,"sense_key":"discovery%1:10:00::","synset_id":"discovery.n.02"},{"index":3,"sense_key":"discovery%1:09:00::","synset_id":"discovery.n.03"},{"index":4,"sense_key":"discovery%1:10:01::","synset_id":"discovery.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"discovery%1:10:00::","call_ids":["semeval2013.d007.s013.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"discovery%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":715,"cached_input_tokens":0,"output_tokens":156,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00299,"input_uncached_usd":0.0014299999999999998,"input_cached_usd":0.0,"output_usd":0.0015600000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.230208041961305}
{"item_id":"semeval2013.d007.s013.t001","gold_sense_keys":["environment%1:26:00::"],"candidates":[{"index":1,"sense_key":"environment%1:26:00::","synset_id":"environment.n.01"},{"index":2,"sense_key":"environment%1:15:00::","synset_id":"environment.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"environment%1:15:00::","call_ids":["semeval2013.d007.s013.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"environment%1:15:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":642,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001954,"input_uncached_usd":0.001284,"input_cached_usd":0.0,"output_usd":0.00067,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5462556249694899}
{"item_id":"semeval2013.d007.s013.t002","gold_sense_keys":["organism%1:03:00::"],"candidates":[{"index":1,"sense_key":"organism%1:03:00::","synset_id":"organism.n.01"},{"index":2,"sense_key":"organism%1:14:00::","synset_id":"organism.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"organism%1:03:00::","call_ids":["semeval2013.d007.s013.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"organism%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":623,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001366,"input_uncached_usd":0.001246,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":4.326842167065479}
{"item_id":"semeval2013.d007.s013.t003","gold_sense_keys":["element%1:27:00::"],"candidates":[{"index":1,"sense_key":"element%1:09:00::","synset_id":"component.n.01"},{"index":2,"sense_key":"element%1:06:00::","synset_id":"component.n.03"},{"index":3,"sense_key":"element%1:27:00::","synset_id":"chemical_element.n.01"},{"index":4,"sense_key":"element%1:15:00::","synset_id":"element.n.04"},{"index":5,"sense_key":"element%1:27:01::","synset_id":"element.n.05"},{"index":6,"sense_key":"element%1:26:00::","synset_id":"element.n.06"},{"index":7,"sense_key":"element%1:25:00::","synset_id":"element.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"element%1:27:00::","call_ids":["semeval2013.d007.s013.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"element%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":969,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002058,"input_uncached_usd":0.0019379999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":4.880386249977164}
{"item_id":"semeval2013.d007.s013.t005","gold_sense_keys":["set%1:14:00::"],"candidates":[{"index":1,"sense_key":"set%1:14:00::","synset_id":"set.n.01"},{"index":2,"sense_key":"set%1:14:02::","synset_id":"set.n.02"},{"index":3,"sense_key":"set%1:04:00::","synset_id":"set.n.03"},{"index":4,"sense_key":"set%1:06:00::","synset_id":"stage_set.n.01"},{"index":5,"sense_key":"set%1:14:01::","synset_id":"set.n.05"},{"index":6,"sense_key":"set%1:09:01::","synset_id":"bent.n.01"},{"index":7,"sense_key":"set%1:04:01::","synset_id":"set.n.07"},{"index":8,"sense_key":"set%1:28:00::","synset_id":"set.n.08"},{"index":9,"sense_key":"set%1:22:00::","synset_id":"hardening.n.02"},{"index":10,"sense_key":"set%1:18:00::","synset_id":"set.n.10"},{"index":11,"sense_key":"set%1:11:00::","synset_id":"set.n.11"},{"index":12,"sense_key":"set%1:09:00::","synset_id":"set.n.12"},{"index":13,"sense_key":"set%1:06:01::","synset_id":"set.n.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"set%1:14:00::","call_ids":["semeval2013.d007.s013.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"set%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1342,"cached_input_tokens":0,"output_tokens":105,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003734,"input_uncached_usd":0.0026839999999999998,"input_cached_usd":0.0,"output_usd":0.0010500000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9573670419631526}
{"item_id":"semeval2013.d007.s013.t006","gold_sense_keys":["element%1:27:00::"],"candidates":[{"index":1,"sense_key":"element%1:09:00::","synset_id":"component.n.01"},{"index":2,"sense_key":"element%1:06:00::","synset_id":"component.n.03"},{"index":3,"sense_key":"element%1:27:00::","synset_id":"chemical_element.n.01"},{"index":4,"sense_key":"element%1:15:00::","synset_id":"element.n.04"},{"index":5,"sense_key":"element%1:27:01::","synset_id":"element.n.05"},{"index":6,"sense_key":"element%1:26:00::","synset_id":"element.n.06"},{"index":7,"sense_key":"element%1:25:00::","synset_id":"element.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"element%1:27:00::","call_ids":["semeval2013.d007.s013.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"element%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":969,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002578,"input_uncached_usd":0.0019379999999999998,"input_cached_usd":0.0,"output_usd":0.00064,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4139273749897256}
{"item_id":"semeval2013.d007.s013.t007","gold_sense_keys":["life%1:19:00::"],"candidates":[{"index":1,"sense_key":"life%1:26:01::","synset_id":"life.n.01"},{"index":2,"sense_key":"life%1:09:00::","synset_id":"life.n.02"},{"index":3,"sense_key":"life%1:26:02::","synset_id":"life.n.03"},{"index":4,"sense_key":"life%1:26:00::","synset_id":"animation.n.01"},{"index":5,"sense_key":"life%1:28:00::","synset_id":"life.n.05"},{"index":6,"sense_key":"life%1:28:01::","synset_id":"life.n.06"},{"index":7,"sense_key":"life%1:28:02::","synset_id":"life.n.07"},{"index":8,"sense_key":"life%1:18:00::","synset_id":"life.n.08"},{"index":9,"sense_key":"life%1:07:01::","synset_id":"liveliness.n.02"},{"index":10,"sense_key":"life%1:03:00::","synset_id":"life.n.10"},{"index":11,"sense_key":"life%1:19:00::","synset_id":"life.n.11"},{"index":12,"sense_key":"life%1:10:00::","synset_id":"biography.n.01"},{"index":13,"sense_key":"life%1:16:00::","synset_id":"life.n.13"},{"index":14,"sense_key":"life%1:28:03::","synset_id":"life_sentence.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"life%1:19:00::","call_ids":["semeval2013.d007.s013.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"life%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1285,"cached_input_tokens":0,"output_tokens":155,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00412,"input_uncached_usd":0.00257,"input_cached_usd":0.0,"output_usd":0.0015500000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.389796999981627}
{"item_id":"semeval2013.d007.s013.t009","gold_sense_keys":["space%1:15:01::"],"candidates":[{"index":1,"sense_key":"space%1:03:00::","synset_id":"space.n.01"},{"index":2,"sense_key":"space%1:25:00::","synset_id":"space.n.02"},{"index":3,"sense_key":"space%1:15:00::","synset_id":"space.n.03"},{"index":4,"sense_key":"space%1:15:01::","synset_id":"outer_space.n.01"},{"index":5,"sense_key":"space%1:10:00::","synset_id":"space.n.05"},{"index":6,"sense_key":"space%1:28:00::","synset_id":"distance.n.05"},{"index":7,"sense_key":"space%1:10:01::","synset_id":"space.n.07"},{"index":8,"sense_key":"space%1:10:02::","synset_id":"space.n.08"},{"index":9,"sense_key":"space%1:06:00::","synset_id":"quad.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"space%1:15:01::","call_ids":["semeval2013.d007.s013.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"space%1:15:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1100,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00292,"input_uncached_usd":0.0021999999999999997,"input_cached_usd":0.0,"output_usd":0.00072,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5679695829749107}
{"item_id":"semeval2013.d007.s013.t010","gold_sense_keys":["science%1:09:00::"],"candidates":[{"index":1,"sense_key":"science%1:09:00::","synset_id":"science.n.01"},{"index":2,"sense_key":"science%1:09:02::","synset_id":"skill.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"science%1:09:00::","call_ids":["semeval2013.d007.s013.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"science%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":653,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001426,"input_uncached_usd":0.0013059999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9599138329504058}
{"item_id":"semeval2013.d007.s014.t000","gold_sense_keys":["part%1:24:00::"],"candidates":[{"index":1,"sense_key":"part%1:24:00::","synset_id":"part.n.01"},{"index":2,"sense_key":"part%1:06:00::","synset_id":"part.n.02"},{"index":3,"sense_key":"part%1:17:00::","synset_id":"part.n.03"},{"index":4,"sense_key":"part%1:09:02::","synset_id":"part.n.04"},{"index":5,"sense_key":"part%1:15:00::","synset_id":"region.n.01"},{"index":6,"sense_key":"part%1:04:01::","synset_id":"function.n.03"},{"index":7,"sense_key":"part%1:09:01::","synset_id":"character.n.04"},{"index":8,"sense_key":"part%1:21:00::","synset_id":"share.n.01"},{"index":9,"sense_key":"part%1:09:00::","synset_id":"part.n.09"},{"index":10,"sense_key":"part%1:08:01::","synset_id":"part.n.10"},{"index":11,"sense_key":"part%1:10:00::","synset_id":"part.n.11"},{"index":12,"sense_key":"part%1:04:03::","synset_id":"contribution.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"part%1:24:00::","call_ids":["semeval2013.d007.s014.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"part%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1337,"cached_input_tokens":0,"output_tokens":187,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004543999999999999,"input_uncached_usd":0.0026739999999999997,"input_cached_usd":0.0,"output_usd":0.0018700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8588159159990028}
{"item_id":"semeval2013.d007.s014.t002","gold_sense_keys":["team%1:14:00::"],"candidates":[{"index":1,"sense_key":"team%1:14:00::","synset_id":"team.n.01"},{"index":2,"sense_key":"team%1:14:01::","synset_id":"team.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"team%1:14:00::","call_ids":["semeval2013.d007.s014.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"team%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":604,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001328,"input_uncached_usd":0.0012079999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4309507500147447}
{"item_id":"semeval2013.d007.s015.t000","gold_sense_keys":["work%1:06:00::"],"candidates":[{"index":1,"sense_key":"work%1:04:00::","synset_id":"work.n.01"},{"index":2,"sense_key":"work%1:06:00::","synset_id":"work.n.02"},{"index":3,"sense_key":"work%1:04:01::","synset_id":"employment.n.02"},{"index":4,"sense_key":"work%1:09:00::","synset_id":"study.n.02"},{"index":5,"sense_key":"work%1:19:00::","synset_id":"work.n.05"},{"index":6,"sense_key":"work%1:06:01::","synset_id":"workplace.n.01"},{"index":7,"sense_key":"work%1:06:02::","synset_id":"oeuvre.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"work%1:06:00::","call_ids":["semeval2013.d007.s015.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"work%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":997,"cached_input_tokens":0,"output_tokens":102,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003014,"input_uncached_usd":0.001994,"input_cached_usd":0.0,"output_usd":0.00102,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7661691249813884}
{"item_id":"semeval2013.d007.s015.t001","gold_sense_keys":["substitution%1:11:00::"],"candidates":[{"index":1,"sense_key":"substitution%1:11:00::","synset_id":"substitution.n.01"},{"index":2,"sense_key":"substitution%1:04:00::","synset_id":"substitution.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"substitution%1:11:00::","call_ids":["semeval2013.d007.s015.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"substitution%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":656,"cached_input_tokens":0,"output_tokens":118,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0024920000000000003,"input_uncached_usd":0.001312,"input_cached_usd":0.0,"output_usd":0.00118,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2017271670047194}
{"item_id":"semeval2013.d007.s015.t002","gold_sense_keys":["element%1:27:00::"],"candidates":[{"index":1,"sense_key":"element%1:09:00::","synset_id":"component.n.01"},{"index":2,"sense_key":"element%1:06:00::","synset_id":"component.n.03"},{"index":3,"sense_key":"element%1:27:00::","synset_id":"chemical_element.n.01"},{"index":4,"sense_key":"element%1:15:00::","synset_id":"element.n.04"},{"index":5,"sense_key":"element%1:27:01::","synset_id":"element.n.05"},{"index":6,"sense_key":"element%1:26:00::","synset_id":"element.n.06"},{"index":7,"sense_key":"element%1:25:00::","synset_id":"element.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"element%1:27:00::","call_ids":["semeval2013.d007.s015.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"element%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":939,"cached_input_tokens":0,"output_tokens":71,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002588,"input_uncached_usd":0.001878,"input_cached_usd":0.0,"output_usd":0.00071,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.003326041973196}
{"item_id":"semeval2013.d007.s015.t004","gold_sense_keys":["structure%1:07:00::"],"candidates":[{"index":1,"sense_key":"structure%1:06:00::","synset_id":"structure.n.01"},{"index":2,"sense_key":"structure%1:07:00::","synset_id":"structure.n.02"},{"index":3,"sense_key":"structure%1:09:00::","synset_id":"structure.n.03"},{"index":4,"sense_key":"structure%1:08:00::","synset_id":"structure.n.04"},{"index":5,"sense_key":"structure%1:14:00::","synset_id":"social_organization.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"structure%1:07:00::","call_ids":["semeval2013.d007.s015.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"structure%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":878,"cached_input_tokens":0,"output_tokens":86,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0026160000000000003,"input_uncached_usd":0.001756,"input_cached_usd":0.0,"output_usd":0.0008600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.845182666904293}
{"item_id":"semeval2013.d007.s016.t000","gold_sense_keys":["discovery%1:04:00::"],"candidates":[{"index":1,"sense_key":"discovery%1:04:00::","synset_id":"discovery.n.01"},{"index":2,"sense_key":"discovery%1:10:00::","synset_id":"discovery.n.02"},{"index":3,"sense_key":"discovery%1:09:00::","synset_id":"discovery.n.03"},{"index":4,"sense_key":"discovery%1:10:01::","synset_id":"discovery.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"discovery%1:04:00::","call_ids":["semeval2013.d007.s016.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"discovery%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":723,"cached_input_tokens":0,"output_tokens":119,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0026360000000000003,"input_uncached_usd":0.001446,"input_cached_usd":0.0,"output_usd":0.00119,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2075560829835013}
{"item_id":"semeval2013.d007.s017.t003","gold_sense_keys":["study%1:04:00::"],"candidates":[{"index":1,"sense_key":"study%1:04:00::","synset_id":"survey.n.01"},{"index":2,"sense_key":"study%1:09:03::","synset_id":"study.n.02"},{"index":3,"sense_key":"study%1:10:00::","synset_id":"report.n.01"},{"index":4,"sense_key":"study%1:09:04::","synset_id":"study.n.04"},{"index":5,"sense_key":"study%1:06:00::","synset_id":"study.n.05"},{"index":6,"sense_key":"study%1:09:02::","synset_id":"discipline.n.01"},{"index":7,"sense_key":"study%1:06:01::","synset_id":"sketch.n.01"},{"index":8,"sense_key":"study%1:09:01::","synset_id":"cogitation.n.02"},{"index":9,"sense_key":"study%1:18:00::","synset_id":"study.n.09"},{"index":10,"sense_key":"study%1:10:01::","synset_id":"study.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"study%1:10:00::","call_ids":["semeval2013.d007.s017.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"study%1:10:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1103,"cached_input_tokens":0,"output_tokens":131,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003516,"input_uncached_usd":0.002206,"input_cached_usd":0.0,"output_usd":0.0013100000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8115462920395657}
{"item_id":"semeval2013.d007.s017.t004","gold_sense_keys":["arsenic%1:27:00::"],"candidates":[{"index":1,"sense_key":"arsenic%1:27:01::","synset_id":"arsenic.n.01"},{"index":2,"sense_key":"arsenic%1:27:00::","synset_id":"arsenic.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"arsenic%1:27:00::","call_ids":["semeval2013.d007.s017.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"arsenic%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":740,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00216,"input_uncached_usd":0.00148,"input_cached_usd":0.0,"output_usd":0.00068,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":4.767489791032858}
{"item_id":"semeval2013.d007.s017.t005","gold_sense_keys":["level%1:07:00::"],"candidates":[{"index":1,"sense_key":"level%1:07:00::","synset_id":"degree.n.01"},{"index":2,"sense_key":"level%1:26:01::","synset_id":"grade.n.02"},{"index":3,"sense_key":"level%1:26:00::","synset_id":"degree.n.02"},{"index":4,"sense_key":"level%1:07:01::","synset_id":"level.n.04"},{"index":5,"sense_key":"level%1:06:00::","synset_id":"level.n.05"},{"index":6,"sense_key":"level%1:06:03::","synset_id":"horizontal_surface.n.01"},{"index":7,"sense_key":"level%1:09:00::","synset_id":"level.n.07"},{"index":8,"sense_key":"level%1:06:01::","synset_id":"floor.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"level%1:07:00::","call_ids":["semeval2013.d007.s017.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"level%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1039,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002648,"input_uncached_usd":0.002078,"input_cached_usd":0.0,"output_usd":0.0005700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":5.027792082983069}
{"item_id":"semeval2013.d007.s018.t000","gold_sense_keys":["arsenic%1:27:00::"],"candidates":[{"index":1,"sense_key":"arsenic%1:27:01::","synset_id":"arsenic.n.01"},{"index":2,"sense_key":"arsenic%1:27:00::","synset_id":"arsenic.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"arsenic%1:27:00::","call_ids":["semeval2013.d007.s018.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"arsenic%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":770,"cached_input_tokens":0,"output_tokens":83,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00237,"input_uncached_usd":0.00154,"input_cached_usd":0.0,"output_usd":0.0008300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7838228329783306}
{"item_id":"semeval2013.d007.s018.t001","gold_sense_keys":["phosphorus%1:27:00::"],"candidates":[{"index":1,"sense_key":"phosphorus%1:27:00::","synset_id":"phosphorus.n.01"},{"index":2,"sense_key":"phosphorus%1:17:00::","synset_id":"morning_star.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"phosphorus%1:27:00::","call_ids":["semeval2013.d007.s018.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"phosphorus%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":731,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001582,"input_uncached_usd":0.001462,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.628912708023563}
{"item_id":"semeval2013.d007.s019.t000","gold_sense_keys":["arsenic%1:27:00::"],"candidates":[{"index":1,"sense_key":"arsenic%1:27:01::","synset_id":"arsenic.n.01"},{"index":2,"sense_key":"arsenic%1:27:00::","synset_id":"arsenic.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"arsenic%1:27:00::","call_ids":["semeval2013.d007.s019.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"arsenic%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":675,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00198,"input_uncached_usd":0.0013499999999999999,"input_cached_usd":0.0,"output_usd":0.00063,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.461117040948011}
{"item_id":"semeval2013.d007.s019.t002","gold_sense_keys":["behavior%1:26:00::"],"candidates":[{"index":1,"sense_key":"behavior%1:04:00::","synset_id":"behavior.n.01"},{"index":2,"sense_key":"behavior%1:26:00::","synset_id":"behavior.n.02"},{"index":3,"sense_key":"behavior%1:07:00::","synset_id":"demeanor.n.01"},{"index":4,"sense_key":"behavior%1:04:01::","synset_id":"behavior.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"behavior%1:26:00::","call_ids":["semeval2013.d007.s019.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"behavior%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":716,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001552,"input_uncached_usd":0.001432,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":4.663491708924994}
{"item_id":"semeval2013.d007.s019.t003","gold_sense_keys":["phosphorus%1:27:00::"],"candidates":[{"index":1,"sense_key":"phosphorus%1:27:00::","synset_id":"phosphorus.n.01"},{"index":2,"sense_key":"phosphorus%1:17:00::","synset_id":"morning_star.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"phosphorus%1:27:00::","call_ids":["semeval2013.d007.s019.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"phosphorus%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":636,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001392,"input_uncached_usd":0.001272,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0225690419320017}
{"item_id":"semeval2013.d007.s020.t000","gold_sense_keys":["result%1:19:00::"],"candidates":[{"index":1,"sense_key":"result%1:19:00::","synset_id":"consequence.n.01"},{"index":2,"sense_key":"result%1:10:00::","synset_id":"solution.n.02"},{"index":3,"sense_key":"result%1:11:00::","synset_id":"result.n.03"},{"index":4,"sense_key":"result%1:10:01::","synset_id":"resultant_role.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"result%1:19:00::","call_ids":["semeval2013.d007.s020.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"result%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":813,"cached_input_tokens":0,"output_tokens":81,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002436,"input_uncached_usd":0.0016259999999999998,"input_cached_usd":0.0,"output_usd":0.0008100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1217917919857427}
{"item_id":"semeval2013.d007.s020.t001","gold_sense_keys":["organism%1:03:00::"],"candidates":[{"index":1,"sense_key":"organism%1:03:00::","synset_id":"organism.n.01"},{"index":2,"sense_key":"organism%1:14:00::","synset_id":"organism.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"organism%1:03:00::","call_ids":["semeval2013.d007.s020.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"organism%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":594,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0013080000000000001,"input_uncached_usd":0.001188,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.531661165994592}
{"item_id":"semeval2013.d007.s020.t002","gold_sense_keys":["hard_time%1:26:00::"],"candidates":[{"index":1,"sense_key":"hard_time%1:26:00::","synset_id":"hard_time.n.01"},{"index":2,"sense_key":"hard_time%1:28:00::","synset_id":"hard_time.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"hard_time%1:26:00::","call_ids":["semeval2013.d007.s020.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"hard_time%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":614,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001348,"input_uncached_usd":0.001228,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.222464625025168}
{"item_id":"semeval2013.d007.s020.t003","gold_sense_keys":["element%1:27:00::"],"candidates":[{"index":1,"sense_key":"element%1:09:00::","synset_id":"component.n.01"},{"index":2,"sense_key":"element%1:06:00::","synset_id":"component.n.03"},{"index":3,"sense_key":"element%1:27:00::","synset_id":"chemical_element.n.01"},{"index":4,"sense_key":"element%1:15:00::","synset_id":"element.n.04"},{"index":5,"sense_key":"element%1:27:01::","synset_id":"element.n.05"},{"index":6,"sense_key":"element%1:26:00::","synset_id":"element.n.06"},{"index":7,"sense_key":"element%1:25:00::","synset_id":"element.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"element%1:27:00::","call_ids":["semeval2013.d007.s020.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"element%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":940,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002,"input_uncached_usd":0.00188,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5716080830898136}
{"item_id":"semeval2013.d007.s021.t000","gold_sense_keys":["arsenic%1:27:00::"],"candidates":[{"index":1,"sense_key":"arsenic%1:27:01::","synset_id":"arsenic.n.01"},{"index":2,"sense_key":"arsenic%1:27:00::","synset_id":"arsenic.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"arsenic%1:27:00::","call_ids":["semeval2013.d007.s021.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"arsenic%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":730,"cached_input_tokens":0,"output_tokens":97,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00243,"input_uncached_usd":0.00146,"input_cached_usd":0.0,"output_usd":0.00097,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.82382054196205}
{"item_id":"semeval2013.d007.s021.t001","gold_sense_keys":["phosphorus%1:27:00::"],"candidates":[{"index":1,"sense_key":"phosphorus%1:27:00::","synset_id":"phosphorus.n.01"},{"index":2,"sense_key":"phosphorus%1:17:00::","synset_id":"morning_star.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"phosphorus%1:27:00::","call_ids":["semeval2013.d007.s021.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"phosphorus%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":692,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0015040000000000001,"input_uncached_usd":0.001384,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":4.263422082993202}
{"item_id":"semeval2013.d007.s022.t002","gold_sense_keys":["concentration%1:07:02::"],"candidates":[{"index":1,"sense_key":"concentration%1:07:02::","synset_id":"concentration.n.01"},{"index":2,"sense_key":"concentration%1:07:03::","synset_id":"concentration.n.02"},{"index":3,"sense_key":"concentration%1:04:01::","synset_id":"concentration.n.03"},{"index":4,"sense_key":"concentration%1:11:00::","synset_id":"concentration.n.04"},{"index":5,"sense_key":"concentration%1:09:00::","synset_id":"concentration.n.05"},{"index":6,"sense_key":"concentration%1:04:02::","synset_id":"concentration.n.06"},{"index":7,"sense_key":"concentration%1:07:01::","synset_id":"assiduity.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"concentration%1:07:02::","call_ids":["semeval2013.d007.s022.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"concentration%1:07:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":882,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001884,"input_uncached_usd":0.001764,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.24296000006143}
{"item_id":"semeval2013.d007.s022.t003","gold_sense_keys":["arsenic%1:27:00::"],"candidates":[{"index":1,"sense_key":"arsenic%1:27:01::","synset_id":"arsenic.n.01"},{"index":2,"sense_key":"arsenic%1:27:00::","synset_id":"arsenic.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"arsenic%1:27:00::","call_ids":["semeval2013.d007.s022.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"arsenic%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":796,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002152,"input_uncached_usd":0.0015919999999999999,"input_cached_usd":0.0,"output_usd":0.0005600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":6.015157541027293}
{"item_id":"semeval2013.d007.s023.t001","gold_sense_keys":["phosphorus%1:27:00::"],"candidates":[{"index":1,"sense_key":"phosphorus%1:27:00::","synset_id":"phosphorus.n.01"},{"index":2,"sense_key":"phosphorus%1:17:00::","synset_id":"morning_star.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"phosphorus%1:27:00::","call_ids":["semeval2013.d007.s023.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"phosphorus%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":773,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001666,"input_uncached_usd":0.0015459999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2634655419969931}
{"item_id":"semeval2013.d007.s023.t002","gold_sense_keys":["arsenic%1:27:00::"],"candidates":[{"index":1,"sense_key":"arsenic%1:27:01::","synset_id":"arsenic.n.01"},{"index":2,"sense_key":"arsenic%1:27:00::","synset_id":"arsenic.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"arsenic%1:27:00::","call_ids":["semeval2013.d007.s023.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"arsenic%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":811,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001742,"input_uncached_usd":0.001622,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":12.724482957972214}
{"item_id":"semeval2013.d007.s023.t003","gold_sense_keys":["possibility%1:26:00::"],"candidates":[{"index":1,"sense_key":"possibility%1:09:03::","synset_id":"possibility.n.01"},{"index":2,"sense_key":"possibility%1:26:00::","synset_id":"possibility.n.02"},{"index":3,"sense_key":"possibility%1:09:00::","synset_id":"hypothesis.n.02"},{"index":4,"sense_key":"possibility%1:09:01::","synset_id":"possibility.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"possibility%1:26:00::","call_ids":["semeval2013.d007.s023.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"possibility%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":889,"cached_input_tokens":0,"output_tokens":95,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002728,"input_uncached_usd":0.0017779999999999998,"input_cached_usd":0.0,"output_usd":0.0009500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6125435000285506}
{"item_id":"semeval2013.d007.s023.t005","gold_sense_keys":["arsenic%1:27:00::"],"candidates":[{"index":1,"sense_key":"arsenic%1:27:01::","synset_id":"arsenic.n.01"},{"index":2,"sense_key":"arsenic%1:27:00::","synset_id":"arsenic.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"arsenic%1:27:00::","call_ids":["semeval2013.d007.s023.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"arsenic%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":811,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001742,"input_uncached_usd":0.001622,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1222822499694303}
{"item_id":"semeval2013.d007.s023.t006","gold_sense_keys":["earth%1:17:00::"],"candidates":[{"index":1,"sense_key":"earth%1:17:00::","synset_id":"earth.n.01"},{"index":2,"sense_key":"earth%1:27:00::","synset_id":"earth.n.02"},{"index":3,"sense_key":"earth%1:17:01::","synset_id":"land.n.04"},{"index":4,"sense_key":"earth%1:15:00::","synset_id":"earth.n.04"},{"index":5,"sense_key":"earth%1:27:01::","synset_id":"earth.n.05"},{"index":6,"sense_key":"earth%1:09:00::","synset_id":"worldly_concern.n.01"},{"index":7,"sense_key":"earth%1:06:00::","synset_id":"ground.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"earth%1:17:00::","call_ids":["semeval2013.d007.s023.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"earth%1:17:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1057,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002234,"input_uncached_usd":0.002114,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1986141660017893}
{"item_id":"semeval2013.d007.s023.t007","gold_sense_keys":["planet%1:17:00::","planet%1:17:01::"],"candidates":[{"index":1,"sense_key":"planet%1:17:00::","synset_id":"planet.n.01"},{"index":2,"sense_key":"planet%1:18:00::","synset_id":"satellite.n.02"},{"index":3,"sense_key":"planet%1:17:01::","synset_id":"planet.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"planet%1:17:01::","call_ids":["semeval2013.d007.s023.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"planet%1:17:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":832,"cached_input_tokens":0,"output_tokens":120,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0028640000000000002,"input_uncached_usd":0.001664,"input_cached_usd":0.0,"output_usd":0.0012000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0332495840266347}
{"item_id":"semeval2013.d007.s023.t008","gold_sense_keys":["moon%1:17:00::"],"candidates":[{"index":1,"sense_key":"moon%1:17:01::","synset_id":"moon.n.01"},{"index":2,"sense_key":"moon%1:17:02::","synset_id":"moon.n.02"},{"index":3,"sense_key":"moon%1:28:00::","synset_id":"lunar_month.n.01"},{"index":4,"sense_key":"moon%1:19:00::","synset_id":"moonlight.n.01"},{"index":5,"sense_key":"moon%1:18:00::","synset_id":"moon.n.05"},{"index":6,"sense_key":"moon%1:17:00::","synset_id":"moon.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"moon%1:17:00::","call_ids":["semeval2013.d007.s023.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"moon%1:17:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":988,"cached_input_tokens":0,"output_tokens":71,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002686,"input_uncached_usd":0.001976,"input_cached_usd":0.0,"output_usd":0.00071,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":4.265863791923039}
{"item_id":"semeval2013.d007.s023.t009","gold_sense_keys":["arsenic%1:27:00::"],"candidates":[{"index":1,"sense_key":"arsenic%1:27:01::","synset_id":"arsenic.n.01"},{"index":2,"sense_key":"arsenic%1:27:00::","synset_id":"arsenic.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"arsenic%1:27:00::","call_ids":["semeval2013.d007.s023.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"arsenic%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":811,"cached_input_tokens":0,"output_tokens":74,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002362,"input_uncached_usd":0.001622,"input_cached_usd":0.0,"output_usd":0.0007400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":4.404540959047154}
{"item_id":"semeval2013.d007.s024.t002","gold_sense_keys":["idea%1:09:00::"],"candidates":[{"index":1,"sense_key":"idea%1:09:00::","synset_id":"idea.n.01"},{"index":2,"sense_key":"idea%1:09:01::","synset_id":"mind.n.06"},{"index":3,"sense_key":"idea%1:09:03::","synset_id":"idea.n.03"},{"index":4,"sense_key":"idea%1:09:02::","synset_id":"estimate.n.01"},{"index":5,"sense_key":"idea%1:10:00::","synset_id":"theme.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"idea%1:09:00::","call_ids":["semeval2013.d007.s024.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"idea%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":931,"cached_input_tokens":0,"output_tokens":114,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0030020000000000003,"input_uncached_usd":0.001862,"input_cached_usd":0.0,"output_usd":0.0011400000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7998915410134941}
{"item_id":"semeval2013.d007.s024.t003","gold_sense_keys":["arsenic%1:27:00::"],"candidates":[{"index":1,"sense_key":"arsenic%1:27:01::","synset_id":"arsenic.n.01"},{"index":2,"sense_key":"arsenic%1:27:00::","synset_id":"arsenic.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"arsenic%1:27:00::","call_ids":["semeval2013.d007.s024.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"arsenic%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":804,"cached_input_tokens":0,"output_tokens":76,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002368,"input_uncached_usd":0.0016079999999999998,"input_cached_usd":0.0,"output_usd":0.00076,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.095588249969296}
{"item_id":"semeval2013.d007.s024.t004","gold_sense_keys":["phosphorus%1:27:00::"],"candidates":[{"index":1,"sense_key":"phosphorus%1:27:00::","synset_id":"phosphorus.n.01"},{"index":2,"sense_key":"phosphorus%1:17:00::","synset_id":"morning_star.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"phosphorus%1:27:00::","call_ids":["semeval2013.d007.s024.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"phosphorus%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":766,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001652,"input_uncached_usd":0.001532,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":14.572961166966707}
{"item_id":"semeval2013.d007.s024.t005","gold_sense_keys":["example%1:11:00::"],"candidates":[{"index":1,"sense_key":"example%1:09:00::","synset_id":"example.n.01"},{"index":2,"sense_key":"example%1:09:02::","synset_id":"model.n.07"},{"index":3,"sense_key":"example%1:09:01::","synset_id":"exemplar.n.01"},{"index":4,"sense_key":"example%1:10:00::","synset_id":"example.n.04"},{"index":5,"sense_key":"example%1:11:00::","synset_id":"case.n.01"},{"index":6,"sense_key":"example%1:04:00::","synset_id":"exercise.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"example%1:11:00::","call_ids":["semeval2013.d007.s024.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"example%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":963,"cached_input_tokens":0,"output_tokens":93,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002856,"input_uncached_usd":0.001926,"input_cached_usd":0.0,"output_usd":0.00093,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1871045000152662}
{"item_id":"semeval2013.d007.s025.t000","gold_sense_keys":["arsenic%1:27:00::"],"candidates":[{"index":1,"sense_key":"arsenic%1:27:01::","synset_id":"arsenic.n.01"},{"index":2,"sense_key":"arsenic%1:27:00::","synset_id":"arsenic.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"arsenic%1:27:00::","call_ids":["semeval2013.d007.s025.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"arsenic%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":824,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001768,"input_uncached_usd":0.001648,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.501517708064057}
{"item_id":"semeval2013.d007.s025.t001","gold_sense_keys":["times%1:28:01::"],"candidates":[{"index":1,"sense_key":"times%1:28:01::","synset_id":"times.n.01"},{"index":2,"sense_key":"times%1:04:00::","synset_id":"multiplication.n.03"},{"index":3,"sense_key":"time%1:11:00::","synset_id":"time.n.01"},{"index":4,"sense_key":"time%1:28:05::","synset_id":"time.n.02"},{"index":5,"sense_key":"time%1:28:00::","synset_id":"time.n.03"},{"index":6,"sense_key":"time%1:28:06::","synset_id":"time.n.04"},{"index":7,"sense_key":"time%1:03:00::","synset_id":"time.n.05"},{"index":8,"sense_key":"time%1:11:01::","synset_id":"time.n.06"},{"index":9,"sense_key":"clock_time%1:28:00::","synset_id":"clock_time.n.01"},{"index":10,"sense_key":"fourth_dimension%1:28:00::","synset_id":"fourth_dimension.n.01"},{"index":11,"sense_key":"meter%1:07:00::","synset_id":"meter.n.04"},{"index":12,"sense_key":"prison_term%1:28:00::","synset_id":"prison_term.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"times%1:28:01::","call_ids":["semeval2013.d007.s025.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"times%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1359,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003248,"input_uncached_usd":0.002718,"input_cached_usd":0.0,"output_usd":0.0005300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.55896712501999}
{"item_id":"semeval2013.d007.s025.t002","gold_sense_keys":["earth%1:17:00::"],"candidates":[{"index":1,"sense_key":"earth%1:17:00::","synset_id":"earth.n.01"},{"index":2,"sense_key":"earth%1:27:00::","synset_id":"earth.n.02"},{"index":3,"sense_key":"earth%1:17:01::","synset_id":"land.n.04"},{"index":4,"sense_key":"earth%1:15:00::","synset_id":"earth.n.04"},{"index":5,"sense_key":"earth%1:27:01::","synset_id":"earth.n.05"},{"index":6,"sense_key":"earth%1:09:00::","synset_id":"worldly_concern.n.01"},{"index":7,"sense_key":"earth%1:06:00::","synset_id":"ground.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"earth%1:17:00::","call_ids":["semeval2013.d007.s025.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"earth%1:17:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1069,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0022579999999999996,"input_uncached_usd":0.0021379999999999997,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3065789999673143}
{"item_id":"semeval2013.d007.s026.t000","gold_sense_keys":["way%1:04:01::","way%1:07:01::"],"candidates":[{"index":1,"sense_key":"way%1:07:01::","synset_id":"manner.n.01"},{"index":2,"sense_key":"way%1:04:01::","synset_id":"means.n.01"},{"index":3,"sense_key":"way%1:15:00::","synset_id":"direction.n.01"},{"index":4,"sense_key":"way%1:26:00::","synset_id":"way.n.04"},{"index":5,"sense_key":"way%1:04:00::","synset_id":"way.n.05"},{"index":6,"sense_key":"way%1:06:00::","synset_id":"way.n.06"},{"index":7,"sense_key":"way%1:04:02::","synset_id":"way.n.07"},{"index":8,"sense_key":"way%1:23:00::","synset_id":"room.n.02"},{"index":9,"sense_key":"way%1:07:02::","synset_id":"way.n.09"},{"index":10,"sense_key":"way%1:09:00::","synset_id":"way.n.10"},{"index":11,"sense_key":"way%1:09:01::","synset_id":"way.n.11"},{"index":12,"sense_key":"way%1:21:00::","synset_id":"way.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"way%1:04:01::","call_ids":["semeval2013.d007.s026.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"way%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1225,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00318,"input_uncached_usd":0.00245,"input_cached_usd":0.0,"output_usd":0.0007300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":11.484008874977008}
{"item_id":"semeval2013.d007.s026.t001","gold_sense_keys":["life%1:19:00::"],"candidates":[{"index":1,"sense_key":"life%1:26:01::","synset_id":"life.n.01"},{"index":2,"sense_key":"life%1:09:00::","synset_id":"life.n.02"},{"index":3,"sense_key":"life%1:26:02::","synset_id":"life.n.03"},{"index":4,"sense_key":"life%1:26:00::","synset_id":"animation.n.01"},{"index":5,"sense_key":"life%1:28:00::","synset_id":"life.n.05"},{"index":6,"sense_key":"life%1:28:01::","synset_id":"life.n.06"},{"index":7,"sense_key":"life%1:28:02::","synset_id":"life.n.07"},{"index":8,"sense_key":"life%1:18:00::","synset_id":"life.n.08"},{"index":9,"sense_key":"life%1:07:01::","synset_id":"liveliness.n.02"},{"index":10,"sense_key":"life%1:03:00::","synset_id":"life.n.10"},{"index":11,"sense_key":"life%1:19:00::","synset_id":"life.n.11"},{"index":12,"sense_key":"life%1:10:00::","synset_id":"biography.n.01"},{"index":13,"sense_key":"life%1:16:00::","synset_id":"life.n.13"},{"index":14,"sense_key":"life%1:28:03::","synset_id":"life_sentence.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"life%1:19:00::","call_ids":["semeval2013.d007.s026.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"life%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1366,"cached_input_tokens":0,"output_tokens":133,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004062,"input_uncached_usd":0.002732,"input_cached_usd":0.0,"output_usd":0.00133,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8824598750798032}
{"item_id":"semeval2013.d007.s026.t002","gold_sense_keys":["life%1:19:00::"],"candidates":[{"index":1,"sense_key":"life%1:26:01::","synset_id":"life.n.01"},{"index":2,"sense_key":"life%1:09:00::","synset_id":"life.n.02"},{"index":3,"sense_key":"life%1:26:02::","synset_id":"life.n.03"},{"index":4,"sense_key":"life%1:26:00::","synset_id":"animation.n.01"},{"index":5,"sense_key":"life%1:28:00::","synset_id":"life.n.05"},{"index":6,"sense_key":"life%1:28:01::","synset_id":"life.n.06"},{"index":7,"sense_key":"life%1:28:02::","synset_id":"life.n.07"},{"index":8,"sense_key":"life%1:18:00::","synset_id":"life.n.08"},{"index":9,"sense_key":"life%1:07:01::","synset_id":"liveliness.n.02"},{"index":10,"sense_key":"life%1:03:00::","synset_id":"life.n.10"},{"index":11,"sense_key":"life%1:19:00::","synset_id":"life.n.11"},{"index":12,"sense_key":"life%1:10:00::","synset_id":"biography.n.01"},{"index":13,"sense_key":"life%1:16:00::","synset_id":"life.n.13"},{"index":14,"sense_key":"life%1:28:03::","synset_id":"life_sentence.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"life%1:19:00::","call_ids":["semeval2013.d007.s026.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"life%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1366,"cached_input_tokens":0,"output_tokens":168,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004412,"input_uncached_usd":0.002732,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":4.629141291021369}
{"item_id":"semeval2013.d007.s026.t003","gold_sense_keys":["earth%1:17:00::"],"candidates":[{"index":1,"sense_key":"earth%1:17:00::","synset_id":"earth.n.01"},{"index":2,"sense_key":"earth%1:27:00::","synset_id":"earth.n.02"},{"index":3,"sense_key":"earth%1:17:01::","synset_id":"land.n.04"},{"index":4,"sense_key":"earth%1:15:00::","synset_id":"earth.n.04"},{"index":5,"sense_key":"earth%1:27:01::","synset_id":"earth.n.05"},{"index":6,"sense_key":"earth%1:09:00::","synset_id":"worldly_concern.n.01"},{"index":7,"sense_key":"earth%1:06:00::","synset_id":"ground.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"earth%1:17:00::","call_ids":["semeval2013.d007.s026.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"earth%1:17:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1076,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0022719999999999997,"input_uncached_usd":0.002152,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2343943750020117}
{"item_id":"semeval2013.d007.s026.t004","gold_sense_keys":["part%1:24:00::"],"candidates":[{"index":1,"sense_key":"part%1:24:00::","synset_id":"part.n.01"},{"index":2,"sense_key":"part%1:06:00::","synset_id":"part.n.02"},{"index":3,"sense_key":"part%1:17:00::","synset_id":"part.n.03"},{"index":4,"sense_key":"part%1:09:02::","synset_id":"part.n.04"},{"index":5,"sense_key":"part%1:15:00::","synset_id":"region.n.01"},{"index":6,"sense_key":"part%1:04:01::","synset_id":"function.n.03"},{"index":7,"sense_key":"part%1:09:01::","synset_id":"character.n.04"},{"index":8,"sense_key":"part%1:21:00::","synset_id":"share.n.01"},{"index":9,"sense_key":"part%1:09:00::","synset_id":"part.n.09"},{"index":10,"sense_key":"part%1:08:01::","synset_id":"part.n.10"},{"index":11,"sense_key":"part%1:10:00::","synset_id":"part.n.11"},{"index":12,"sense_key":"part%1:04:03::","synset_id":"contribution.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"part%1:24:00::","call_ids":["semeval2013.d007.s026.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"part%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1416,"cached_input_tokens":0,"output_tokens":134,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004172,"input_uncached_usd":0.002832,"input_cached_usd":0.0,"output_usd":0.00134,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.561403625062667}
{"item_id":"semeval2013.d007.s026.t006","gold_sense_keys":["team%1:14:00::"],"candidates":[{"index":1,"sense_key":"team%1:14:00::","synset_id":"team.n.01"},{"index":2,"sense_key":"team%1:14:01::","synset_id":"team.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"team%1:14:00::","call_ids":["semeval2013.d007.s026.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"team%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":683,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0014860000000000001,"input_uncached_usd":0.001366,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1633352920180187}
{"item_id":"semeval2013.d008.s000.t000","gold_sense_keys":["routine%1:04:00::"],"candidates":[{"index":1,"sense_key":"routine%1:04:00::","synset_id":"routine.n.01"},{"index":2,"sense_key":"routine%1:10:01::","synset_id":"act.n.04"},{"index":3,"sense_key":"routine%1:10:00::","synset_id":"routine.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"routine%1:04:00::","call_ids":["semeval2013.d008.s000.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"routine%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":448,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001016,"input_uncached_usd":0.000896,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4659339170902967}
{"item_id":"semeval2013.d008.s001.t000","gold_sense_keys":["support%1:04:00::","support%1:09:00::"],"candidates":[{"index":1,"sense_key":"support%1:04:04::","synset_id":"support.n.01"},{"index":2,"sense_key":"support%1:04:00::","synset_id":"support.n.02"},{"index":3,"sense_key":"support%1:09:00::","synset_id":"support.n.03"},{"index":4,"sense_key":"support%1:04:03::","synset_id":"support.n.04"},{"index":5,"sense_key":"support%1:04:02::","synset_id":"documentation.n.03"},{"index":6,"sense_key":"support%1:21:00::","synset_id":"support.n.06"},{"index":7,"sense_key":"support%1:06:01::","synset_id":"support.n.07"},{"index":8,"sense_key":"support%1:04:01::","synset_id":"support.n.08"},{"index":9,"sense_key":"support%1:10:01::","synset_id":"accompaniment.n.02"},{"index":10,"sense_key":"support%1:06:00::","synset_id":"support.n.10"},{"index":11,"sense_key":"support%1:21:01::","synset_id":"support.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"support%1:09:00::","call_ids":["semeval2013.d008.s001.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"support%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1010,"cached_input_tokens":0,"output_tokens":123,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0032500000000000003,"input_uncached_usd":0.00202,"input_cached_usd":0.0,"output_usd":0.0012300000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9107386249816045}
{"item_id":"semeval2013.d008.s001.t003","gold_sense_keys":["backing%1:04:00::"],"candidates":[{"index":1,"sense_key":"backing%1:04:00::","synset_id":"backing.n.01"},{"index":2,"sense_key":"backing%1:06:00::","synset_id":"backing.n.02"},{"index":3,"sense_key":"backing%1:21:00::","synset_id":"support.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"backing%1:04:00::","call_ids":["semeval2013.d008.s001.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"backing%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":468,"cached_input_tokens":0,"output_tokens":50,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001436,"input_uncached_usd":0.000936,"input_cached_usd":0.0,"output_usd":0.0005,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1985086249187589}
{"item_id":"semeval2013.d008.s002.t000","gold_sense_keys":["crime%1:04:00::"],"candidates":[{"index":1,"sense_key":"crime%1:04:00::","synset_id":"crime.n.01"},{"index":2,"sense_key":"crime%1:04:01::","synset_id":"crime.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"crime%1:04:00::","call_ids":["semeval2013.d008.s002.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"crime%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":437,"cached_input_tokens":0,"output_tokens":98,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0018540000000000002,"input_uncached_usd":0.000874,"input_cached_usd":0.0,"output_usd":0.0009800000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6093306659022346}
{"item_id":"semeval2013.d008.s002.t001","gold_sense_keys":["worry%1:09:00::"],"candidates":[{"index":1,"sense_key":"worry%1:09:00::","synset_id":"concern.n.04"},{"index":2,"sense_key":"worry%1:12:00::","synset_id":"worry.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"worry%1:09:00::","call_ids":["semeval2013.d008.s002.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"worry%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":453,"cached_input_tokens":0,"output_tokens":80,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001706,"input_uncached_usd":0.000906,"input_cached_usd":0.0,"output_usd":0.0008,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":15.25099633296486}
{"item_id":"semeval2013.d008.s003.t000","gold_sense_keys":["brazil%1:15:00::"],"candidates":[{"index":1,"sense_key":"brazil%1:15:00::","synset_id":"brazil.n.01"},{"index":2,"sense_key":"brazil%1:13:00::","synset_id":"brazil_nut.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"brazil%1:15:00::","call_ids":["semeval2013.d008.s003.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"brazil%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":501,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001122,"input_uncached_usd":0.0010019999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8413532910635695}
{"item_id":"semeval2013.d008.s003.t001","gold_sense_keys":["united_states%1:15:00::"],"candidates":[{"index":1,"sense_key":"united_states%1:15:00::","synset_id":"united_states.n.01"},{"index":2,"sense_key":"united_states%1:14:00::","synset_id":"united_states_government.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"united_states%1:15:00::","call_ids":["semeval2013.d008.s003.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"united_states%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":522,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0017040000000000002,"input_uncached_usd":0.001044,"input_cached_usd":0.0,"output_usd":0.0006600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7666687089949846}
{"item_id":"semeval2013.d008.s004.t000","gold_sense_keys":["finding%1:17:00::"],"candidates":[{"index":1,"sense_key":"finding%1:04:00::","synset_id":"determination.n.01"},{"index":2,"sense_key":"finding%1:04:01::","synset_id":"finding.n.02"},{"index":3,"sense_key":"finding%1:17:00::","synset_id":"finding.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"finding%1:17:00::","call_ids":["semeval2013.d008.s004.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"finding%1:17:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":545,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0012100000000000001,"input_uncached_usd":0.00109,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1953898330684751}
{"item_id":"semeval2013.d008.s004.t001","gold_sense_keys":["poll%1:09:00::"],"candidates":[{"index":1,"sense_key":"poll%1:09:00::","synset_id":"poll.n.01"},{"index":2,"sense_key":"poll%1:08:01::","synset_id":"pate.n.02"},{"index":3,"sense_key":"poll%1:08:02::","synset_id":"poll.n.03"},{"index":4,"sense_key":"poll%1:05:00::","synset_id":"poll.n.04"},{"index":5,"sense_key":"poll%1:04:00::","synset_id":"poll.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"poll%1:09:00::","call_ids":["semeval2013.d008.s004.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"poll%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":588,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001296,"input_uncached_usd":0.001176,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.497862207936123}
{"item_id":"semeval2013.d008.s004.t002","gold_sense_keys":["country%1:14:00::","country%1:15:00::"],"candidates":[{"index":1,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":2,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":3,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":4,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":5,"sense_key":"country%1:15:02::","synset_id":"area.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"country%1:15:00::","call_ids":["semeval2013.d008.s004.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"country%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":729,"cached_input_tokens":0,"output_tokens":117,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002628,"input_uncached_usd":0.0014579999999999999,"input_cached_usd":0.0,"output_usd":0.00117,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6447856669547036}
{"item_id":"semeval2013.d008.s005.t000","gold_sense_keys":["poll%1:09:00::"],"candidates":[{"index":1,"sense_key":"poll%1:09:00::","synset_id":"poll.n.01"},{"index":2,"sense_key":"poll%1:08:01::","synset_id":"pate.n.02"},{"index":3,"sense_key":"poll%1:08:02::","synset_id":"poll.n.03"},{"index":4,"sense_key":"poll%1:05:00::","synset_id":"poll.n.04"},{"index":5,"sense_key":"poll%1:04:00::","synset_id":"poll.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"poll%1:09:00::","call_ids":["semeval2013.d008.s005.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"poll%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":613,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001346,"input_uncached_usd":0.0012259999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9515204580966383}
{"item_id":"semeval2013.d008.s005.t001","gold_sense_keys":["change%1:19:00::","change%1:24:00::"],"candidates":[{"index":1,"sense_key":"change%1:11:00::","synset_id":"change.n.01"},{"index":2,"sense_key":"change%1:24:00::","synset_id":"change.n.02"},{"index":3,"sense_key":"change%1:04:00::","synset_id":"change.n.03"},{"index":4,"sense_key":"change%1:19:00::","synset_id":"change.n.04"},{"index":5,"sense_key":"change%1:21:02::","synset_id":"change.n.05"},{"index":6,"sense_key":"change%1:06:01::","synset_id":"change.n.06"},{"index":7,"sense_key":"change%1:06:00::","synset_id":"change.n.07"},{"index":8,"sense_key":"change%1:21:01::","synset_id":"change.n.08"},{"index":9,"sense_key":"change%1:21:03::","synset_id":"change.n.09"},{"index":10,"sense_key":"change%1:07:00::","synset_id":"variety.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"change%1:24:00::","call_ids":["semeval2013.d008.s005.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"change%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":986,"cached_input_tokens":0,"output_tokens":106,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003032,"input_uncached_usd":0.001972,"input_cached_usd":0.0,"output_usd":0.0010600000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.3250200839247555}
{"item_id":"semeval2013.d008.s005.t002","gold_sense_keys":["attitude%1:09:00::"],"candidates":[{"index":1,"sense_key":"attitude%1:09:00::","synset_id":"attitude.n.01"},{"index":2,"sense_key":"attitude%1:07:01::","synset_id":"position.n.04"},{"index":3,"sense_key":"attitude%1:04:00::","synset_id":"attitude.n.03"},{"index":4,"sense_key":"attitude%1:24:00::","synset_id":"attitude.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"attitude%1:09:00::","call_ids":["semeval2013.d008.s005.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"attitude%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":651,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001422,"input_uncached_usd":0.001302,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8783177919685841}
{"item_id":"semeval2013.d008.s006.t000","gold_sense_keys":["thing%1:09:02::"],"candidates":[{"index":1,"sense_key":"thing%1:26:00::","synset_id":"thing.n.01"},{"index":2,"sense_key":"thing%1:04:00::","synset_id":"thing.n.02"},{"index":3,"sense_key":"thing%1:09:00::","synset_id":"thing.n.03"},{"index":4,"sense_key":"thing%1:06:00::","synset_id":"thing.n.04"},{"index":5,"sense_key":"thing%1:11:00::","synset_id":"thing.n.05"},{"index":6,"sense_key":"thing%1:09:02::","synset_id":"matter.n.01"},{"index":7,"sense_key":"thing%1:10:00::","synset_id":"thing.n.07"},{"index":8,"sense_key":"thing%1:06:01::","synset_id":"thing.n.08"},{"index":9,"sense_key":"thing%1:07:00::","synset_id":"thing.n.09"},{"index":10,"sense_key":"thing%1:09:01::","synset_id":"thing.n.10"},{"index":11,"sense_key":"thing%1:12:00::","synset_id":"thing.n.11"},{"index":12,"sense_key":"thing%1:03:00::","synset_id":"thing.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"thing%1:07:00::","call_ids":["semeval2013.d008.s006.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"thing%1:07:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":969,"cached_input_tokens":0,"output_tokens":271,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004648,"input_uncached_usd":0.0019379999999999998,"input_cached_usd":0.0,"output_usd":0.00271,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.7787149579962716}
{"item_id":"semeval2013.d008.s006.t001","gold_sense_keys":["result%1:11:00::"],"candidates":[{"index":1,"sense_key":"result%1:19:00::","synset_id":"consequence.n.01"},{"index":2,"sense_key":"result%1:10:00::","synset_id":"solution.n.02"},{"index":3,"sense_key":"result%1:11:00::","synset_id":"result.n.03"},{"index":4,"sense_key":"result%1:10:01::","synset_id":"resultant_role.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"result%1:11:00::","call_ids":["semeval2013.d008.s006.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"result%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":730,"cached_input_tokens":0,"output_tokens":102,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00248,"input_uncached_usd":0.00146,"input_cached_usd":0.0,"output_usd":0.00102,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6338287079706788}
{"item_id":"semeval2013.d008.s006.t002","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["semeval2013.d008.s006.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":673,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001466,"input_uncached_usd":0.001346,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3966912080068141}
{"item_id":"semeval2013.d008.s006.t003","gold_sense_keys":["poll%1:09:00::"],"candidates":[{"index":1,"sense_key":"poll%1:09:00::","synset_id":"poll.n.01"},{"index":2,"sense_key":"poll%1:08:01::","synset_id":"pate.n.02"},{"index":3,"sense_key":"poll%1:08:02::","synset_id":"poll.n.03"},{"index":4,"sense_key":"poll%1:05:00::","synset_id":"poll.n.04"},{"index":5,"sense_key":"poll%1:04:00::","synset_id":"poll.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"poll%1:09:00::","call_ids":["semeval2013.d008.s006.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"poll%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":624,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001368,"input_uncached_usd":0.001248,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":13.84664462506771}
{"item_id":"semeval2013.d008.s007.t001","gold_sense_keys":["mood%1:12:00::","mood%1:26:00::"],"candidates":[{"index":1,"sense_key":"mood%1:12:00::","synset_id":"temper.n.02"},{"index":2,"sense_key":"mood%1:26:00::","synset_id":"climate.n.02"},{"index":3,"sense_key":"mood%1:24:00::","synset_id":"mood.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"mood%1:26:00::","call_ids":["semeval2013.d008.s007.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"mood%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":619,"cached_input_tokens":0,"output_tokens":115,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0023880000000000004,"input_uncached_usd":0.001238,"input_cached_usd":0.0,"output_usd":0.0011500000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7828474999405444}
{"item_id":"semeval2013.d008.s008.t000","gold_sense_keys":["second%5:00:00:ordinal:00"],"candidates":[{"index":1,"sense_key":"second%5:00:00:ordinal:00","synset_id":"second.s.01"},{"index":2,"sense_key":"second%3:00:00::","synset_id":"second.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"second%5:00:00:ordinal:00","call_ids":["semeval2013.d008.s008.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"second%5:00:00:ordinal:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":549,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0012180000000000001,"input_uncached_usd":0.001098,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0808198750019073}
{"item_id":"semeval2013.d008.s008.t001","gold_sense_keys":["stability%1:07:01::"],"candidates":[{"index":1,"sense_key":"stability%1:07:00::","synset_id":"stability.n.01"},{"index":2,"sense_key":"stability%1:26:00::","synset_id":"stability.n.02"},{"index":3,"sense_key":"stability%1:07:01::","synset_id":"constancy.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"stability%1:07:01::","call_ids":["semeval2013.d008.s008.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"stability%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":574,"cached_input_tokens":0,"output_tokens":110,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002248,"input_uncached_usd":0.001148,"input_cached_usd":0.0,"output_usd":0.0011,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.974878333043307}
{"item_id":"semeval2013.d008.s008.t002","gold_sense_keys":["attitude%1:09:00::"],"candidates":[{"index":1,"sense_key":"attitude%1:09:00::","synset_id":"attitude.n.01"},{"index":2,"sense_key":"attitude%1:07:01::","synset_id":"position.n.04"},{"index":3,"sense_key":"attitude%1:04:00::","synset_id":"attitude.n.03"},{"index":4,"sense_key":"attitude%1:24:00::","synset_id":"attitude.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"attitude%1:09:00::","call_ids":["semeval2013.d008.s008.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"attitude%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":677,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001474,"input_uncached_usd":0.001354,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3593957079574466}
{"item_id":"semeval2013.d008.s008.t004","gold_sense_keys":["institution%1:09:00::"],"candidates":[{"index":1,"sense_key":"institution%1:14:00::","synset_id":"institution.n.01"},{"index":2,"sense_key":"institution%1:06:00::","synset_id":"institution.n.02"},{"index":3,"sense_key":"institution%1:09:00::","synset_id":"institution.n.03"},{"index":4,"sense_key":"institution%1:04:00::","synset_id":"initiation.n.02"},{"index":5,"sense_key":"institution%1:06:01::","synset_id":"mental_hospital.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"institution%1:14:00::","call_ids":["semeval2013.d008.s008.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"institution%1:14:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":763,"cached_input_tokens":0,"output_tokens":125,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002776,"input_uncached_usd":0.001526,"input_cached_usd":0.0,"output_usd":0.00125,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1385463329497725}
{"item_id":"semeval2013.d008.s009.t000","gold_sense_keys":["support%1:04:00::","support%1:09:00::"],"candidates":[{"index":1,"sense_key":"support%1:04:04::","synset_id":"support.n.01"},{"index":2,"sense_key":"support%1:04:00::","synset_id":"support.n.02"},{"index":3,"sense_key":"support%1:09:00::","synset_id":"support.n.03"},{"index":4,"sense_key":"support%1:04:03::","synset_id":"support.n.04"},{"index":5,"sense_key":"support%1:04:02::","synset_id":"documentation.n.03"},{"index":6,"sense_key":"support%1:21:00::","synset_id":"support.n.06"},{"index":7,"sense_key":"support%1:06:01::","synset_id":"support.n.07"},{"index":8,"sense_key":"support%1:04:01::","synset_id":"support.n.08"},{"index":9,"sense_key":"support%1:10:01::","synset_id":"accompaniment.n.02"},{"index":10,"sense_key":"support%1:06:00::","synset_id":"support.n.10"},{"index":11,"sense_key":"support%1:21:01::","synset_id":"support.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"support%1:09:00::","call_ids":["semeval2013.d008.s009.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"support%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1186,"cached_input_tokens":0,"output_tokens":151,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003882,"input_uncached_usd":0.002372,"input_cached_usd":0.0,"output_usd":0.00151,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8160778749734163}
{"item_id":"semeval2013.d008.s009.t002","gold_sense_keys":["country%1:14:00::","country%1:15:00::"],"candidates":[{"index":1,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":2,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":3,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":4,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":5,"sense_key":"country%1:15:02::","synset_id":"area.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"country%1:15:00::","call_ids":["semeval2013.d008.s009.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"country%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":810,"cached_input_tokens":0,"output_tokens":162,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00324,"input_uncached_usd":0.00162,"input_cached_usd":0.0,"output_usd":0.0016200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0584211670793593}
{"item_id":"semeval2013.d008.s009.t003","gold_sense_keys":["south_america%1:17:00::"],"candidates":[{"index":1,"sense_key":"south_america%1:17:00::","synset_id":"south_america.n.01"},{"index":2,"sense_key":"south_america%1:14:00::","synset_id":"south_america.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"south_america%1:14:00::","call_ids":["semeval2013.d008.s009.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"south_america%1:14:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":562,"cached_input_tokens":0,"output_tokens":123,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002354,"input_uncached_usd":0.001124,"input_cached_usd":0.0,"output_usd":0.0012300000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":14.54953249997925}
{"item_id":"semeval2013.d008.s009.t004","gold_sense_keys":["table%1:14:00::"],"candidates":[{"index":1,"sense_key":"table%1:14:00::","synset_id":"table.n.01"},{"index":2,"sense_key":"table%1:06:01::","synset_id":"table.n.02"},{"index":3,"sense_key":"table%1:06:02::","synset_id":"table.n.03"},{"index":4,"sense_key":"table%1:17:00::","synset_id":"mesa.n.01"},{"index":5,"sense_key":"table%1:14:01::","synset_id":"table.n.05"},{"index":6,"sense_key":"table%1:13:01::","synset_id":"board.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"table%1:14:00::","call_ids":["semeval2013.d008.s009.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"table%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":808,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0017360000000000001,"input_uncached_usd":0.001616,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8586899170186371}
{"item_id":"semeval2013.d008.s010.t003","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["semeval2013.d008.s010.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":706,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001532,"input_uncached_usd":0.0014119999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5275922500295565}
{"item_id":"semeval2013.d008.s010.t004","gold_sense_keys":["support%1:04:00::","support%1:09:00::"],"candidates":[{"index":1,"sense_key":"support%1:04:04::","synset_id":"support.n.01"},{"index":2,"sense_key":"support%1:04:00::","synset_id":"support.n.02"},{"index":3,"sense_key":"support%1:09:00::","synset_id":"support.n.03"},{"index":4,"sense_key":"support%1:04:03::","synset_id":"support.n.04"},{"index":5,"sense_key":"support%1:04:02::","synset_id":"documentation.n.03"},{"index":6,"sense_key":"support%1:21:00::","synset_id":"support.n.06"},{"index":7,"sense_key":"support%1:06:01::","synset_id":"support.n.07"},{"index":8,"sense_key":"support%1:04:01::","synset_id":"support.n.08"},{"index":9,"sense_key":"support%1:10:01::","synset_id":"accompaniment.n.02"},{"index":10,"sense_key":"support%1:06:00::","synset_id":"support.n.10"},{"index":11,"sense_key":"support%1:21:01::","synset_id":"support.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"support%1:09:00::","call_ids":["semeval2013.d008.s010.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"support%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1174,"cached_input_tokens":0,"output_tokens":113,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003478,"input_uncached_usd":0.002348,"input_cached_usd":0.0,"output_usd":0.0011300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.644105082959868}
{"item_id":"semeval2013.d008.s010.t006","gold_sense_keys":["low%1:07:00::"],"candidates":[{"index":1,"sense_key":"low%1:26:00::","synset_id":"low.n.01"},{"index":2,"sense_key":"low%1:18:00::","synset_id":"low.n.02"},{"index":3,"sense_key":"low%1:07:00::","synset_id":"low.n.03"},{"index":4,"sense_key":"low%1:06:00::","synset_id":"first_gear.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"low%1:07:00::","call_ids":["semeval2013.d008.s010.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"low%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":711,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001542,"input_uncached_usd":0.0014219999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9780716249952093}
{"item_id":"semeval2013.d008.s010.t007","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["semeval2013.d008.s010.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":706,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001532,"input_uncached_usd":0.0014119999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3530300420243293}
{"item_id":"semeval2013.d008.s011.t001","gold_sense_keys":["economy%1:14:00::"],"candidates":[{"index":1,"sense_key":"economy%1:14:00::","synset_id":"economy.n.01"},{"index":2,"sense_key":"economy%1:09:01::","synset_id":"economy.n.02"},{"index":3,"sense_key":"economy%1:07:00::","synset_id":"economy.n.03"},{"index":4,"sense_key":"economy%1:04:00::","synset_id":"economy.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"economy%1:14:00::","call_ids":["semeval2013.d008.s011.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"economy%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":685,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00149,"input_uncached_usd":0.00137,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4622211669338867}
{"item_id":"semeval2013.d008.s011.t002","gold_sense_keys":["drop%1:07:00::"],"candidates":[{"index":1,"sense_key":"drop%1:25:00::","synset_id":"drop.n.01"},{"index":2,"sense_key":"drop%1:23:00::","synset_id":"drop.n.02"},{"index":3,"sense_key":"drop%1:07:00::","synset_id":"drop.n.03"},{"index":4,"sense_key":"drop%1:17:00::","synset_id":"cliff.n.01"},{"index":5,"sense_key":"drop%1:15:00::","synset_id":"drop.n.05"},{"index":6,"sense_key":"drop%1:11:00::","synset_id":"drop.n.06"},{"index":7,"sense_key":"drop%1:06:01::","synset_id":"drop_curtain.n.01"},{"index":8,"sense_key":"drop%1:06:00::","synset_id":"drop.n.08"},{"index":9,"sense_key":"drop%1:04:00::","synset_id":"drop.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"drop%1:07:00::","call_ids":["semeval2013.d008.s011.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"drop%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1066,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0022519999999999997,"input_uncached_usd":0.0021319999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":13.871488541015424}
{"item_id":"semeval2013.d008.s011.t003","gold_sense_keys":["output%1:04:00::","output%1:06:01::","output%1:23:00::"],"candidates":[{"index":1,"sense_key":"output%1:06:00::","synset_id":"end_product.n.01"},{"index":2,"sense_key":"output%1:04:00::","synset_id":"output.n.02"},{"index":3,"sense_key":"output%1:10:02::","synset_id":"output_signal.n.01"},{"index":4,"sense_key":"output%1:23:00::","synset_id":"output.n.04"},{"index":5,"sense_key":"output%1:06:01::","synset_id":"output.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"output%1:23:00::","call_ids":["semeval2013.d008.s011.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"output%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":694,"cached_input_tokens":0,"output_tokens":148,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002868,"input_uncached_usd":0.001388,"input_cached_usd":0.0,"output_usd":0.0014800000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.345128000015393}
{"item_id":"semeval2013.d008.s011.t004","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["semeval2013.d008.s011.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":727,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001574,"input_uncached_usd":0.001454,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4220519579248503}
{"item_id":"semeval2013.d008.s012.t000","gold_sense_keys":["half%1:23:00::"],"candidates":[{"index":1,"sense_key":"half%1:23:00::","synset_id":"one-half.n.01"},{"index":2,"sense_key":"half%1:28:00::","synset_id":"half.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"half%1:23:00::","call_ids":["semeval2013.d008.s012.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"half%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":614,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001348,"input_uncached_usd":0.001228,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0953637500060722}
{"item_id":"semeval2013.d008.s012.t001","gold_sense_keys":["respondent%1:18:00::"],"candidates":[{"index":1,"sense_key":"respondent%1:18:01::","synset_id":"respondent.n.01"},{"index":2,"sense_key":"respondent%1:18:00::","synset_id":"respondent.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"respondent%1:18:00::","call_ids":["semeval2013.d008.s012.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"respondent%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":605,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00133,"input_uncached_usd":0.00121,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.372844332945533}
{"item_id":"semeval2013.d008.s012.t003","gold_sense_keys":["brazil%1:15:00::"],"candidates":[{"index":1,"sense_key":"brazil%1:15:00::","synset_id":"brazil.n.01"},{"index":2,"sense_key":"brazil%1:13:00::","synset_id":"brazil_nut.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"brazil%1:15:00::","call_ids":["semeval2013.d008.s012.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"brazil%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":662,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001444,"input_uncached_usd":0.0013239999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1428049170644954}
{"item_id":"semeval2013.d008.s012.t005","gold_sense_keys":["country%1:14:00::","country%1:15:00::"],"candidates":[{"index":1,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":2,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":3,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":4,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":5,"sense_key":"country%1:15:02::","synset_id":"area.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"country%1:14:00::","call_ids":["semeval2013.d008.s012.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"country%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":859,"cached_input_tokens":0,"output_tokens":297,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004688,"input_uncached_usd":0.001718,"input_cached_usd":0.0,"output_usd":0.0029700000000000004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.112222582916729}
{"item_id":"semeval2013.d008.s012.t006","gold_sense_keys":["democrat%1:18:00::"],"candidates":[{"index":1,"sense_key":"democrat%1:18:01::","synset_id":"democrat.n.01"},{"index":2,"sense_key":"democrat%1:18:00::","synset_id":"democrat.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"democrat%1:18:00::","call_ids":["semeval2013.d008.s012.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"democrat%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":584,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001288,"input_uncached_usd":0.001168,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9880503749009222}
{"item_id":"semeval2013.d008.s012.t007","gold_sense_keys":["average%1:09:00::"],"candidates":[{"index":1,"sense_key":"average%1:09:00::","synset_id":"average.n.01"},{"index":2,"sense_key":"average%1:24:00::","synset_id":"average.n.02"},{"index":3,"sense_key":"average%1:09:01::","synset_id":"average.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"average%1:09:00::","call_ids":["semeval2013.d008.s012.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"average%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":659,"cached_input_tokens":0,"output_tokens":107,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002388,"input_uncached_usd":0.001318,"input_cached_usd":0.0,"output_usd":0.00107,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":14.834785750019364}
{"item_id":"semeval2013.d008.s012.t008","gold_sense_keys":["chart%1:10:00::"],"candidates":[{"index":1,"sense_key":"chart%1:10:00::","synset_id":"chart.n.01"},{"index":2,"sense_key":"chart%1:06:00::","synset_id":"chart.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"chart%1:10:00::","call_ids":["semeval2013.d008.s012.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"chart%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":573,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001266,"input_uncached_usd":0.0011459999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.165738792042248}
{"item_id":"semeval2013.d008.s013.t000","gold_sense_keys":["respondent%1:18:00::"],"candidates":[{"index":1,"sense_key":"respondent%1:18:01::","synset_id":"respondent.n.01"},{"index":2,"sense_key":"respondent%1:18:00::","synset_id":"respondent.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"respondent%1:18:00::","call_ids":["semeval2013.d008.s013.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"respondent%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":644,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001408,"input_uncached_usd":0.0012879999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9850924999918789}
{"item_id":"semeval2013.d008.s013.t002","gold_sense_keys":["country%1:14:00::"],"candidates":[{"index":1,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":2,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":3,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":4,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":5,"sense_key":"country%1:15:02::","synset_id":"area.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"country%1:14:00::","call_ids":["semeval2013.d008.s013.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"country%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":898,"cached_input_tokens":0,"output_tokens":116,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002956,"input_uncached_usd":0.0017959999999999999,"input_cached_usd":0.0,"output_usd":0.00116,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.841131875058636}
{"item_id":"semeval2013.d008.s013.t003","gold_sense_keys":["democracy%1:14:00::"],"candidates":[{"index":1,"sense_key":"democracy%1:09:00::","synset_id":"democracy.n.01"},{"index":2,"sense_key":"democracy%1:14:00::","synset_id":"democracy.n.02"},{"index":3,"sense_key":"democracy%1:09:01::","synset_id":"majority_rule.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"democracy%1:14:00::","call_ids":["semeval2013.d008.s013.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"democracy%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":721,"cached_input_tokens":0,"output_tokens":46,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001902,"input_uncached_usd":0.001442,"input_cached_usd":0.0,"output_usd":0.00046,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6928496250184253}
{"item_id":"semeval2013.d008.s013.t005","gold_sense_keys":["figure%1:07:01::"],"candidates":[{"index":1,"sense_key":"figure%1:10:00::","synset_id":"figure.n.01"},{"index":2,"sense_key":"figure%1:08:00::","synset_id":"human_body.n.01"},{"index":3,"sense_key":"figure%1:23:00::","synset_id":"digit.n.01"},{"index":4,"sense_key":"figure%1:06:00::","synset_id":"figure.n.04"},{"index":5,"sense_key":"figure%1:18:00::","synset_id":"name.n.04"},{"index":6,"sense_key":"figure%1:25:00::","synset_id":"figure.n.06"},{"index":7,"sense_key":"figure%1:21:00::","synset_id":"figure.n.07"},{"index":8,"sense_key":"figure%1:07:00::","synset_id":"figure.n.08"},{"index":9,"sense_key":"figure%1:07:01::","synset_id":"number.n.01"},{"index":10,"sense_key":"figure%1:10:01::","synset_id":"trope.n.01"},{"index":11,"sense_key":"figure%1:09:00::","synset_id":"figure.n.11"},{"index":12,"sense_key":"figure%1:06:01::","synset_id":"design.n.04"},{"index":13,"sense_key":"figure%1:04:00::","synset_id":"figure.n.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"figure%1:21:00::","call_ids":["semeval2013.d008.s013.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"figure%1:21:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1297,"cached_input_tokens":0,"output_tokens":219,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004784,"input_uncached_usd":0.002594,"input_cached_usd":0.0,"output_usd":0.00219,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.8831035828916356}
{"item_id":"semeval2013.d008.s013.t006","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["semeval2013.d008.s013.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":805,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00173,"input_uncached_usd":0.0016099999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4260865000542253}
{"item_id":"semeval2013.d008.s013.t007","gold_sense_keys":["increase%1:07:00::","increase%1:11:00::"],"candidates":[{"index":1,"sense_key":"increase%1:23:00::","synset_id":"addition.n.03"},{"index":2,"sense_key":"increase%1:11:00::","synset_id":"increase.n.02"},{"index":3,"sense_key":"increase%1:22:00::","synset_id":"increase.n.03"},{"index":4,"sense_key":"increase%1:07:00::","synset_id":"increase.n.04"},{"index":5,"sense_key":"increase%1:04:00::","synset_id":"increase.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"increase%1:07:00::","call_ids":["semeval2013.d008.s013.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"increase%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":852,"cached_input_tokens":0,"output_tokens":95,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002654,"input_uncached_usd":0.001704,"input_cached_usd":0.0,"output_usd":0.0009500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6269297499675304}
{"item_id":"semeval2013.d008.s013.t008","gold_sense_keys":["chart%1:10:00::"],"candidates":[{"index":1,"sense_key":"chart%1:10:00::","synset_id":"chart.n.01"},{"index":2,"sense_key":"chart%1:06:00::","synset_id":"chart.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"chart%1:10:00::","call_ids":["semeval2013.d008.s013.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"chart%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":612,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0013440000000000001,"input_uncached_usd":0.001224,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4095594579121098}
{"item_id":"semeval2013.d008.s014.t000","gold_sense_keys":["confidence%1:12:00::","confidence%1:26:02::"],"candidates":[{"index":1,"sense_key":"confidence%1:09:00::","synset_id":"assurance.n.01"},{"index":2,"sense_key":"confidence%1:12:00::","synset_id":"confidence.n.02"},{"index":3,"sense_key":"confidence%1:26:02::","synset_id":"confidence.n.03"},{"index":4,"sense_key":"confidence%1:26:03::","synset_id":"confidence.n.04"},{"index":5,"sense_key":"confidence%1:10:00::","synset_id":"confidence.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"confidence%1:12:00::","call_ids":["semeval2013.d008.s014.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"confidence%1:12:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":887,"cached_input_tokens":0,"output_tokens":139,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003164,"input_uncached_usd":0.001774,"input_cached_usd":0.0,"output_usd":0.0013900000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.4891772500704974}
{"item_id":"semeval2013.d008.s014.t001","gold_sense_keys":["country%1:14:00::"],"candidates":[{"index":1,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":2,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":3,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":4,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":5,"sense_key":"country%1:15:02::","synset_id":"area.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"country%1:14:00::","call_ids":["semeval2013.d008.s014.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"country%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":895,"cached_input_tokens":0,"output_tokens":100,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00279,"input_uncached_usd":0.00179,"input_cached_usd":0.0,"output_usd":0.001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6142461659619585}
{"item_id":"semeval2013.d008.s014.t002","gold_sense_keys":["congress%1:14:02::"],"candidates":[{"index":1,"sense_key":"congress%1:14:01::","synset_id":"congress.n.01"},{"index":2,"sense_key":"congress%1:14:00::","synset_id":"congress.n.02"},{"index":3,"sense_key":"congress%1:14:02::","synset_id":"congress.n.03"},{"index":4,"sense_key":"congress%1:04:01::","synset_id":"sexual_intercourse.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"congress%1:14:02::","call_ids":["semeval2013.d008.s014.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"congress%1:14:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":766,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002062,"input_uncached_usd":0.001532,"input_cached_usd":0.0,"output_usd":0.0005300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7300650000106543}
{"item_id":"semeval2013.d008.s014.t003","gold_sense_keys":["government%1:14:00::"],"candidates":[{"index":1,"sense_key":"government%1:14:00::","synset_id":"government.n.01"},{"index":2,"sense_key":"government%1:04:00::","synset_id":"government.n.02"},{"index":3,"sense_key":"government%1:09:00::","synset_id":"government.n.03"},{"index":4,"sense_key":"government%1:09:01::","synset_id":"politics.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"government%1:14:00::","call_ids":["semeval2013.d008.s014.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"government%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":783,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001686,"input_uncached_usd":0.001566,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8501441669650376}
{"item_id":"semeval2013.d008.s014.t004","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["semeval2013.d008.s014.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":802,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001724,"input_uncached_usd":0.001604,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.032808208023198}
{"item_id":"semeval2013.d008.s014.t005","gold_sense_keys":["case%1:11:00::"],"candidates":[{"index":1,"sense_key":"case%1:11:00::","synset_id":"case.n.01"},{"index":2,"sense_key":"case%1:26:00::","synset_id":"event.n.02"},{"index":3,"sense_key":"case%1:04:00::","synset_id":"lawsuit.n.01"},{"index":4,"sense_key":"case%1:09:00::","synset_id":"case.n.04"},{"index":5,"sense_key":"case%1:06:00::","synset_id":"case.n.05"},{"index":6,"sense_key":"case%1:18:03::","synset_id":"case.n.06"},{"index":7,"sense_key":"case%1:18:02::","synset_id":"subject.n.06"},{"index":8,"sense_key":"case%1:10:02::","synset_id":"case.n.08"},{"index":9,"sense_key":"case%1:10:01::","synset_id":"case.n.09"},{"index":10,"sense_key":"case%1:23:00::","synset_id":"case.n.10"},{"index":11,"sense_key":"case%1:10:00::","synset_id":"case.n.11"},{"index":12,"sense_key":"case%1:26:02::","synset_id":"case.n.12"},{"index":13,"sense_key":"case%1:18:04::","synset_id":"character.n.05"},{"index":14,"sense_key":"case%1:10:03::","synset_id":"font.n.01"},{"index":15,"sense_key":"case%1:08:00::","synset_id":"sheath.n.02"},{"index":16,"sense_key":"case%1:06:06::","synset_id":"shell.n.08"},{"index":17,"sense_key":"case%1:06:07::","synset_id":"casing.n.03"},{"index":18,"sense_key":"case%1:06:03::","synset_id":"case.n.18"},{"index":19,"sense_key":"case%1:06:02::","synset_id":"case.n.19"},{"index":20,"sense_key":"case%1:06:01::","synset_id":"case.n.20"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"case%1:11:00::","call_ids":["semeval2013.d008.s014.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"case%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1797,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004244,"input_uncached_usd":0.003594,"input_cached_usd":0.0,"output_usd":0.0006500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.335561417043209}
{"item_id":"semeval2013.d008.s015.t000","gold_sense_keys":["decade%1:28:00::"],"candidates":[{"index":1,"sense_key":"decade%1:28:00::","synset_id":"decade.n.01"},{"index":2,"sense_key":"decade%1:23:00::","synset_id":"ten.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"decade%1:28:00::","call_ids":["semeval2013.d008.s015.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"decade%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":663,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001446,"input_uncached_usd":0.001326,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.7699547080555931}
{"item_id":"semeval2013.d008.s015.t001","gold_sense_keys":["woe%1:26:00::"],"candidates":[{"index":1,"sense_key":"woe%1:26:00::","synset_id":"suffering.n.02"},{"index":2,"sense_key":"woe%1:12:00::","synset_id":"woe.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"woe%1:26:00::","call_ids":["semeval2013.d008.s015.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"woe%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":636,"cached_input_tokens":0,"output_tokens":95,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002222,"input_uncached_usd":0.001272,"input_cached_usd":0.0,"output_usd":0.0009500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5611512910109013}
{"item_id":"semeval2013.d008.s015.t003","gold_sense_keys":["support%1:04:00::","support%1:09:00::"],"candidates":[{"index":1,"sense_key":"support%1:04:04::","synset_id":"support.n.01"},{"index":2,"sense_key":"support%1:04:00::","synset_id":"support.n.02"},{"index":3,"sense_key":"support%1:09:00::","synset_id":"support.n.03"},{"index":4,"sense_key":"support%1:04:03::","synset_id":"support.n.04"},{"index":5,"sense_key":"support%1:04:02::","synset_id":"documentation.n.03"},{"index":6,"sense_key":"support%1:21:00::","synset_id":"support.n.06"},{"index":7,"sense_key":"support%1:06:01::","synset_id":"support.n.07"},{"index":8,"sense_key":"support%1:04:01::","synset_id":"support.n.08"},{"index":9,"sense_key":"support%1:10:01::","synset_id":"accompaniment.n.02"},{"index":10,"sense_key":"support%1:06:00::","synset_id":"support.n.10"},{"index":11,"sense_key":"support%1:21:01::","synset_id":"support.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"support%1:09:00::","call_ids":["semeval2013.d008.s015.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"support%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1283,"cached_input_tokens":0,"output_tokens":185,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004416,"input_uncached_usd":0.0025659999999999997,"input_cached_usd":0.0,"output_usd":0.00185,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1645046250196174}
{"item_id":"semeval2013.d008.s016.t001","gold_sense_keys":["country%1:14:00::"],"candidates":[{"index":1,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":2,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":3,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":4,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":5,"sense_key":"country%1:15:02::","synset_id":"area.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"country%1:14:00::","call_ids":["semeval2013.d008.s016.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"country%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":877,"cached_input_tokens":0,"output_tokens":123,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002984,"input_uncached_usd":0.001754,"input_cached_usd":0.0,"output_usd":0.0012300000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.188277709065005}
{"item_id":"semeval2013.d008.s016.t002","gold_sense_keys":["recession%1:26:00::"],"candidates":[{"index":1,"sense_key":"recession%1:26:00::","synset_id":"recession.n.01"},{"index":2,"sense_key":"recession%1:25:00::","synset_id":"recess.n.02"},{"index":3,"sense_key":"recession%1:14:00::","synset_id":"recession.n.03"},{"index":4,"sense_key":"recession%1:04:01::","synset_id":"recession.n.04"},{"index":5,"sense_key":"recession%1:04:00::","synset_id":"receding.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"recession%1:26:00::","call_ids":["semeval2013.d008.s016.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"recession%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":764,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001648,"input_uncached_usd":0.0015279999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1020614160224795}
{"item_id":"semeval2013.d008.s016.t003","gold_sense_keys":["recovery%1:11:00::"],"candidates":[{"index":1,"sense_key":"recovery%1:11:00::","synset_id":"recovery.n.01"},{"index":2,"sense_key":"recovery%1:22:00::","synset_id":"convalescence.n.01"},{"index":3,"sense_key":"recovery%1:04:00::","synset_id":"recovery.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"recovery%1:11:00::","call_ids":["semeval2013.d008.s016.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"recovery%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":693,"cached_input_tokens":0,"output_tokens":95,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002336,"input_uncached_usd":0.0013859999999999999,"input_cached_usd":0.0,"output_usd":0.0009500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8848212499869987}
{"item_id":"semeval2013.d008.s017.t001","gold_sense_keys":["safety_net%1:10:00::"],"candidates":[{"index":1,"sense_key":"safety_net%1:10:00::","synset_id":"safety_net.n.01"},{"index":2,"sense_key":"safety_net%1:06:00::","synset_id":"safety_net.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"safety_net%1:10:00::","call_ids":["semeval2013.d008.s017.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"safety_net%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":653,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001426,"input_uncached_usd":0.0013059999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3139183749444783}
{"item_id":"semeval2013.d008.s018.t002","gold_sense_keys":["week%1:28:02::"],"candidates":[{"index":1,"sense_key":"week%1:28:00::","synset_id":"week.n.01"},{"index":2,"sense_key":"week%1:28:01::","synset_id":"workweek.n.01"},{"index":3,"sense_key":"week%1:28:02::","synset_id":"week.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"week%1:28:00::","call_ids":["semeval2013.d008.s018.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"week%1:28:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":685,"cached_input_tokens":0,"output_tokens":174,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00311,"input_uncached_usd":0.00137,"input_cached_usd":0.0,"output_usd":0.0017400000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":14.98527062498033}
{"item_id":"semeval2013.d008.s018.t003","gold_sense_keys":["proportion%1:24:00::"],"candidates":[{"index":1,"sense_key":"proportion%1:24:00::","synset_id":"proportion.n.01"},{"index":2,"sense_key":"proportion%1:07:00::","synset_id":"proportion.n.02"},{"index":3,"sense_key":"proportion%1:25:00::","synset_id":"symmetry.n.02"},{"index":4,"sense_key":"proportion%1:24:01::","synset_id":"proportion.n.04"},{"index":5,"sense_key":"proportion%1:07:01::","synset_id":"proportion.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"proportion%1:24:00::","call_ids":["semeval2013.d008.s018.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"proportion%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":885,"cached_input_tokens":0,"output_tokens":238,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00415,"input_uncached_usd":0.0017699999999999999,"input_cached_usd":0.0,"output_usd":0.00238,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.7442496670410037}
{"item_id":"semeval2013.d008.s018.t005","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["semeval2013.d008.s018.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":789,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001698,"input_uncached_usd":0.001578,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4634417499182746}
{"item_id":"semeval2013.d008.s019.t000","gold_sense_keys":["mood%1:12:00::","mood%1:26:00::"],"candidates":[{"index":1,"sense_key":"mood%1:12:00::","synset_id":"temper.n.02"},{"index":2,"sense_key":"mood%1:26:00::","synset_id":"climate.n.02"},{"index":3,"sense_key":"mood%1:24:00::","synset_id":"mood.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"mood%1:26:00::","call_ids":["semeval2013.d008.s019.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"mood%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":731,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001582,"input_uncached_usd":0.001462,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4765286250039935}
{"item_id":"semeval2013.d008.s019.t001","gold_sense_keys":["country%1:15:00::"],"candidates":[{"index":1,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":2,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":3,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":4,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":5,"sense_key":"country%1:15:02::","synset_id":"area.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"country%1:14:00::","call_ids":["semeval2013.d008.s019.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"country%1:14:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":873,"cached_input_tokens":0,"output_tokens":127,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003016,"input_uncached_usd":0.001746,"input_cached_usd":0.0,"output_usd":0.00127,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.781045125098899}
{"item_id":"semeval2013.d008.s019.t002","gold_sense_keys":["country%1:15:00::"],"candidates":[{"index":1,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":2,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":3,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":4,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":5,"sense_key":"country%1:15:02::","synset_id":"area.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"country%1:14:00::","call_ids":["semeval2013.d008.s019.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"country%1:14:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":873,"cached_input_tokens":0,"output_tokens":229,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0040360000000000005,"input_uncached_usd":0.001746,"input_cached_usd":0.0,"output_usd":0.0022900000000000004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":4.0039823750266805}
{"item_id":"semeval2013.d008.s019.t003","gold_sense_keys":["chart%1:10:00::"],"candidates":[{"index":1,"sense_key":"chart%1:10:00::","synset_id":"chart.n.01"},{"index":2,"sense_key":"chart%1:06:00::","synset_id":"chart.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"chart%1:10:00::","call_ids":["semeval2013.d008.s019.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"chart%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":588,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001296,"input_uncached_usd":0.001176,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0281461249105632}
{"item_id":"semeval2013.d008.s020.t000","gold_sense_keys":["brazil%1:15:00::"],"candidates":[{"index":1,"sense_key":"brazil%1:15:00::","synset_id":"brazil.n.01"},{"index":2,"sense_key":"brazil%1:13:00::","synset_id":"brazil_nut.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"brazil%1:15:00::","call_ids":["semeval2013.d008.s020.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"brazil%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":656,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0014320000000000001,"input_uncached_usd":0.001312,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1454509169561788}
{"item_id":"semeval2013.d008.s020.t003","gold_sense_keys":["president%1:18:00::"],"candidates":[{"index":1,"sense_key":"president%1:18:01::","synset_id":"president.n.01"},{"index":2,"sense_key":"president%1:18:04::","synset_id":"president_of_the_united_states.n.01"},{"index":3,"sense_key":"president%1:18:00::","synset_id":"president.n.03"},{"index":4,"sense_key":"president%1:18:03::","synset_id":"president.n.04"},{"index":5,"sense_key":"president%1:18:02::","synset_id":"president.n.05"},{"index":6,"sense_key":"president%1:04:04::","synset_id":"president_of_the_united_states.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"president%1:18:00::","call_ids":["semeval2013.d008.s020.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"president%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":812,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001744,"input_uncached_usd":0.001624,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0112601249711588}
{"item_id":"semeval2013.d008.s020.t004","gold_sense_keys":["sense%1:09:05::"],"candidates":[{"index":1,"sense_key":"sense%1:09:05::","synset_id":"sense.n.01"},{"index":2,"sense_key":"sense%1:10:00::","synset_id":"sense.n.02"},{"index":3,"sense_key":"sense%1:09:02::","synset_id":"sense.n.03"},{"index":4,"sense_key":"sense%1:09:04::","synset_id":"common_sense.n.01"},{"index":5,"sense_key":"sense%1:09:06::","synset_id":"sense.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"sense%1:09:05::","call_ids":["semeval2013.d008.s020.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"sense%1:09:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":862,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001844,"input_uncached_usd":0.0017239999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8532400419935584}
{"item_id":"semeval2013.d008.s020.t005","gold_sense_keys":["progress%1:04:01::"],"candidates":[{"index":1,"sense_key":"progress%1:04:01::","synset_id":"advancement.n.03"},{"index":2,"sense_key":"progress%1:04:00::","synset_id":"progress.n.02"},{"index":3,"sense_key":"progress%1:11:00::","synset_id":"progress.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"progress%1:04:01::","call_ids":["semeval2013.d008.s020.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"progress%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":681,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001482,"input_uncached_usd":0.001362,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8286460000090301}
{"item_id":"semeval2013.d008.s021.t002","gold_sense_keys":["recession%1:26:00::"],"candidates":[{"index":1,"sense_key":"recession%1:26:00::","synset_id":"recession.n.01"},{"index":2,"sense_key":"recession%1:25:00::","synset_id":"recess.n.02"},{"index":3,"sense_key":"recession%1:14:00::","synset_id":"recession.n.03"},{"index":4,"sense_key":"recession%1:04:01::","synset_id":"recession.n.04"},{"index":5,"sense_key":"recession%1:04:00::","synset_id":"receding.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"recession%1:26:00::","call_ids":["semeval2013.d008.s021.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"recession%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":770,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00166,"input_uncached_usd":0.00154,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":11.724597834050655}
{"item_id":"semeval2013.d008.s021.t003","gold_sense_keys":["crime%1:04:00::"],"candidates":[{"index":1,"sense_key":"crime%1:04:00::","synset_id":"crime.n.01"},{"index":2,"sense_key":"crime%1:04:01::","synset_id":"crime.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"crime%1:04:00::","call_ids":["semeval2013.d008.s021.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"crime%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":663,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001916,"input_uncached_usd":0.001326,"input_cached_usd":0.0,"output_usd":0.00059,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7456852920586243}
{"item_id":"semeval2013.d008.s022.t000","gold_sense_keys":["time%1:11:00::"],"candidates":[{"index":1,"sense_key":"time%1:11:00::","synset_id":"time.n.01"},{"index":2,"sense_key":"time%1:28:05::","synset_id":"time.n.02"},{"index":3,"sense_key":"time%1:28:00::","synset_id":"time.n.03"},{"index":4,"sense_key":"time%1:28:06::","synset_id":"time.n.04"},{"index":5,"sense_key":"time%1:03:00::","synset_id":"time.n.05"},{"index":6,"sense_key":"time%1:11:01::","synset_id":"time.n.06"},{"index":7,"sense_key":"time%1:28:03::","synset_id":"clock_time.n.01"},{"index":8,"sense_key":"time%1:28:04::","synset_id":"fourth_dimension.n.01"},{"index":9,"sense_key":"time%1:07:01::","synset_id":"meter.n.04"},{"index":10,"sense_key":"time%1:28:01::","synset_id":"prison_term.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"time%1:11:00::","call_ids":["semeval2013.d008.s022.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"time%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1103,"cached_input_tokens":0,"output_tokens":45,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002656,"input_uncached_usd":0.002206,"input_cached_usd":0.0,"output_usd":0.00045000000000000004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7058509170310572}
{"item_id":"semeval2013.d008.s022.t001","gold_sense_keys":["poll%1:09:00::"],"candidates":[{"index":1,"sense_key":"poll%1:09:00::","synset_id":"poll.n.01"},{"index":2,"sense_key":"poll%1:08:01::","synset_id":"pate.n.02"},{"index":3,"sense_key":"poll%1:08:02::","synset_id":"poll.n.03"},{"index":4,"sense_key":"poll%1:05:00::","synset_id":"poll.n.04"},{"index":5,"sense_key":"poll%1:04:00::","synset_id":"poll.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"poll%1:09:00::","call_ids":["semeval2013.d008.s022.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"poll%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":744,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001608,"input_uncached_usd":0.001488,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":13.680025375098921}
{"item_id":"semeval2013.d008.s022.t002","gold_sense_keys":["respondent%1:18:00::"],"candidates":[{"index":1,"sense_key":"respondent%1:18:01::","synset_id":"respondent.n.01"},{"index":2,"sense_key":"respondent%1:18:00::","synset_id":"respondent.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"respondent%1:18:00::","call_ids":["semeval2013.d008.s022.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"respondent%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":631,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001382,"input_uncached_usd":0.0012619999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3889226249884814}
{"item_id":"semeval2013.d008.s022.t004","gold_sense_keys":["whole%1:09:00::"],"candidates":[{"index":1,"sense_key":"whole%1:09:00::","synset_id":"whole.n.01"},{"index":2,"sense_key":"whole%1:03:00::","synset_id":"whole.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"whole%1:09:00::","call_ids":["semeval2013.d008.s022.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"whole%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":660,"cached_input_tokens":0,"output_tokens":41,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00173,"input_uncached_usd":0.00132,"input_cached_usd":0.0,"output_usd":0.00041000000000000005,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3820752910105512}
{"item_id":"semeval2013.d008.s022.t005","gold_sense_keys":["crime%1:04:00::"],"candidates":[{"index":1,"sense_key":"crime%1:04:00::","synset_id":"crime.n.01"},{"index":2,"sense_key":"crime%1:04:01::","synset_id":"crime.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"crime%1:04:00::","call_ids":["semeval2013.d008.s022.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"crime%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":665,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0014500000000000001,"input_uncached_usd":0.00133,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":7.672648707986809}
{"item_id":"semeval2013.d008.s022.t007","gold_sense_keys":["problem%1:09:00::","problem%1:26:00::"],"candidates":[{"index":1,"sense_key":"problem%1:26:00::","synset_id":"problem.n.01"},{"index":2,"sense_key":"problem%1:10:00::","synset_id":"problem.n.02"},{"index":3,"sense_key":"problem%1:09:00::","synset_id":"trouble.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"problem%1:09:00::","call_ids":["semeval2013.d008.s022.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"problem%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":712,"cached_input_tokens":0,"output_tokens":135,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002774,"input_uncached_usd":0.001424,"input_cached_usd":0.0,"output_usd":0.00135,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7805701670004055}
{"item_id":"semeval2013.d008.s022.t008","gold_sense_keys":["country%1:14:00::"],"candidates":[{"index":1,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":2,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":3,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":4,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":5,"sense_key":"country%1:15:02::","synset_id":"area.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"country%1:14:00::","call_ids":["semeval2013.d008.s022.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"country%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":884,"cached_input_tokens":0,"output_tokens":122,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002988,"input_uncached_usd":0.0017679999999999998,"input_cached_usd":0.0,"output_usd":0.0012200000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2282624170184135}
{"item_id":"semeval2013.d008.s022.t009","gold_sense_keys":["chart%1:10:00::"],"candidates":[{"index":1,"sense_key":"chart%1:10:00::","synset_id":"chart.n.01"},{"index":2,"sense_key":"chart%1:06:00::","synset_id":"chart.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"chart%1:10:00::","call_ids":["semeval2013.d008.s022.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"chart%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":599,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001318,"input_uncached_usd":0.0011979999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":10.361269750050269}
{"item_id":"semeval2013.d008.s023.t000","gold_sense_keys":["relative%1:18:00::"],"candidates":[{"index":1,"sense_key":"relative%1:18:00::","synset_id":"relative.n.01"},{"index":2,"sense_key":"relative%1:05:00::","synset_id":"relative.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"relative%1:18:00::","call_ids":["semeval2013.d008.s023.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"relative%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":725,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00157,"input_uncached_usd":0.00145,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.125018542050384}
{"item_id":"semeval2013.d008.s023.t001","gold_sense_keys":["victim%1:18:00::"],"candidates":[{"index":1,"sense_key":"victim%1:18:00::","synset_id":"victim.n.01"},{"index":2,"sense_key":"victim%1:18:01::","synset_id":"victim.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"victim%1:18:00::","call_ids":["semeval2013.d008.s023.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"victim%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":667,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001454,"input_uncached_usd":0.001334,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9975437079556286}
{"item_id":"semeval2013.d008.s023.t002","gold_sense_keys":["crime%1:04:00::"],"candidates":[{"index":1,"sense_key":"crime%1:04:00::","synset_id":"crime.n.01"},{"index":2,"sense_key":"crime%1:04:01::","synset_id":"crime.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"crime%1:04:00::","call_ids":["semeval2013.d008.s023.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"crime%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":714,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001998,"input_uncached_usd":0.001428,"input_cached_usd":0.0,"output_usd":0.0005700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7295928749954328}
{"item_id":"semeval2013.d008.s023.t003","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["semeval2013.d008.s023.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":841,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001802,"input_uncached_usd":0.001682,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0522148750023916}
{"item_id":"semeval2013.d008.s023.t004","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["semeval2013.d008.s023.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":841,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001802,"input_uncached_usd":0.001682,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8611274169525132}
{"item_id":"semeval2013.d008.s023.t005","gold_sense_keys":["figure%1:07:01::"],"candidates":[{"index":1,"sense_key":"figure%1:10:00::","synset_id":"figure.n.01"},{"index":2,"sense_key":"figure%1:08:00::","synset_id":"human_body.n.01"},{"index":3,"sense_key":"figure%1:23:00::","synset_id":"digit.n.01"},{"index":4,"sense_key":"figure%1:06:00::","synset_id":"figure.n.04"},{"index":5,"sense_key":"figure%1:18:00::","synset_id":"name.n.04"},{"index":6,"sense_key":"figure%1:25:00::","synset_id":"figure.n.06"},{"index":7,"sense_key":"figure%1:21:00::","synset_id":"figure.n.07"},{"index":8,"sense_key":"figure%1:07:00::","synset_id":"figure.n.08"},{"index":9,"sense_key":"figure%1:07:01::","synset_id":"number.n.01"},{"index":10,"sense_key":"figure%1:10:01::","synset_id":"trope.n.01"},{"index":11,"sense_key":"figure%1:09:00::","synset_id":"figure.n.11"},{"index":12,"sense_key":"figure%1:06:01::","synset_id":"design.n.04"},{"index":13,"sense_key":"figure%1:04:00::","synset_id":"figure.n.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"figure%1:07:01::","call_ids":["semeval2013.d008.s023.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"figure%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1333,"cached_input_tokens":0,"output_tokens":173,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004396000000000001,"input_uncached_usd":0.002666,"input_cached_usd":0.0,"output_usd":0.0017300000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.4707394998986274}
{"item_id":"semeval2013.d008.s024.t001","gold_sense_keys":["support%1:04:00::","support%1:09:00::"],"candidates":[{"index":1,"sense_key":"support%1:04:04::","synset_id":"support.n.01"},{"index":2,"sense_key":"support%1:04:00::","synset_id":"support.n.02"},{"index":3,"sense_key":"support%1:09:00::","synset_id":"support.n.03"},{"index":4,"sense_key":"support%1:04:03::","synset_id":"support.n.04"},{"index":5,"sense_key":"support%1:04:02::","synset_id":"documentation.n.03"},{"index":6,"sense_key":"support%1:21:00::","synset_id":"support.n.06"},{"index":7,"sense_key":"support%1:06:01::","synset_id":"support.n.07"},{"index":8,"sense_key":"support%1:04:01::","synset_id":"support.n.08"},{"index":9,"sense_key":"support%1:10:01::","synset_id":"accompaniment.n.02"},{"index":10,"sense_key":"support%1:06:00::","synset_id":"support.n.10"},{"index":11,"sense_key":"support%1:21:01::","synset_id":"support.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"support%1:09:00::","call_ids":["semeval2013.d008.s024.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"support%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1272,"cached_input_tokens":0,"output_tokens":118,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003724,"input_uncached_usd":0.002544,"input_cached_usd":0.0,"output_usd":0.00118,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6473594579147175}
{"item_id":"semeval2013.d008.s024.t004","gold_sense_keys":["respondent%1:18:00::"],"candidates":[{"index":1,"sense_key":"respondent%1:18:01::","synset_id":"respondent.n.01"},{"index":2,"sense_key":"respondent%1:18:00::","synset_id":"respondent.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"respondent%1:18:00::","call_ids":["semeval2013.d008.s024.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"respondent%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":643,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0014060000000000001,"input_uncached_usd":0.001286,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4589810830075294}
{"item_id":"semeval2013.d008.s024.t007","gold_sense_keys":["development%1:04:01::","development%1:22:02::","development%1:26:00::"],"candidates":[{"index":1,"sense_key":"development%1:04:01::","synset_id":"development.n.01"},{"index":2,"sense_key":"development%1:22:02::","synset_id":"development.n.02"},{"index":3,"sense_key":"development%1:22:01::","synset_id":"growth.n.01"},{"index":4,"sense_key":"development%1:11:01::","synset_id":"development.n.04"},{"index":5,"sense_key":"development%1:04:02::","synset_id":"exploitation.n.01"},{"index":6,"sense_key":"development%1:15:00::","synset_id":"development.n.06"},{"index":7,"sense_key":"development%1:26:00::","synset_id":"development.n.07"},{"index":8,"sense_key":"development%1:22:00::","synset_id":"development.n.08"},{"index":9,"sense_key":"development%1:10:00::","synset_id":"development.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"development%1:22:02::","call_ids":["semeval2013.d008.s024.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"development%1:22:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1209,"cached_input_tokens":0,"output_tokens":191,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004328,"input_uncached_usd":0.002418,"input_cached_usd":0.0,"output_usd":0.0019100000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0580822499468923}
{"item_id":"semeval2013.d008.s024.t008","gold_sense_keys":["chart%1:10:00::"],"candidates":[{"index":1,"sense_key":"chart%1:10:00::","synset_id":"chart.n.01"},{"index":2,"sense_key":"chart%1:06:00::","synset_id":"chart.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"chart%1:10:00::","call_ids":["semeval2013.d008.s024.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"chart%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":611,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001342,"input_uncached_usd":0.001222,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.838858624920249}
{"item_id":"semeval2013.d008.s025.t001","gold_sense_keys":["system%1:14:00::"],"candidates":[{"index":1,"sense_key":"system%1:06:00::","synset_id":"system.n.01"},{"index":2,"sense_key":"system%1:14:00::","synset_id":"system.n.02"},{"index":3,"sense_key":"system%1:27:00::","synset_id":"system.n.03"},{"index":4,"sense_key":"system%1:09:00::","synset_id":"system.n.04"},{"index":5,"sense_key":"system%1:09:02::","synset_id":"arrangement.n.03"},{"index":6,"sense_key":"system%1:08:00::","synset_id":"system.n.06"},{"index":7,"sense_key":"system%1:09:01::","synset_id":"system.n.07"},{"index":8,"sense_key":"system%1:08:01::","synset_id":"system.n.08"},{"index":9,"sense_key":"system%1:07:00::","synset_id":"organization.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"system%1:14:00::","call_ids":["semeval2013.d008.s025.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"system%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1254,"cached_input_tokens":0,"output_tokens":128,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0037879999999999997,"input_uncached_usd":0.002508,"input_cached_usd":0.0,"output_usd":0.00128,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7575702919857576}
{"item_id":"semeval2013.d008.s025.t002","gold_sense_keys":["country%1:14:00::"],"candidates":[{"index":1,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":2,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":3,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":4,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":5,"sense_key":"country%1:15:02::","synset_id":"area.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"country%1:14:00::","call_ids":["semeval2013.d008.s025.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"country%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":941,"cached_input_tokens":0,"output_tokens":138,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003262,"input_uncached_usd":0.001882,"input_cached_usd":0.0,"output_usd":0.0013800000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.040253582992591}
{"item_id":"semeval2013.d008.s025.t003","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["semeval2013.d008.s025.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":849,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001818,"input_uncached_usd":0.0016979999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8695949589600787}
{"item_id":"semeval2013.d008.s026.t000","gold_sense_keys":["respondent%1:18:00::"],"candidates":[{"index":1,"sense_key":"respondent%1:18:01::","synset_id":"respondent.n.01"},{"index":2,"sense_key":"respondent%1:18:00::","synset_id":"respondent.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"respondent%1:18:00::","call_ids":["semeval2013.d008.s026.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"respondent%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":708,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001536,"input_uncached_usd":0.001416,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9342194580240175}
{"item_id":"semeval2013.d008.s026.t002","gold_sense_keys":["view%1:09:02::","view%1:09:04::"],"candidates":[{"index":1,"sense_key":"view%1:09:02::","synset_id":"position.n.03"},{"index":2,"sense_key":"view%1:09:01::","synset_id":"view.n.02"},{"index":3,"sense_key":"view%1:04:00::","synset_id":"view.n.03"},{"index":4,"sense_key":"view%1:15:00::","synset_id":"view.n.04"},{"index":5,"sense_key":"view%1:09:04::","synset_id":"opinion.n.01"},{"index":6,"sense_key":"view%1:10:00::","synset_id":"opinion.n.02"},{"index":7,"sense_key":"view%1:09:00::","synset_id":"view.n.07"},{"index":8,"sense_key":"view%1:06:00::","synset_id":"scene.n.08"},{"index":9,"sense_key":"view%1:07:00::","synset_id":"horizon.n.02"},{"index":10,"sense_key":"view%1:07:01::","synset_id":"view.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"view%1:09:02::","call_ids":["semeval2013.d008.s026.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"view%1:09:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1304,"cached_input_tokens":0,"output_tokens":132,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003928,"input_uncached_usd":0.002608,"input_cached_usd":0.0,"output_usd":0.0013200000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.042248541023582}
{"item_id":"semeval2013.d008.s026.t004","gold_sense_keys":["president%1:18:00::"],"candidates":[{"index":1,"sense_key":"president%1:18:01::","synset_id":"president.n.01"},{"index":2,"sense_key":"president%1:18:04::","synset_id":"president_of_the_united_states.n.01"},{"index":3,"sense_key":"president%1:18:00::","synset_id":"president.n.03"},{"index":4,"sense_key":"president%1:18:03::","synset_id":"president.n.04"},{"index":5,"sense_key":"president%1:18:02::","synset_id":"president.n.05"},{"index":6,"sense_key":"president%1:04:04::","synset_id":"president_of_the_united_states.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"president%1:18:00::","call_ids":["semeval2013.d008.s026.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"president%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":922,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001964,"input_uncached_usd":0.001844,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0160450839903206}
{"item_id":"semeval2013.d008.s026.t005","gold_sense_keys":["company%1:14:01::"],"candidates":[{"index":1,"sense_key":"company%1:14:01::","synset_id":"company.n.01"},{"index":2,"sense_key":"company%1:14:03::","synset_id":"company.n.02"},{"index":3,"sense_key":"company%1:26:00::","synset_id":"company.n.03"},{"index":4,"sense_key":"company%1:14:02::","synset_id":"company.n.04"},{"index":5,"sense_key":"company%1:18:00::","synset_id":"caller.n.01"},{"index":6,"sense_key":"company%1:14:00::","synset_id":"company.n.06"},{"index":7,"sense_key":"company%1:14:06::","synset_id":"party.n.03"},{"index":8,"sense_key":"company%1:14:04::","synset_id":"ship's_company.n.01"},{"index":9,"sense_key":"company%1:14:05::","synset_id":"company.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"company%1:14:01::","call_ids":["semeval2013.d008.s026.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"company%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1107,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0023339999999999997,"input_uncached_usd":0.002214,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5262102910783142}
{"item_id":"semeval2013.d008.s027.t000","gold_sense_keys":["brazil%1:15:00::"],"candidates":[{"index":1,"sense_key":"brazil%1:15:00::","synset_id":"brazil.n.01"},{"index":2,"sense_key":"brazil%1:13:00::","synset_id":"brazil_nut.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"brazil%1:15:00::","call_ids":["semeval2013.d008.s027.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"brazil%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":799,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0017180000000000001,"input_uncached_usd":0.001598,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9966598330065608}
{"item_id":"semeval2013.d008.s027.t001","gold_sense_keys":["country%1:14:00::"],"candidates":[{"index":1,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":2,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":3,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":4,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":5,"sense_key":"country%1:15:02::","synset_id":"area.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"country%1:14:00::","call_ids":["semeval2013.d008.s027.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"country%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":995,"cached_input_tokens":0,"output_tokens":112,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00311,"input_uncached_usd":0.00199,"input_cached_usd":0.0,"output_usd":0.0011200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7125969999469817}
{"item_id":"semeval2013.d008.s027.t002","gold_sense_keys":["leadership%1:04:00::","leadership%1:09:00::","leadership%1:26:00::"],"candidates":[{"index":1,"sense_key":"leadership%1:04:00::","synset_id":"leadership.n.01"},{"index":2,"sense_key":"leadership%1:14:00::","synset_id":"leadership.n.02"},{"index":3,"sense_key":"leadership%1:26:00::","synset_id":"leadership.n.03"},{"index":4,"sense_key":"leadership%1:09:00::","synset_id":"leadership.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"leadership%1:04:00::","call_ids":["semeval2013.d008.s027.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"leadership%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":841,"cached_input_tokens":0,"output_tokens":107,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002752,"input_uncached_usd":0.001682,"input_cached_usd":0.0,"output_usd":0.00107,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5884765420341864}
{"item_id":"semeval2013.d008.s027.t004","gold_sense_keys":["respondent%1:18:00::"],"candidates":[{"index":1,"sense_key":"respondent%1:18:01::","synset_id":"respondent.n.01"},{"index":2,"sense_key":"respondent%1:18:00::","synset_id":"respondent.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"respondent%1:18:00::","call_ids":["semeval2013.d008.s027.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"respondent%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":742,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001604,"input_uncached_usd":0.0014839999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0097747499821708}
{"item_id":"semeval2013.d008.s027.t005","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["semeval2013.d008.s027.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":903,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001926,"input_uncached_usd":0.0018059999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3925589169375598}
{"item_id":"semeval2013.d008.s027.t006","gold_sense_keys":["united_states%1:15:00::"],"candidates":[{"index":1,"sense_key":"united_states%1:15:00::","synset_id":"united_states.n.01"},{"index":2,"sense_key":"united_states%1:14:00::","synset_id":"united_states_government.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"united_states%1:15:00::","call_ids":["semeval2013.d008.s027.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"united_states%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":820,"cached_input_tokens":0,"output_tokens":74,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00238,"input_uncached_usd":0.00164,"input_cached_usd":0.0,"output_usd":0.0007400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8581017079995945}
{"item_id":"semeval2013.d008.s027.t007","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["semeval2013.d008.s027.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":903,"cached_input_tokens":0,"output_tokens":52,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002326,"input_uncached_usd":0.0018059999999999999,"input_cached_usd":0.0,"output_usd":0.0005200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.406603124924004}
{"item_id":"semeval2013.d008.s027.t009","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["semeval2013.d008.s027.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":903,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001926,"input_uncached_usd":0.0018059999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4819551670225337}
{"item_id":"semeval2013.d008.s028.t000","gold_sense_keys":["united_states%1:15:00::"],"candidates":[{"index":1,"sense_key":"united_states%1:15:00::","synset_id":"united_states.n.01"},{"index":2,"sense_key":"united_states%1:14:00::","synset_id":"united_states_government.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"united_states%1:15:00::","call_ids":["semeval2013.d008.s028.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"united_states%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":766,"cached_input_tokens":0,"output_tokens":81,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002342,"input_uncached_usd":0.001532,"input_cached_usd":0.0,"output_usd":0.0008100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6513142919866368}
{"item_id":"semeval2013.d008.s028.t001","gold_sense_keys":["country%1:14:00::"],"candidates":[{"index":1,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":2,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":3,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":4,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":5,"sense_key":"country%1:15:02::","synset_id":"area.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"country%1:14:00::","call_ids":["semeval2013.d008.s028.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"country%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":941,"cached_input_tokens":0,"output_tokens":130,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003182,"input_uncached_usd":0.001882,"input_cached_usd":0.0,"output_usd":0.0013000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1377445419784635}
{"item_id":"semeval2013.d008.s028.t002","gold_sense_keys":["respondent%1:18:00::"],"candidates":[{"index":1,"sense_key":"respondent%1:18:01::","synset_id":"respondent.n.01"},{"index":2,"sense_key":"respondent%1:18:00::","synset_id":"respondent.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"respondent%1:18:00::","call_ids":["semeval2013.d008.s028.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"respondent%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":688,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001496,"input_uncached_usd":0.0013759999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.057164124911651}
{"item_id":"semeval2013.d008.s028.t006","gold_sense_keys":["distinction%1:07:01::"],"candidates":[{"index":1,"sense_key":"distinction%1:09:00::","synset_id":"differentiation.n.01"},{"index":2,"sense_key":"distinction%1:26:00::","synset_id":"eminence.n.01"},{"index":3,"sense_key":"distinction%1:07:01::","synset_id":"distinction.n.03"},{"index":4,"sense_key":"distinction%1:07:00::","synset_id":"distinction.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"distinction%1:07:01::","call_ids":["semeval2013.d008.s028.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"distinction%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":827,"cached_input_tokens":0,"output_tokens":136,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003014,"input_uncached_usd":0.0016539999999999999,"input_cached_usd":0.0,"output_usd":0.00136,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1139395410427824}
{"item_id":"semeval2013.d009.s000.t001","gold_sense_keys":["law%1:10:00::"],"candidates":[{"index":1,"sense_key":"law%1:14:00::","synset_id":"law.n.01"},{"index":2,"sense_key":"law%1:10:00::","synset_id":"law.n.02"},{"index":3,"sense_key":"law%1:09:00::","synset_id":"law.n.03"},{"index":4,"sense_key":"law%1:09:01::","synset_id":"law.n.04"},{"index":5,"sense_key":"law%1:09:02::","synset_id":"jurisprudence.n.01"},{"index":6,"sense_key":"law%1:04:00::","synset_id":"law.n.06"},{"index":7,"sense_key":"law%1:14:01::","synset_id":"police.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"law%1:10:00::","call_ids":["semeval2013.d009.s000.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"law%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":819,"cached_input_tokens":0,"output_tokens":80,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002438,"input_uncached_usd":0.001638,"input_cached_usd":0.0,"output_usd":0.0008,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":14.221582083031535}
{"item_id":"semeval2013.d009.s000.t003","gold_sense_keys":["court%1:14:00::"],"candidates":[{"index":1,"sense_key":"court%1:14:00::","synset_id":"court.n.01"},{"index":2,"sense_key":"court%1:06:02::","synset_id":"court.n.02"},{"index":3,"sense_key":"court%1:14:01::","synset_id":"court.n.03"},{"index":4,"sense_key":"court%1:06:01::","synset_id":"court.n.04"},{"index":5,"sense_key":"court%1:18:00::","synset_id":"court.n.05"},{"index":6,"sense_key":"court%1:14:02::","synset_id":"court.n.06"},{"index":7,"sense_key":"court%1:06:03::","synset_id":"motor_hotel.n.01"},{"index":8,"sense_key":"court%1:06:05::","synset_id":"court.n.08"},{"index":9,"sense_key":"court%1:06:04::","synset_id":"court.n.09"},{"index":10,"sense_key":"court%1:06:00::","synset_id":"court.n.10"},{"index":11,"sense_key":"court%1:04:00::","synset_id":"court.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"court%1:14:00::","call_ids":["semeval2013.d009.s000.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"court%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":976,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004672,"input_uncached_usd":0.001952,"input_cached_usd":0.0,"output_usd":0.00272,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.8371597919613123}
{"item_id":"semeval2013.d009.s000.t004","gold_sense_keys":["case%1:04:00::"],"candidates":[{"index":1,"sense_key":"case%1:11:00::","synset_id":"case.n.01"},{"index":2,"sense_key":"case%1:26:00::","synset_id":"event.n.02"},{"index":3,"sense_key":"case%1:04:00::","synset_id":"lawsuit.n.01"},{"index":4,"sense_key":"case%1:09:00::","synset_id":"case.n.04"},{"index":5,"sense_key":"case%1:06:00::","synset_id":"case.n.05"},{"index":6,"sense_key":"case%1:18:03::","synset_id":"case.n.06"},{"index":7,"sense_key":"case%1:18:02::","synset_id":"subject.n.06"},{"index":8,"sense_key":"case%1:10:02::","synset_id":"case.n.08"},{"index":9,"sense_key":"case%1:10:01::","synset_id":"case.n.09"},{"index":10,"sense_key":"case%1:23:00::","synset_id":"case.n.10"},{"index":11,"sense_key":"case%1:10:00::","synset_id":"case.n.11"},{"index":12,"sense_key":"case%1:26:02::","synset_id":"case.n.12"},{"index":13,"sense_key":"case%1:18:04::","synset_id":"character.n.05"},{"index":14,"sense_key":"case%1:10:03::","synset_id":"font.n.01"},{"index":15,"sense_key":"case%1:08:00::","synset_id":"sheath.n.02"},{"index":16,"sense_key":"case%1:06:06::","synset_id":"shell.n.08"},{"index":17,"sense_key":"case%1:06:07::","synset_id":"casing.n.03"},{"index":18,"sense_key":"case%1:06:03::","synset_id":"case.n.18"},{"index":19,"sense_key":"case%1:06:02::","synset_id":"case.n.19"},{"index":20,"sense_key":"case%1:06:01::","synset_id":"case.n.20"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"case%1:04:00::","call_ids":["semeval2013.d009.s000.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"case%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1611,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003342,"input_uncached_usd":0.003222,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4118913339916617}
{"item_id":"semeval2013.d009.s001.t000","gold_sense_keys":["day%1:28:03::"],"candidates":[{"index":1,"sense_key":"day%1:28:00::","synset_id":"day.n.01"},{"index":2,"sense_key":"day%1:28:03::","synset_id":"day.n.02"},{"index":3,"sense_key":"day%1:28:01::","synset_id":"day.n.03"},{"index":4,"sense_key":"day%1:28:02::","synset_id":"day.n.04"},{"index":5,"sense_key":"day%1:28:04::","synset_id":"day.n.05"},{"index":6,"sense_key":"day%1:28:05::","synset_id":"day.n.06"},{"index":7,"sense_key":"day%1:28:07::","synset_id":"day.n.07"},{"index":8,"sense_key":"day%1:28:06::","synset_id":"sidereal_day.n.01"},{"index":9,"sense_key":"day%1:26:00::","synset_id":"day.n.09"},{"index":10,"sense_key":"day%1:18:00::","synset_id":"day.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"day%1:28:03::","call_ids":["semeval2013.d009.s001.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"day%1:28:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1044,"cached_input_tokens":0,"output_tokens":118,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003268,"input_uncached_usd":0.002088,"input_cached_usd":0.0,"output_usd":0.00118,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.124023374984972}
{"item_id":"semeval2013.d009.s001.t001","gold_sense_keys":["supreme_court%1:14:00::"],"candidates":[{"index":1,"sense_key":"supreme_court%1:14:00::","synset_id":"supreme_court.n.01"},{"index":2,"sense_key":"supreme_court%1:14:01::","synset_id":"supreme_court.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"supreme_court%1:14:00::","call_ids":["semeval2013.d009.s001.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"supreme_court%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":538,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001196,"input_uncached_usd":0.001076,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0805073750671}
{"item_id":"semeval2013.d009.s001.t002","gold_sense_keys":["question%1:10:01::"],"candidates":[{"index":1,"sense_key":"question%1:10:00::","synset_id":"question.n.01"},{"index":2,"sense_key":"question%1:10:01::","synset_id":"question.n.02"},{"index":3,"sense_key":"question%1:10:03::","synset_id":"question.n.03"},{"index":4,"sense_key":"question%1:07:00::","synset_id":"doubt.n.02"},{"index":5,"sense_key":"question%1:10:02::","synset_id":"motion.n.05"},{"index":6,"sense_key":"question%1:10:04::","synset_id":"question.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"question%1:10:01::","call_ids":["semeval2013.d009.s001.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"question%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":825,"cached_input_tokens":0,"output_tokens":70,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00235,"input_uncached_usd":0.00165,"input_cached_usd":0.0,"output_usd":0.0007000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4118667090078816}
{"item_id":"semeval2013.d009.s001.t005","gold_sense_keys":["law%1:10:00::"],"candidates":[{"index":1,"sense_key":"law%1:14:00::","synset_id":"law.n.01"},{"index":2,"sense_key":"law%1:10:00::","synset_id":"law.n.02"},{"index":3,"sense_key":"law%1:09:00::","synset_id":"law.n.03"},{"index":4,"sense_key":"law%1:09:01::","synset_id":"law.n.04"},{"index":5,"sense_key":"law%1:09:02::","synset_id":"jurisprudence.n.01"},{"index":6,"sense_key":"law%1:04:00::","synset_id":"law.n.06"},{"index":7,"sense_key":"law%1:14:01::","synset_id":"police.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"law%1:10:00::","call_ids":["semeval2013.d009.s001.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"law%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":852,"cached_input_tokens":0,"output_tokens":78,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002484,"input_uncached_usd":0.001704,"input_cached_usd":0.0,"output_usd":0.0007800000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":5.0812405839096755}
{"item_id":"semeval2013.d009.s001.t006","gold_sense_keys":["justice%1:18:00::"],"candidates":[{"index":1,"sense_key":"justice%1:07:00::","synset_id":"justice.n.01"},{"index":2,"sense_key":"justice%1:04:00::","synset_id":"justice.n.02"},{"index":3,"sense_key":"justice%1:18:00::","synset_id":"judge.n.01"},{"index":4,"sense_key":"justice%1:14:00::","synset_id":"department_of_justice.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"justice%1:18:00::","call_ids":["semeval2013.d009.s001.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"justice%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":617,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001354,"input_uncached_usd":0.0012339999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9750832910649478}
{"item_id":"semeval2013.d009.s001.t007","gold_sense_keys":["dinner%1:14:00::"],"candidates":[{"index":1,"sense_key":"dinner%1:13:00::","synset_id":"dinner.n.01"},{"index":2,"sense_key":"dinner%1:14:00::","synset_id":"dinner.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"dinner%1:14:00::","call_ids":["semeval2013.d009.s001.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"dinner%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":526,"cached_input_tokens":0,"output_tokens":71,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0017620000000000001,"input_uncached_usd":0.001052,"input_cached_usd":0.0,"output_usd":0.00071,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.771162207936868}
{"item_id":"semeval2013.d009.s001.t009","gold_sense_keys":["case%1:04:00::"],"candidates":[{"index":1,"sense_key":"case%1:11:00::","synset_id":"case.n.01"},{"index":2,"sense_key":"case%1:26:00::","synset_id":"event.n.02"},{"index":3,"sense_key":"case%1:04:00::","synset_id":"lawsuit.n.01"},{"index":4,"sense_key":"case%1:09:00::","synset_id":"case.n.04"},{"index":5,"sense_key":"case%1:06:00::","synset_id":"case.n.05"},{"index":6,"sense_key":"case%1:18:03::","synset_id":"case.n.06"},{"index":7,"sense_key":"case%1:18:02::","synset_id":"subject.n.06"},{"index":8,"sense_key":"case%1:10:02::","synset_id":"case.n.08"},{"index":9,"sense_key":"case%1:10:01::","synset_id":"case.n.09"},{"index":10,"sense_key":"case%1:23:00::","synset_id":"case.n.10"},{"index":11,"sense_key":"case%1:10:00::","synset_id":"case.n.11"},{"index":12,"sense_key":"case%1:26:02::","synset_id":"case.n.12"},{"index":13,"sense_key":"case%1:18:04::","synset_id":"character.n.05"},{"index":14,"sense_key":"case%1:10:03::","synset_id":"font.n.01"},{"index":15,"sense_key":"case%1:08:00::","synset_id":"sheath.n.02"},{"index":16,"sense_key":"case%1:06:06::","synset_id":"shell.n.08"},{"index":17,"sense_key":"case%1:06:07::","synset_id":"casing.n.03"},{"index":18,"sense_key":"case%1:06:03::","synset_id":"case.n.18"},{"index":19,"sense_key":"case%1:06:02::","synset_id":"case.n.19"},{"index":20,"sense_key":"case%1:06:01::","synset_id":"case.n.20"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"case%1:04:00::","call_ids":["semeval2013.d009.s001.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"case%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1644,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0034079999999999996,"input_uncached_usd":0.0032879999999999997,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0169219159288332}
{"item_id":"semeval2013.d009.s002.t002","gold_sense_keys":["justice%1:18:00::"],"candidates":[{"index":1,"sense_key":"justice%1:07:00::","synset_id":"justice.n.01"},{"index":2,"sense_key":"justice%1:04:00::","synset_id":"justice.n.02"},{"index":3,"sense_key":"justice%1:18:00::","synset_id":"judge.n.01"},{"index":4,"sense_key":"justice%1:14:00::","synset_id":"department_of_justice.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"justice%1:18:00::","call_ids":["semeval2013.d009.s002.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"justice%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":683,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0014860000000000001,"input_uncached_usd":0.001366,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9686879999935627}
{"item_id":"semeval2013.d009.s002.t003","gold_sense_keys":["conference%1:10:00::","conference%1:14:00::"],"candidates":[{"index":1,"sense_key":"conference%1:14:00::","synset_id":"conference.n.01"},{"index":2,"sense_key":"conference%1:14:01::","synset_id":"league.n.01"},{"index":3,"sense_key":"conference%1:10:00::","synset_id":"conference.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"conference%1:14:00::","call_ids":["semeval2013.d009.s002.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"conference%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":610,"cached_input_tokens":0,"output_tokens":94,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00216,"input_uncached_usd":0.00122,"input_cached_usd":0.0,"output_usd":0.0009400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3085115839494392}
{"item_id":"semeval2013.d009.s002.t004","gold_sense_keys":["petition%1:10:00::"],"candidates":[{"index":1,"sense_key":"petition%1:10:00::","synset_id":"request.n.01"},{"index":2,"sense_key":"petition%1:10:01::","synset_id":"prayer.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"petition%1:10:00::","call_ids":["semeval2013.d009.s002.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"petition%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":552,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001224,"input_uncached_usd":0.001104,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8521589160664007}
{"item_id":"semeval2013.d009.s002.t005","gold_sense_keys":["review%1:04:02::","review%1:04:03::","review%1:09:00::"],"candidates":[{"index":1,"sense_key":"review%1:09:00::","synset_id":"reappraisal.n.01"},{"index":2,"sense_key":"review%1:10:01::","synset_id":"review.n.02"},{"index":3,"sense_key":"review%1:04:00::","synset_id":"follow-up.n.03"},{"index":4,"sense_key":"review%1:21:00::","synset_id":"review.n.04"},{"index":5,"sense_key":"review%1:10:02::","synset_id":"revue.n.01"},{"index":6,"sense_key":"review%1:10:03::","synset_id":"review.n.06"},{"index":7,"sense_key":"review%1:10:00::","synset_id":"recapitulation.n.03"},{"index":8,"sense_key":"review%1:04:03::","synset_id":"review.n.08"},{"index":9,"sense_key":"review%1:04:01::","synset_id":"review.n.09"},{"index":10,"sense_key":"review%1:04:02::","synset_id":"inspection.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"review%1:04:03::","call_ids":["semeval2013.d009.s002.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"review%1:04:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1017,"cached_input_tokens":0,"output_tokens":52,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002554,"input_uncached_usd":0.002034,"input_cached_usd":0.0,"output_usd":0.0005200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":14.198281250079162}
{"item_id":"semeval2013.d009.s003.t000","gold_sense_keys":["case%1:04:00::"],"candidates":[{"index":1,"sense_key":"case%1:11:00::","synset_id":"case.n.01"},{"index":2,"sense_key":"case%1:26:00::","synset_id":"event.n.02"},{"index":3,"sense_key":"case%1:04:00::","synset_id":"lawsuit.n.01"},{"index":4,"sense_key":"case%1:09:00::","synset_id":"case.n.04"},{"index":5,"sense_key":"case%1:06:00::","synset_id":"case.n.05"},{"index":6,"sense_key":"case%1:18:03::","synset_id":"case.n.06"},{"index":7,"sense_key":"case%1:18:02::","synset_id":"subject.n.06"},{"index":8,"sense_key":"case%1:10:02::","synset_id":"case.n.08"},{"index":9,"sense_key":"case%1:10:01::","synset_id":"case.n.09"},{"index":10,"sense_key":"case%1:23:00::","synset_id":"case.n.10"},{"index":11,"sense_key":"case%1:10:00::","synset_id":"case.n.11"},{"index":12,"sense_key":"case%1:26:02::","synset_id":"case.n.12"},{"index":13,"sense_key":"case%1:18:04::","synset_id":"character.n.05"},{"index":14,"sense_key":"case%1:10:03::","synset_id":"font.n.01"},{"index":15,"sense_key":"case%1:08:00::","synset_id":"sheath.n.02"},{"index":16,"sense_key":"case%1:06:06::","synset_id":"shell.n.08"},{"index":17,"sense_key":"case%1:06:07::","synset_id":"casing.n.03"},{"index":18,"sense_key":"case%1:06:03::","synset_id":"case.n.18"},{"index":19,"sense_key":"case%1:06:02::","synset_id":"case.n.19"},{"index":20,"sense_key":"case%1:06:01::","synset_id":"case.n.20"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"case%1:04:00::","call_ids":["semeval2013.d009.s003.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"case%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1788,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0036959999999999996,"input_uncached_usd":0.0035759999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1271552910329774}
{"item_id":"semeval2013.d009.s003.t002","gold_sense_keys":["suit%1:04:00::"],"candidates":[{"index":1,"sense_key":"suit%1:06:00::","synset_id":"suit.n.01"},{"index":2,"sense_key":"suit%1:04:00::","synset_id":"lawsuit.n.01"},{"index":3,"sense_key":"suit%1:18:00::","synset_id":"suit.n.03"},{"index":4,"sense_key":"suit%1:10:01::","synset_id":"courtship.n.01"},{"index":5,"sense_key":"suit%1:10:00::","synset_id":"suit.n.05"},{"index":6,"sense_key":"suit%1:06:01::","synset_id":"suit.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"suit%1:04:00::","call_ids":["semeval2013.d009.s003.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"suit%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":965,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0020499999999999997,"input_uncached_usd":0.0019299999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.054726833011955}
{"item_id":"semeval2013.d009.s003.t003","gold_sense_keys":["state%1:15:01::"],"candidates":[{"index":1,"sense_key":"state%1:15:01::","synset_id":"state.n.01"},{"index":2,"sense_key":"state%1:03:00::","synset_id":"state.n.02"},{"index":3,"sense_key":"state%1:14:01::","synset_id":"state.n.03"},{"index":4,"sense_key":"state%1:14:00::","synset_id":"state.n.04"},{"index":5,"sense_key":"state%1:26:02::","synset_id":"state_of_matter.n.01"},{"index":6,"sense_key":"state%1:26:01::","synset_id":"state.n.06"},{"index":7,"sense_key":"state%1:15:00::","synset_id":"country.n.02"},{"index":8,"sense_key":"state%1:14:02::","synset_id":"department_of_state.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"state%1:15:01::","call_ids":["semeval2013.d009.s003.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"state%1:15:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1065,"cached_input_tokens":0,"output_tokens":224,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004370000000000001,"input_uncached_usd":0.00213,"input_cached_usd":0.0,"output_usd":0.0022400000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.4611370840575546}
{"item_id":"semeval2013.d009.s003.t005","gold_sense_keys":["overhaul%1:04:01::"],"candidates":[{"index":1,"sense_key":"overhaul%1:04:00::","synset_id":"overhaul.n.01"},{"index":2,"sense_key":"overhaul%1:04:01::","synset_id":"renovation.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"overhaul%1:04:01::","call_ids":["semeval2013.d009.s003.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"overhaul%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":699,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001518,"input_uncached_usd":0.001398,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4268906249199063}
{"item_id":"semeval2013.d009.s003.t006","gold_sense_keys":["congress%1:14:01::"],"candidates":[{"index":1,"sense_key":"congress%1:14:01::","synset_id":"congress.n.01"},{"index":2,"sense_key":"congress%1:14:00::","synset_id":"congress.n.02"},{"index":3,"sense_key":"congress%1:14:02::","synset_id":"congress.n.03"},{"index":4,"sense_key":"congress%1:04:01::","synset_id":"sexual_intercourse.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"congress%1:14:01::","call_ids":["semeval2013.d009.s003.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"congress%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":757,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001634,"input_uncached_usd":0.001514,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8039322500117123}
{"item_id":"semeval2013.d009.s003.t007","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["semeval2013.d009.s003.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":793,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001706,"input_uncached_usd":0.001586,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1420816669706255}
{"item_id":"semeval2013.d009.s003.t008","gold_sense_keys":["law%1:10:00::"],"candidates":[{"index":1,"sense_key":"law%1:14:00::","synset_id":"law.n.01"},{"index":2,"sense_key":"law%1:10:00::","synset_id":"law.n.02"},{"index":3,"sense_key":"law%1:09:00::","synset_id":"law.n.03"},{"index":4,"sense_key":"law%1:09:01::","synset_id":"law.n.04"},{"index":5,"sense_key":"law%1:09:02::","synset_id":"jurisprudence.n.01"},{"index":6,"sense_key":"law%1:04:00::","synset_id":"law.n.06"},{"index":7,"sense_key":"law%1:14:01::","synset_id":"police.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"law%1:10:00::","call_ids":["semeval2013.d009.s003.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"law%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":995,"cached_input_tokens":0,"output_tokens":88,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00287,"input_uncached_usd":0.00199,"input_cached_usd":0.0,"output_usd":0.00088,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8743930839700624}
{"item_id":"semeval2013.d009.s003.t009","gold_sense_keys":["rallying_cry%1:10:00::"],"candidates":[{"index":1,"sense_key":"rallying_cry%1:10:00::","synset_id":"war_cry.n.01"},{"index":2,"sense_key":"rallying_cry%1:10:01::","synset_id":"war_cry.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"rallying_cry%1:10:00::","call_ids":["semeval2013.d009.s003.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"rallying_cry%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":687,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001494,"input_uncached_usd":0.001374,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.260170958004892}
{"item_id":"semeval2013.d009.s004.t000","gold_sense_keys":["justice%1:18:00::"],"candidates":[{"index":1,"sense_key":"justice%1:07:00::","synset_id":"justice.n.01"},{"index":2,"sense_key":"justice%1:04:00::","synset_id":"justice.n.02"},{"index":3,"sense_key":"justice%1:18:00::","synset_id":"judge.n.01"},{"index":4,"sense_key":"justice%1:14:00::","synset_id":"department_of_justice.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"justice%1:18:00::","call_ids":["semeval2013.d009.s004.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"justice%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":819,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001758,"input_uncached_usd":0.001638,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.059590584016405}
{"item_id":"semeval2013.d009.s004.t001","gold_sense_keys":["suit%1:04:00::"],"candidates":[{"index":1,"sense_key":"suit%1:06:00::","synset_id":"suit.n.01"},{"index":2,"sense_key":"suit%1:04:00::","synset_id":"lawsuit.n.01"},{"index":3,"sense_key":"suit%1:18:00::","synset_id":"suit.n.03"},{"index":4,"sense_key":"suit%1:10:01::","synset_id":"courtship.n.01"},{"index":5,"sense_key":"suit%1:10:00::","synset_id":"suit.n.05"},{"index":6,"sense_key":"suit%1:06:01::","synset_id":"suit.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"suit%1:04:00::","call_ids":["semeval2013.d009.s004.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"suit%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1024,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002168,"input_uncached_usd":0.002048,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1249744580127299}
{"item_id":"semeval2013.d009.s004.t002","gold_sense_keys":["hour%1:28:00::"],"candidates":[{"index":1,"sense_key":"hour%1:28:00::","synset_id":"hour.n.01"},{"index":2,"sense_key":"hour%1:28:01::","synset_id":"hour.n.02"},{"index":3,"sense_key":"hour%1:28:02::","synset_id":"hour.n.03"},{"index":4,"sense_key":"hour%1:07:00::","synset_id":"hour.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"hour%1:28:00::","call_ids":["semeval2013.d009.s004.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"hour%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":810,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00174,"input_uncached_usd":0.00162,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0154694999800995}
{"item_id":"semeval2013.d009.s004.t003","gold_sense_keys":["argument%1:10:00::"],"candidates":[{"index":1,"sense_key":"argument%1:10:02::","synset_id":"argument.n.01"},{"index":2,"sense_key":"argument%1:10:03::","synset_id":"controversy.n.01"},{"index":3,"sense_key":"argument%1:10:00::","synset_id":"argument.n.03"},{"index":4,"sense_key":"argument%1:10:01::","synset_id":"argument.n.04"},{"index":5,"sense_key":"argument%1:10:04::","synset_id":"argument.n.05"},{"index":6,"sense_key":"argument%1:09:00::","synset_id":"argument.n.06"},{"index":7,"sense_key":"argument%1:09:01::","synset_id":"argumentation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"argument%1:10:00::","call_ids":["semeval2013.d009.s004.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"argument%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1115,"cached_input_tokens":0,"output_tokens":99,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0032199999999999998,"input_uncached_usd":0.0022299999999999998,"input_cached_usd":0.0,"output_usd":0.00099,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6786681660450995}
{"item_id":"semeval2013.d009.s004.t004","gold_sense_keys":["march%1:28:00::"],"candidates":[{"index":1,"sense_key":"march%1:28:00::","synset_id":"march.n.01"},{"index":2,"sense_key":"march%1:04:00::","synset_id":"march.n.02"},{"index":3,"sense_key":"march%1:04:02::","synset_id":"march.n.03"},{"index":4,"sense_key":"march%1:14:00::","synset_id":"march.n.04"},{"index":5,"sense_key":"march%1:15:00::","synset_id":"borderland.n.01"},{"index":6,"sense_key":"march%1:10:01::","synset_id":"marching_music.n.01"},{"index":7,"sense_key":"march%1:10:00::","synset_id":"master_of_architecture.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"march%1:28:00::","call_ids":["semeval2013.d009.s004.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"march%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":972,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002064,"input_uncached_usd":0.001944,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9868179170880467}
{"item_id":"semeval2013.d009.s004.t006","gold_sense_keys":["race%1:11:01::"],"candidates":[{"index":1,"sense_key":"race%1:11:01::","synset_id":"race.n.01"},{"index":2,"sense_key":"race%1:11:00::","synset_id":"race.n.02"},{"index":3,"sense_key":"race%1:14:00::","synset_id":"race.n.03"},{"index":4,"sense_key":"race%1:14:01::","synset_id":"subspecies.n.01"},{"index":5,"sense_key":"race%1:19:00::","synset_id":"slipstream.n.01"},{"index":6,"sense_key":"race%1:06:00::","synset_id":"raceway.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"race%1:11:01::","call_ids":["semeval2013.d009.s004.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"race%1:11:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":898,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001916,"input_uncached_usd":0.0017959999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.060807166970335}
{"item_id":"semeval2013.d009.s004.t007","gold_sense_keys":["swing%1:26:01::"],"candidates":[{"index":1,"sense_key":"swing%1:26:01::","synset_id":"swing.n.01"},{"index":2,"sense_key":"swing%1:06:00::","synset_id":"swing.n.02"},{"index":3,"sense_key":"swing%1:04:06::","synset_id":"swing.n.03"},{"index":4,"sense_key":"swing%1:04:00::","synset_id":"swing.n.04"},{"index":5,"sense_key":"swing%1:10:01::","synset_id":"swing.n.05"},{"index":6,"sense_key":"swing%1:07:00::","synset_id":"lilt.n.01"},{"index":7,"sense_key":"swing%1:04:03::","synset_id":"golf_stroke.n.01"},{"index":8,"sense_key":"swing%1:04:02::","synset_id":"baseball_swing.n.01"},{"index":9,"sense_key":"swing%1:04:04::","synset_id":"swing.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"swing%1:26:01::","call_ids":["semeval2013.d009.s004.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"swing%1:26:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1094,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0023079999999999997,"input_uncached_usd":0.002188,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.144653958035633}
{"item_id":"semeval2013.d009.s004.t009","gold_sense_keys":["court%1:14:00::"],"candidates":[{"index":1,"sense_key":"court%1:14:00::","synset_id":"court.n.01"},{"index":2,"sense_key":"court%1:06:02::","synset_id":"court.n.02"},{"index":3,"sense_key":"court%1:14:01::","synset_id":"court.n.03"},{"index":4,"sense_key":"court%1:06:01::","synset_id":"court.n.04"},{"index":5,"sense_key":"court%1:18:00::","synset_id":"court.n.05"},{"index":6,"sense_key":"court%1:14:02::","synset_id":"court.n.06"},{"index":7,"sense_key":"court%1:06:03::","synset_id":"motor_hotel.n.01"},{"index":8,"sense_key":"court%1:06:05::","synset_id":"court.n.08"},{"index":9,"sense_key":"court%1:06:04::","synset_id":"court.n.09"},{"index":10,"sense_key":"court%1:06:00::","synset_id":"court.n.10"},{"index":11,"sense_key":"court%1:04:00::","synset_id":"court.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"court%1:14:00::","call_ids":["semeval2013.d009.s004.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"court%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1212,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004504,"input_uncached_usd":0.002424,"input_cached_usd":0.0,"output_usd":0.0020800000000000003,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.530769875040278}
{"item_id":"semeval2013.d009.s004.t010","gold_sense_keys":["decision%1:09:00::"],"candidates":[{"index":1,"sense_key":"decision%1:04:00::","synset_id":"decision.n.01"},{"index":2,"sense_key":"decision%1:09:00::","synset_id":"decision.n.02"},{"index":3,"sense_key":"decision%1:11:02::","synset_id":"decision.n.03"},{"index":4,"sense_key":"decision%1:11:01::","synset_id":"decision.n.04"},{"index":5,"sense_key":"decision%1:07:00::","synset_id":"decisiveness.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"decision%1:09:00::","call_ids":["semeval2013.d009.s004.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"decision%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":930,"cached_input_tokens":0,"output_tokens":90,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00276,"input_uncached_usd":0.0018599999999999999,"input_cached_usd":0.0,"output_usd":0.0009000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6629462090786546}
{"item_id":"semeval2013.d009.s005.t001","gold_sense_keys":["court%1:14:00::"],"candidates":[{"index":1,"sense_key":"court%1:14:00::","synset_id":"court.n.01"},{"index":2,"sense_key":"court%1:06:02::","synset_id":"court.n.02"},{"index":3,"sense_key":"court%1:14:01::","synset_id":"court.n.03"},{"index":4,"sense_key":"court%1:06:01::","synset_id":"court.n.04"},{"index":5,"sense_key":"court%1:18:00::","synset_id":"court.n.05"},{"index":6,"sense_key":"court%1:14:02::","synset_id":"court.n.06"},{"index":7,"sense_key":"court%1:06:03::","synset_id":"motor_hotel.n.01"},{"index":8,"sense_key":"court%1:06:05::","synset_id":"court.n.08"},{"index":9,"sense_key":"court%1:06:04::","synset_id":"court.n.09"},{"index":10,"sense_key":"court%1:06:00::","synset_id":"court.n.10"},{"index":11,"sense_key":"court%1:04:00::","synset_id":"court.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"court%1:14:00::","call_ids":["semeval2013.d009.s005.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"court%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1284,"cached_input_tokens":0,"output_tokens":238,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004948,"input_uncached_usd":0.002568,"input_cached_usd":0.0,"output_usd":0.00238,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.4855842500692233}
{"item_id":"semeval2013.d009.s005.t002","gold_sense_keys":["law%1:10:00::"],"candidates":[{"index":1,"sense_key":"law%1:14:00::","synset_id":"law.n.01"},{"index":2,"sense_key":"law%1:10:00::","synset_id":"law.n.02"},{"index":3,"sense_key":"law%1:09:00::","synset_id":"law.n.03"},{"index":4,"sense_key":"law%1:09:01::","synset_id":"law.n.04"},{"index":5,"sense_key":"law%1:09:02::","synset_id":"jurisprudence.n.01"},{"index":6,"sense_key":"law%1:04:00::","synset_id":"law.n.06"},{"index":7,"sense_key":"law%1:14:01::","synset_id":"police.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"law%1:10:00::","call_ids":["semeval2013.d009.s005.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"law%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1127,"cached_input_tokens":0,"output_tokens":86,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003114,"input_uncached_usd":0.002254,"input_cached_usd":0.0,"output_usd":0.0008600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8073472500545904}
{"item_id":"semeval2013.d009.s006.t003","gold_sense_keys":["dinner%1:14:00::"],"candidates":[{"index":1,"sense_key":"dinner%1:13:00::","synset_id":"dinner.n.01"},{"index":2,"sense_key":"dinner%1:14:00::","synset_id":"dinner.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"dinner%1:14:00::","call_ids":["semeval2013.d009.s006.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"dinner%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":824,"cached_input_tokens":0,"output_tokens":84,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0024879999999999998,"input_uncached_usd":0.001648,"input_cached_usd":0.0,"output_usd":0.00084,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9342478340258822}
{"item_id":"semeval2013.d009.s007.t001","gold_sense_keys":["dinner%1:14:00::"],"candidates":[{"index":1,"sense_key":"dinner%1:13:00::","synset_id":"dinner.n.01"},{"index":2,"sense_key":"dinner%1:14:00::","synset_id":"dinner.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"dinner%1:14:00::","call_ids":["semeval2013.d009.s007.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"dinner%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":767,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0022040000000000002,"input_uncached_usd":0.001534,"input_cached_usd":0.0,"output_usd":0.00067,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.54994745797012}
{"item_id":"semeval2013.d009.s007.t002","gold_sense_keys":["law%1:10:00::"],"candidates":[{"index":1,"sense_key":"law%1:14:00::","synset_id":"law.n.01"},{"index":2,"sense_key":"law%1:10:00::","synset_id":"law.n.02"},{"index":3,"sense_key":"law%1:09:00::","synset_id":"law.n.03"},{"index":4,"sense_key":"law%1:09:01::","synset_id":"law.n.04"},{"index":5,"sense_key":"law%1:09:02::","synset_id":"jurisprudence.n.01"},{"index":6,"sense_key":"law%1:04:00::","synset_id":"law.n.06"},{"index":7,"sense_key":"law%1:14:01::","synset_id":"police.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"law%1:10:00::","call_ids":["semeval2013.d009.s007.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"law%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1093,"cached_input_tokens":0,"output_tokens":69,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002876,"input_uncached_usd":0.002186,"input_cached_usd":0.0,"output_usd":0.0006900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4475284580839798}
{"item_id":"semeval2013.d009.s008.t000","gold_sense_keys":["sponsor%1:18:00::"],"candidates":[{"index":1,"sense_key":"sponsor%1:18:00::","synset_id":"patron.n.03"},{"index":2,"sense_key":"sponsor%1:18:01::","synset_id":"presenter.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"sponsor%1:18:00::","call_ids":["semeval2013.d009.s008.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"sponsor%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":735,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0022,"input_uncached_usd":0.00147,"input_cached_usd":0.0,"output_usd":0.0007300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9892751249717548}
{"item_id":"semeval2013.d009.s008.t001","gold_sense_keys":["giant%1:14:00::"],"candidates":[{"index":1,"sense_key":"giant%1:05:00::","synset_id":"giant.n.01"},{"index":2,"sense_key":"giant%1:18:01::","synset_id":"colossus.n.02"},{"index":3,"sense_key":"giant%1:14:00::","synset_id":"giant.n.03"},{"index":4,"sense_key":"giant%1:18:03::","synset_id":"giant.n.04"},{"index":5,"sense_key":"giant%1:18:00::","synset_id":"giant.n.05"},{"index":6,"sense_key":"giant%1:18:02::","synset_id":"giant.n.06"},{"index":7,"sense_key":"giant%1:17:00::","synset_id":"giant_star.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"giant%1:14:00::","call_ids":["semeval2013.d009.s008.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"giant%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":975,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00207,"input_uncached_usd":0.00195,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2227565000066534}
{"item_id":"semeval2013.d009.s008.t002","gold_sense_keys":["stake%1:21:02::"],"candidates":[{"index":1,"sense_key":"stake%1:21:02::","synset_id":"interest.n.05"},{"index":2,"sense_key":"stake%1:10:00::","synset_id":"post.n.09"},{"index":3,"sense_key":"stake%1:06:01::","synset_id":"stake.n.03"},{"index":4,"sense_key":"stake%1:21:00::","synset_id":"stake.n.04"},{"index":5,"sense_key":"stake%1:06:00::","synset_id":"stake.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"stake%1:21:02::","call_ids":["semeval2013.d009.s008.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"stake%1:21:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":922,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001964,"input_uncached_usd":0.001844,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0497784580802545}
{"item_id":"semeval2013.d009.s009.t000","gold_sense_keys":["dinner%1:14:00::"],"candidates":[{"index":1,"sense_key":"dinner%1:13:00::","synset_id":"dinner.n.01"},{"index":2,"sense_key":"dinner%1:14:00::","synset_id":"dinner.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"dinner%1:14:00::","call_ids":["semeval2013.d009.s009.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"dinner%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":737,"cached_input_tokens":0,"output_tokens":98,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002454,"input_uncached_usd":0.0014739999999999998,"input_cached_usd":0.0,"output_usd":0.0009800000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":14.411720667034388}
{"item_id":"semeval2013.d009.s009.t001","gold_sense_keys":["washington%1:15:01::"],"candidates":[{"index":1,"sense_key":"washington%1:15:01::","synset_id":"washington.n.01"},{"index":2,"sense_key":"washington%1:15:00::","synset_id":"washington.n.02"},{"index":3,"sense_key":"washington%1:14:00::","synset_id":"capital.n.06"},{"index":4,"sense_key":"washington%1:18:00::","synset_id":"washington.n.04"},{"index":5,"sense_key":"washington%1:18:01::","synset_id":"washington.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"washington%1:15:01::","call_ids":["semeval2013.d009.s009.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"washington%1:15:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":952,"cached_input_tokens":0,"output_tokens":47,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002374,"input_uncached_usd":0.0019039999999999999,"input_cached_usd":0.0,"output_usd":0.00047000000000000004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3847653340781108}
{"item_id":"semeval2013.d009.s009.t003","gold_sense_keys":["hour%1:28:00::"],"candidates":[{"index":1,"sense_key":"hour%1:28:00::","synset_id":"hour.n.01"},{"index":2,"sense_key":"hour%1:28:01::","synset_id":"hour.n.02"},{"index":3,"sense_key":"hour%1:28:02::","synset_id":"hour.n.03"},{"index":4,"sense_key":"hour%1:07:00::","synset_id":"hour.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"hour%1:28:00::","call_ids":["semeval2013.d009.s009.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"hour%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":819,"cached_input_tokens":0,"output_tokens":51,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0021479999999999997,"input_uncached_usd":0.001638,"input_cached_usd":0.0,"output_usd":0.00051,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3318916669813916}
{"item_id":"semeval2013.d009.s009.t004","gold_sense_keys":["court%1:14:00::"],"candidates":[{"index":1,"sense_key":"court%1:14:00::","synset_id":"court.n.01"},{"index":2,"sense_key":"court%1:06:02::","synset_id":"court.n.02"},{"index":3,"sense_key":"court%1:14:01::","synset_id":"court.n.03"},{"index":4,"sense_key":"court%1:06:01::","synset_id":"court.n.04"},{"index":5,"sense_key":"court%1:18:00::","synset_id":"court.n.05"},{"index":6,"sense_key":"court%1:14:02::","synset_id":"court.n.06"},{"index":7,"sense_key":"court%1:06:03::","synset_id":"motor_hotel.n.01"},{"index":8,"sense_key":"court%1:06:05::","synset_id":"court.n.08"},{"index":9,"sense_key":"court%1:06:04::","synset_id":"court.n.09"},{"index":10,"sense_key":"court%1:06:00::","synset_id":"court.n.10"},{"index":11,"sense_key":"court%1:04:00::","synset_id":"court.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"court%1:14:00::","call_ids":["semeval2013.d009.s009.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"court%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1221,"cached_input_tokens":0,"output_tokens":110,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003542,"input_uncached_usd":0.0024419999999999997,"input_cached_usd":0.0,"output_usd":0.0011,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6037328749662265}
{"item_id":"semeval2013.d009.s009.t005","gold_sense_keys":["conference%1:10:00::","conference%1:14:00::"],"candidates":[{"index":1,"sense_key":"conference%1:14:00::","synset_id":"conference.n.01"},{"index":2,"sense_key":"conference%1:14:01::","synset_id":"league.n.01"},{"index":3,"sense_key":"conference%1:10:00::","synset_id":"conference.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"conference%1:14:00::","call_ids":["semeval2013.d009.s009.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"conference%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":755,"cached_input_tokens":0,"output_tokens":141,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00292,"input_uncached_usd":0.0015099999999999998,"input_cached_usd":0.0,"output_usd":0.00141,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0868961670203134}
{"item_id":"semeval2013.d009.s009.t006","gold_sense_keys":["case%1:04:00::"],"candidates":[{"index":1,"sense_key":"case%1:11:00::","synset_id":"case.n.01"},{"index":2,"sense_key":"case%1:26:00::","synset_id":"event.n.02"},{"index":3,"sense_key":"case%1:04:00::","synset_id":"lawsuit.n.01"},{"index":4,"sense_key":"case%1:09:00::","synset_id":"case.n.04"},{"index":5,"sense_key":"case%1:06:00::","synset_id":"case.n.05"},{"index":6,"sense_key":"case%1:18:03::","synset_id":"case.n.06"},{"index":7,"sense_key":"case%1:18:02::","synset_id":"subject.n.06"},{"index":8,"sense_key":"case%1:10:02::","synset_id":"case.n.08"},{"index":9,"sense_key":"case%1:10:01::","synset_id":"case.n.09"},{"index":10,"sense_key":"case%1:23:00::","synset_id":"case.n.10"},{"index":11,"sense_key":"case%1:10:00::","synset_id":"case.n.11"},{"index":12,"sense_key":"case%1:26:02::","synset_id":"case.n.12"},{"index":13,"sense_key":"case%1:18:04::","synset_id":"character.n.05"},{"index":14,"sense_key":"case%1:10:03::","synset_id":"font.n.01"},{"index":15,"sense_key":"case%1:08:00::","synset_id":"sheath.n.02"},{"index":16,"sense_key":"case%1:06:06::","synset_id":"shell.n.08"},{"index":17,"sense_key":"case%1:06:07::","synset_id":"casing.n.03"},{"index":18,"sense_key":"case%1:06:03::","synset_id":"case.n.18"},{"index":19,"sense_key":"case%1:06:02::","synset_id":"case.n.19"},{"index":20,"sense_key":"case%1:06:01::","synset_id":"case.n.20"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"case%1:04:00::","call_ids":["semeval2013.d009.s009.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"case%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1855,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0038299999999999996,"input_uncached_usd":0.0037099999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9977426669793203}
{"item_id":"semeval2013.d009.s010.t000","gold_sense_keys":["attendance%1:04:00::"],"candidates":[{"index":1,"sense_key":"attendance%1:04:00::","synset_id":"attendance.n.01"},{"index":2,"sense_key":"attendance%1:28:00::","synset_id":"attendance.n.02"},{"index":3,"sense_key":"attendance%1:14:00::","synset_id":"attendance.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"attendance%1:04:00::","call_ids":["semeval2013.d009.s010.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"attendance%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":695,"cached_input_tokens":0,"output_tokens":94,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00233,"input_uncached_usd":0.00139,"input_cached_usd":0.0,"output_usd":0.0009400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6482410000171512}
{"item_id":"semeval2013.d009.s010.t001","gold_sense_keys":["senate%1:14:01::"],"candidates":[{"index":1,"sense_key":"senate%1:14:00::","synset_id":"senate.n.01"},{"index":2,"sense_key":"senate%1:14:01::","synset_id":"united_states_senate.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"senate%1:14:01::","call_ids":["semeval2013.d009.s010.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"senate%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":635,"cached_input_tokens":0,"output_tokens":69,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00196,"input_uncached_usd":0.0012699999999999999,"input_cached_usd":0.0,"output_usd":0.0006900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":14.596816333010793}
{"item_id":"semeval2013.d009.s010.t002","gold_sense_keys":["opponent%1:18:00::"],"candidates":[{"index":1,"sense_key":"opponent%1:18:02::","synset_id":"opposition.n.04"},{"index":2,"sense_key":"opponent%1:18:00::","synset_id":"adversary.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"opponent%1:18:00::","call_ids":["semeval2013.d009.s010.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"opponent%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":647,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0014140000000000001,"input_uncached_usd":0.001294,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1347243329510093}
{"item_id":"semeval2013.d009.s011.t000","gold_sense_keys":["guest%1:18:00::"],"candidates":[{"index":1,"sense_key":"guest%1:18:00::","synset_id":"guest.n.01"},{"index":2,"sense_key":"guest%1:18:02::","synset_id":"guest.n.02"},{"index":3,"sense_key":"guest%1:18:01::","synset_id":"guest.n.03"},{"index":4,"sense_key":"guest%1:06:00::","synset_id":"node.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"guest%1:18:00::","call_ids":["semeval2013.d009.s011.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"guest%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":721,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001562,"input_uncached_usd":0.001442,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1508237500675023}
{"item_id":"semeval2013.d009.s011.t001","gold_sense_keys":["dinner%1:14:00::"],"candidates":[{"index":1,"sense_key":"dinner%1:13:00::","synset_id":"dinner.n.01"},{"index":2,"sense_key":"dinner%1:14:00::","synset_id":"dinner.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"dinner%1:14:00::","call_ids":["semeval2013.d009.s011.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"dinner%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":647,"cached_input_tokens":0,"output_tokens":79,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002084,"input_uncached_usd":0.001294,"input_cached_usd":0.0,"output_usd":0.00079,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4790094170020893}
{"item_id":"semeval2013.d009.s012.t000","gold_sense_keys":["justice%1:18:00::"],"candidates":[{"index":1,"sense_key":"justice%1:07:00::","synset_id":"justice.n.01"},{"index":2,"sense_key":"justice%1:04:00::","synset_id":"justice.n.02"},{"index":3,"sense_key":"justice%1:18:00::","synset_id":"judge.n.01"},{"index":4,"sense_key":"justice%1:14:00::","synset_id":"department_of_justice.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"justice%1:18:00::","call_ids":["semeval2013.d009.s012.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"justice%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":684,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001488,"input_uncached_usd":0.0013679999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.205930416006595}
{"item_id":"semeval2013.d009.s012.t001","gold_sense_keys":["event%1:03:00::"],"candidates":[{"index":1,"sense_key":"event%1:03:00::","synset_id":"event.n.01"},{"index":2,"sense_key":"event%1:26:00::","synset_id":"event.n.02"},{"index":3,"sense_key":"event%1:19:00::","synset_id":"event.n.03"},{"index":4,"sense_key":"event%1:19:01::","synset_id":"consequence.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"event%1:03:00::","call_ids":["semeval2013.d009.s012.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"event%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":715,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00155,"input_uncached_usd":0.0014299999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2046290419530123}
{"item_id":"semeval2013.d009.s012.t002","gold_sense_keys":["years%1:28:01::"],"candidates":[{"index":1,"sense_key":"years%1:28:00::","synset_id":"old_age.n.01"},{"index":2,"sense_key":"years%1:28:01::","synset_id":"long_time.n.01"},{"index":3,"sense_key":"years%1:28:02::","synset_id":"days.n.01"},{"index":4,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":5,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":6,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":7,"sense_key":"class%1:14:04::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"years%1:28:01::","call_ids":["semeval2013.d009.s012.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"years%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":915,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0023799999999999997,"input_uncached_usd":0.0018299999999999998,"input_cached_usd":0.0,"output_usd":0.00055,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":14.495480666053481}
{"item_id":"semeval2013.d009.s013.t002","gold_sense_keys":["rule%1:10:00::"],"candidates":[{"index":1,"sense_key":"rule%1:09:00::","synset_id":"rule.n.01"},{"index":2,"sense_key":"rule%1:09:01::","synset_id":"convention.n.02"},{"index":3,"sense_key":"rule%1:10:00::","synset_id":"rule.n.03"},{"index":4,"sense_key":"rule%1:10:01::","synset_id":"rule.n.04"},{"index":5,"sense_key":"rule%1:09:03::","synset_id":"principle.n.01"},{"index":6,"sense_key":"rule%1:28:00::","synset_id":"rule.n.06"},{"index":7,"sense_key":"rule%1:26:00::","synset_id":"dominion.n.01"},{"index":8,"sense_key":"rule%1:10:02::","synset_id":"rule.n.08"},{"index":9,"sense_key":"rule%1:10:03::","synset_id":"rule.n.09"},{"index":10,"sense_key":"rule%1:09:02::","synset_id":"principle.n.04"},{"index":11,"sense_key":"rule%1:09:04::","synset_id":"rule.n.11"},{"index":12,"sense_key":"rule%1:06:00::","synset_id":"rule.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"rule%1:10:00::","call_ids":["semeval2013.d009.s013.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"rule%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1259,"cached_input_tokens":0,"output_tokens":106,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003578,"input_uncached_usd":0.002518,"input_cached_usd":0.0,"output_usd":0.0010600000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":15.57302037498448}
{"item_id":"semeval2013.d009.s014.t001","gold_sense_keys":["justice%1:18:00::"],"candidates":[{"index":1,"sense_key":"justice%1:07:00::","synset_id":"justice.n.01"},{"index":2,"sense_key":"justice%1:04:00::","synset_id":"justice.n.02"},{"index":3,"sense_key":"justice%1:18:00::","synset_id":"judge.n.01"},{"index":4,"sense_key":"justice%1:14:00::","synset_id":"department_of_justice.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"justice%1:18:00::","call_ids":["semeval2013.d009.s014.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"justice%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":698,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001516,"input_uncached_usd":0.0013959999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":9.250193874933757}
{"item_id":"semeval2013.d009.s014.t003","gold_sense_keys":["action%1:04:02::"],"candidates":[{"index":1,"sense_key":"action%1:04:02::","synset_id":"action.n.01"},{"index":2,"sense_key":"action%1:26:00::","synset_id":"action.n.02"},{"index":3,"sense_key":"action%1:04:03::","synset_id":"military_action.n.01"},{"index":4,"sense_key":"action%1:22:00::","synset_id":"natural_process.n.01"},{"index":5,"sense_key":"action%1:10:00::","synset_id":"action.n.05"},{"index":6,"sense_key":"action%1:07:00::","synset_id":"action.n.06"},{"index":7,"sense_key":"action%1:06:00::","synset_id":"action.n.07"},{"index":8,"sense_key":"action%1:04:01::","synset_id":"legal_action.n.01"},{"index":9,"sense_key":"action%1:04:04::","synset_id":"action.n.09"},{"index":10,"sense_key":"action%1:04:00::","synset_id":"action.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"action%1:04:02::","call_ids":["semeval2013.d009.s014.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"action%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1096,"cached_input_tokens":0,"output_tokens":70,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002892,"input_uncached_usd":0.002192,"input_cached_usd":0.0,"output_usd":0.0007000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8725630000699311}
{"item_id":"semeval2013.d009.s014.t004","gold_sense_keys":["justice%1:18:00::"],"candidates":[{"index":1,"sense_key":"justice%1:07:00::","synset_id":"justice.n.01"},{"index":2,"sense_key":"justice%1:04:00::","synset_id":"justice.n.02"},{"index":3,"sense_key":"justice%1:18:00::","synset_id":"judge.n.01"},{"index":4,"sense_key":"justice%1:14:00::","synset_id":"department_of_justice.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"justice%1:18:00::","call_ids":["semeval2013.d009.s014.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"justice%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":698,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001516,"input_uncached_usd":0.0013959999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1689206660958007}
{"item_id":"semeval2013.d009.s015.t000","gold_sense_keys":["code%1:10:01::"],"candidates":[{"index":1,"sense_key":"code%1:10:01::","synset_id":"code.n.01"},{"index":2,"sense_key":"code%1:10:00::","synset_id":"code.n.02"},{"index":3,"sense_key":"code%1:10:02::","synset_id":"code.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"code%1:10:01::","call_ids":["semeval2013.d009.s015.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"code%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":667,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001454,"input_uncached_usd":0.001334,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":13.856674375012517}
{"item_id":"semeval2013.d009.s015.t002","gold_sense_keys":["judge%1:18:00::"],"candidates":[{"index":1,"sense_key":"judge%1:18:00::","synset_id":"judge.n.01"},{"index":2,"sense_key":"judge%1:18:01::","synset_id":"evaluator.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"judge%1:18:00::","call_ids":["semeval2013.d009.s015.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"judge%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":614,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001348,"input_uncached_usd":0.001228,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8370035830885172}
{"item_id":"semeval2013.d009.s015.t003","gold_sense_keys":["event%1:03:00::"],"candidates":[{"index":1,"sense_key":"event%1:03:00::","synset_id":"event.n.01"},{"index":2,"sense_key":"event%1:26:00::","synset_id":"event.n.02"},{"index":3,"sense_key":"event%1:19:00::","synset_id":"event.n.03"},{"index":4,"sense_key":"event%1:19:01::","synset_id":"consequence.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"event%1:03:00::","call_ids":["semeval2013.d009.s015.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"event%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":770,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00166,"input_uncached_usd":0.00154,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.068953083944507}
{"item_id":"semeval2013.d009.s015.t004","gold_sense_keys":["organization%1:14:00::","organization%1:14:01::"],"candidates":[{"index":1,"sense_key":"organization%1:14:00::","synset_id":"organization.n.01"},{"index":2,"sense_key":"organization%1:09:00::","synset_id":"arrangement.n.03"},{"index":3,"sense_key":"organization%1:14:01::","synset_id":"administration.n.02"},{"index":4,"sense_key":"organization%1:04:00::","synset_id":"organization.n.04"},{"index":5,"sense_key":"organization%1:07:00::","synset_id":"organization.n.05"},{"index":6,"sense_key":"organization%1:04:01::","synset_id":"organization.n.06"},{"index":7,"sense_key":"organization%1:04:02::","synset_id":"constitution.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"organization%1:14:00::","call_ids":["semeval2013.d009.s015.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"organization%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1056,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0022319999999999996,"input_uncached_usd":0.0021119999999999997,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0443665410857648}
{"item_id":"semeval2013.d009.s015.t005","gold_sense_keys":["judge%1:18:00::"],"candidates":[{"index":1,"sense_key":"judge%1:18:00::","synset_id":"judge.n.01"},{"index":2,"sense_key":"judge%1:18:01::","synset_id":"evaluator.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"judge%1:18:00::","call_ids":["semeval2013.d009.s015.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"judge%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":614,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001348,"input_uncached_usd":0.001228,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4280647920677438}
{"item_id":"semeval2013.d009.s015.t006","gold_sense_keys":["speaker%1:18:00::"],"candidates":[{"index":1,"sense_key":"speaker%1:18:00::","synset_id":"speaker.n.01"},{"index":2,"sense_key":"speaker%1:06:00::","synset_id":"loudspeaker.n.01"},{"index":3,"sense_key":"speaker%1:18:01::","synset_id":"speaker.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"speaker%1:18:00::","call_ids":["semeval2013.d009.s015.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"speaker%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":775,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00167,"input_uncached_usd":0.00155,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0164531250484288}
{"item_id":"semeval2013.d009.s015.t008","gold_sense_keys":["program%1:10:00::"],"candidates":[{"index":1,"sense_key":"program%1:09:00::","synset_id":"plan.n.01"},{"index":2,"sense_key":"program%1:09:01::","synset_id":"program.n.02"},{"index":3,"sense_key":"program%1:10:01::","synset_id":"broadcast.n.02"},{"index":4,"sense_key":"program%1:10:04::","synset_id":"platform.n.02"},{"index":5,"sense_key":"program%1:10:00::","synset_id":"program.n.05"},{"index":6,"sense_key":"program%1:10:05::","synset_id":"course_of_study.n.01"},{"index":7,"sense_key":"program%1:10:02::","synset_id":"program.n.07"},{"index":8,"sense_key":"program%1:04:00::","synset_id":"program.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"program%1:10:00::","call_ids":["semeval2013.d009.s015.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"program%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1069,"cached_input_tokens":0,"output_tokens":125,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0033879999999999995,"input_uncached_usd":0.0021379999999999997,"input_cached_usd":0.0,"output_usd":0.00125,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.943148041027598}
{"item_id":"semeval2013.d009.s015.t009","gold_sense_keys":["event%1:03:00::"],"candidates":[{"index":1,"sense_key":"event%1:03:00::","synset_id":"event.n.01"},{"index":2,"sense_key":"event%1:26:00::","synset_id":"event.n.02"},{"index":3,"sense_key":"event%1:19:00::","synset_id":"event.n.03"},{"index":4,"sense_key":"event%1:19:01::","synset_id":"consequence.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"event%1:03:00::","call_ids":["semeval2013.d009.s015.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"event%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":770,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00166,"input_uncached_usd":0.00154,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7125584159512073}
{"item_id":"semeval2013.d009.s016.t000","gold_sense_keys":["proximity%1:07:00::"],"candidates":[{"index":1,"sense_key":"proximity%1:07:00::","synset_id":"proximity.n.01"},{"index":2,"sense_key":"proximity%1:15:00::","synset_id":"proximity.n.02"},{"index":3,"sense_key":"proximity%1:09:00::","synset_id":"proximity.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"proximity%1:07:00::","call_ids":["semeval2013.d009.s016.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"proximity%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":693,"cached_input_tokens":0,"output_tokens":79,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002176,"input_uncached_usd":0.0013859999999999999,"input_cached_usd":0.0,"output_usd":0.00079,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2499018750386313}
{"item_id":"semeval2013.d009.s016.t002","gold_sense_keys":["case%1:04:00::"],"candidates":[{"index":1,"sense_key":"case%1:11:00::","synset_id":"case.n.01"},{"index":2,"sense_key":"case%1:26:00::","synset_id":"event.n.02"},{"index":3,"sense_key":"case%1:04:00::","synset_id":"lawsuit.n.01"},{"index":4,"sense_key":"case%1:09:00::","synset_id":"case.n.04"},{"index":5,"sense_key":"case%1:06:00::","synset_id":"case.n.05"},{"index":6,"sense_key":"case%1:18:03::","synset_id":"case.n.06"},{"index":7,"sense_key":"case%1:18:02::","synset_id":"subject.n.06"},{"index":8,"sense_key":"case%1:10:02::","synset_id":"case.n.08"},{"index":9,"sense_key":"case%1:10:01::","synset_id":"case.n.09"},{"index":10,"sense_key":"case%1:23:00::","synset_id":"case.n.10"},{"index":11,"sense_key":"case%1:10:00::","synset_id":"case.n.11"},{"index":12,"sense_key":"case%1:26:02::","synset_id":"case.n.12"},{"index":13,"sense_key":"case%1:18:04::","synset_id":"character.n.05"},{"index":14,"sense_key":"case%1:10:03::","synset_id":"font.n.01"},{"index":15,"sense_key":"case%1:08:00::","synset_id":"sheath.n.02"},{"index":16,"sense_key":"case%1:06:06::","synset_id":"shell.n.08"},{"index":17,"sense_key":"case%1:06:07::","synset_id":"casing.n.03"},{"index":18,"sense_key":"case%1:06:03::","synset_id":"case.n.18"},{"index":19,"sense_key":"case%1:06:02::","synset_id":"case.n.19"},{"index":20,"sense_key":"case%1:06:01::","synset_id":"case.n.20"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"case%1:04:00::","call_ids":["semeval2013.d009.s016.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"case%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1788,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0041259999999999995,"input_uncached_usd":0.0035759999999999998,"input_cached_usd":0.0,"output_usd":0.00055,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3411948749562725}
{"item_id":"semeval2013.d009.s016.t003","gold_sense_keys":["company%1:14:01::"],"candidates":[{"index":1,"sense_key":"company%1:14:01::","synset_id":"company.n.01"},{"index":2,"sense_key":"company%1:14:03::","synset_id":"company.n.02"},{"index":3,"sense_key":"company%1:26:00::","synset_id":"company.n.03"},{"index":4,"sense_key":"company%1:14:02::","synset_id":"company.n.04"},{"index":5,"sense_key":"company%1:18:00::","synset_id":"caller.n.01"},{"index":6,"sense_key":"company%1:14:00::","synset_id":"company.n.06"},{"index":7,"sense_key":"company%1:14:06::","synset_id":"party.n.03"},{"index":8,"sense_key":"company%1:14:04::","synset_id":"ship's_company.n.01"},{"index":9,"sense_key":"company%1:14:05::","synset_id":"company.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"company%1:14:01::","call_ids":["semeval2013.d009.s016.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"company%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1031,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002182,"input_uncached_usd":0.002062,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1401336659910157}
{"item_id":"semeval2013.d009.s016.t004","gold_sense_keys":["interest%1:21:03::"],"candidates":[{"index":1,"sense_key":"interest%1:09:00::","synset_id":"interest.n.01"},{"index":2,"sense_key":"interest%1:07:01::","synset_id":"sake.n.01"},{"index":3,"sense_key":"interest%1:07:02::","synset_id":"interest.n.03"},{"index":4,"sense_key":"interest%1:21:00::","synset_id":"interest.n.04"},{"index":5,"sense_key":"interest%1:21:03::","synset_id":"interest.n.05"},{"index":6,"sense_key":"interest%1:14:00::","synset_id":"interest.n.06"},{"index":7,"sense_key":"interest%1:04:01::","synset_id":"pastime.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"interest%1:21:03::","call_ids":["semeval2013.d009.s016.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"interest%1:21:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1028,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002176,"input_uncached_usd":0.002056,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.407259542029351}
{"item_id":"semeval2013.d009.s017.t000","gold_sense_keys":["breach%1:04:01::"],"candidates":[{"index":1,"sense_key":"breach%1:04:01::","synset_id":"breach.n.01"},{"index":2,"sense_key":"breach%1:17:00::","synset_id":"breach.n.02"},{"index":3,"sense_key":"breach%1:11:00::","synset_id":"rupture.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"breach%1:04:01::","call_ids":["semeval2013.d009.s017.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"breach%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":703,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001996,"input_uncached_usd":0.001406,"input_cached_usd":0.0,"output_usd":0.00059,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8907844589557499}
{"item_id":"semeval2013.d009.s017.t002","gold_sense_keys":["indifference%1:07:01::"],"candidates":[{"index":1,"sense_key":"indifference%1:12:00::","synset_id":"indifference.n.01"},{"index":2,"sense_key":"indifference%1:12:01::","synset_id":"emotionlessness.n.01"},{"index":3,"sense_key":"indifference%1:07:00::","synset_id":"apathy.n.02"},{"index":4,"sense_key":"indifference%1:07:01::","synset_id":"nonchalance.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"indifference%1:07:01::","call_ids":["semeval2013.d009.s017.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"indifference%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":796,"cached_input_tokens":0,"output_tokens":172,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003312,"input_uncached_usd":0.0015919999999999999,"input_cached_usd":0.0,"output_usd":0.0017200000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.069826832972467}
{"item_id":"semeval2013.d009.s017.t003","gold_sense_keys":["code%1:10:01::"],"candidates":[{"index":1,"sense_key":"code%1:10:01::","synset_id":"code.n.01"},{"index":2,"sense_key":"code%1:10:00::","synset_id":"code.n.02"},{"index":3,"sense_key":"code%1:10:02::","synset_id":"code.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"code%1:10:01::","call_ids":["semeval2013.d009.s017.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"code%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":703,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001526,"input_uncached_usd":0.001406,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":9.728271708940156}
{"item_id":"semeval2013.d009.s017.t004","gold_sense_keys":["claim%1:10:02::"],"candidates":[{"index":1,"sense_key":"claim%1:10:00::","synset_id":"claim.n.01"},{"index":2,"sense_key":"claim%1:10:02::","synset_id":"claim.n.02"},{"index":3,"sense_key":"claim%1:04:00::","synset_id":"claim.n.03"},{"index":4,"sense_key":"claim%1:07:02::","synset_id":"claim.n.04"},{"index":5,"sense_key":"claim%1:07:01::","synset_id":"title.n.07"},{"index":6,"sense_key":"claim%1:10:01::","synset_id":"call.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"claim%1:10:02::","call_ids":["semeval2013.d009.s017.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"claim%1:10:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":878,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001876,"input_uncached_usd":0.001756,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.7840698339277878}
{"item_id":"semeval2013.d009.s017.t005","gold_sense_keys":["justice%1:18:00::"],"candidates":[{"index":1,"sense_key":"justice%1:07:00::","synset_id":"justice.n.01"},{"index":2,"sense_key":"justice%1:04:00::","synset_id":"justice.n.02"},{"index":3,"sense_key":"justice%1:18:00::","synset_id":"judge.n.01"},{"index":4,"sense_key":"justice%1:14:00::","synset_id":"department_of_justice.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"justice%1:18:00::","call_ids":["semeval2013.d009.s017.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"justice%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":776,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001672,"input_uncached_usd":0.001552,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.311929333023727}
{"item_id":"semeval2013.d009.s017.t006","gold_sense_keys":["court%1:14:00::"],"candidates":[{"index":1,"sense_key":"court%1:14:00::","synset_id":"court.n.01"},{"index":2,"sense_key":"court%1:06:02::","synset_id":"court.n.02"},{"index":3,"sense_key":"court%1:14:01::","synset_id":"court.n.03"},{"index":4,"sense_key":"court%1:06:01::","synset_id":"court.n.04"},{"index":5,"sense_key":"court%1:18:00::","synset_id":"court.n.05"},{"index":6,"sense_key":"court%1:14:02::","synset_id":"court.n.06"},{"index":7,"sense_key":"court%1:06:03::","synset_id":"motor_hotel.n.01"},{"index":8,"sense_key":"court%1:06:05::","synset_id":"court.n.08"},{"index":9,"sense_key":"court%1:06:04::","synset_id":"court.n.09"},{"index":10,"sense_key":"court%1:06:00::","synset_id":"court.n.10"},{"index":11,"sense_key":"court%1:04:00::","synset_id":"court.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"court%1:14:00::","call_ids":["semeval2013.d009.s017.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"court%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1168,"cached_input_tokens":0,"output_tokens":136,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003696,"input_uncached_usd":0.002336,"input_cached_usd":0.0,"output_usd":0.00136,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0197881249478087}
{"item_id":"semeval2013.d009.s017.t007","gold_sense_keys":["rule%1:10:00::"],"candidates":[{"index":1,"sense_key":"rule%1:09:00::","synset_id":"rule.n.01"},{"index":2,"sense_key":"rule%1:09:01::","synset_id":"convention.n.02"},{"index":3,"sense_key":"rule%1:10:00::","synset_id":"rule.n.03"},{"index":4,"sense_key":"rule%1:10:01::","synset_id":"rule.n.04"},{"index":5,"sense_key":"rule%1:09:03::","synset_id":"principle.n.01"},{"index":6,"sense_key":"rule%1:28:00::","synset_id":"rule.n.06"},{"index":7,"sense_key":"rule%1:26:00::","synset_id":"dominion.n.01"},{"index":8,"sense_key":"rule%1:10:02::","synset_id":"rule.n.08"},{"index":9,"sense_key":"rule%1:10:03::","synset_id":"rule.n.09"},{"index":10,"sense_key":"rule%1:09:02::","synset_id":"principle.n.04"},{"index":11,"sense_key":"rule%1:09:04::","synset_id":"rule.n.11"},{"index":12,"sense_key":"rule%1:06:00::","synset_id":"rule.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"rule%1:10:00::","call_ids":["semeval2013.d009.s017.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"rule%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1366,"cached_input_tokens":0,"output_tokens":101,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003742,"input_uncached_usd":0.002732,"input_cached_usd":0.0,"output_usd":0.00101,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.2453403749968857}
{"item_id":"semeval2013.d009.s017.t008","gold_sense_keys":["president%1:18:01::"],"candidates":[{"index":1,"sense_key":"president%1:18:01::","synset_id":"president.n.01"},{"index":2,"sense_key":"president%1:18:04::","synset_id":"president_of_the_united_states.n.01"},{"index":3,"sense_key":"president%1:18:00::","synset_id":"president.n.03"},{"index":4,"sense_key":"president%1:18:03::","synset_id":"president.n.04"},{"index":5,"sense_key":"president%1:18:02::","synset_id":"president.n.05"},{"index":6,"sense_key":"president%1:04:04::","synset_id":"president_of_the_united_states.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"president%1:18:03::","call_ids":["semeval2013.d009.s017.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"president%1:18:03::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":862,"cached_input_tokens":0,"output_tokens":112,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002844,"input_uncached_usd":0.0017239999999999998,"input_cached_usd":0.0,"output_usd":0.0011200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6113721249857917}
{"item_id":"semeval2013.d009.s018.t000","gold_sense_keys":["justice%1:18:00::"],"candidates":[{"index":1,"sense_key":"justice%1:07:00::","synset_id":"justice.n.01"},{"index":2,"sense_key":"justice%1:04:00::","synset_id":"justice.n.02"},{"index":3,"sense_key":"justice%1:18:00::","synset_id":"judge.n.01"},{"index":4,"sense_key":"justice%1:14:00::","synset_id":"department_of_justice.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"justice%1:18:00::","call_ids":["semeval2013.d009.s018.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"justice%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":789,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001698,"input_uncached_usd":0.001578,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":6.735976208001375}
{"item_id":"semeval2013.d009.s018.t001","gold_sense_keys":["fundraiser%1:11:00::"],"candidates":[{"index":1,"sense_key":"fundraiser%1:18:00::","synset_id":"fundraiser.n.01"},{"index":2,"sense_key":"fundraiser%1:11:00::","synset_id":"fundraiser.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"fundraiser%1:11:00::","call_ids":["semeval2013.d009.s018.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"fundraiser%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":646,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001922,"input_uncached_usd":0.001292,"input_cached_usd":0.0,"output_usd":0.00063,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6828376250341535}
{"item_id":"semeval2013.d009.s018.t003","gold_sense_keys":["case%1:04:00::"],"candidates":[{"index":1,"sense_key":"case%1:11:00::","synset_id":"case.n.01"},{"index":2,"sense_key":"case%1:26:00::","synset_id":"event.n.02"},{"index":3,"sense_key":"case%1:04:00::","synset_id":"lawsuit.n.01"},{"index":4,"sense_key":"case%1:09:00::","synset_id":"case.n.04"},{"index":5,"sense_key":"case%1:06:00::","synset_id":"case.n.05"},{"index":6,"sense_key":"case%1:18:03::","synset_id":"case.n.06"},{"index":7,"sense_key":"case%1:18:02::","synset_id":"subject.n.06"},{"index":8,"sense_key":"case%1:10:02::","synset_id":"case.n.08"},{"index":9,"sense_key":"case%1:10:01::","synset_id":"case.n.09"},{"index":10,"sense_key":"case%1:23:00::","synset_id":"case.n.10"},{"index":11,"sense_key":"case%1:10:00::","synset_id":"case.n.11"},{"index":12,"sense_key":"case%1:26:02::","synset_id":"case.n.12"},{"index":13,"sense_key":"case%1:18:04::","synset_id":"character.n.05"},{"index":14,"sense_key":"case%1:10:03::","synset_id":"font.n.01"},{"index":15,"sense_key":"case%1:08:00::","synset_id":"sheath.n.02"},{"index":16,"sense_key":"case%1:06:06::","synset_id":"shell.n.08"},{"index":17,"sense_key":"case%1:06:07::","synset_id":"casing.n.03"},{"index":18,"sense_key":"case%1:06:03::","synset_id":"case.n.18"},{"index":19,"sense_key":"case%1:06:02::","synset_id":"case.n.19"},{"index":20,"sense_key":"case%1:06:01::","synset_id":"case.n.20"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"case%1:04:00::","call_ids":["semeval2013.d009.s018.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"case%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1817,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0037539999999999995,"input_uncached_usd":0.0036339999999999996,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9686468750005588}
{"item_id":"semeval2013.d009.s018.t004","gold_sense_keys":["court%1:14:00::"],"candidates":[{"index":1,"sense_key":"court%1:14:00::","synset_id":"court.n.01"},{"index":2,"sense_key":"court%1:06:02::","synset_id":"court.n.02"},{"index":3,"sense_key":"court%1:14:01::","synset_id":"court.n.03"},{"index":4,"sense_key":"court%1:06:01::","synset_id":"court.n.04"},{"index":5,"sense_key":"court%1:18:00::","synset_id":"court.n.05"},{"index":6,"sense_key":"court%1:14:02::","synset_id":"court.n.06"},{"index":7,"sense_key":"court%1:06:03::","synset_id":"motor_hotel.n.01"},{"index":8,"sense_key":"court%1:06:05::","synset_id":"court.n.08"},{"index":9,"sense_key":"court%1:06:04::","synset_id":"court.n.09"},{"index":10,"sense_key":"court%1:06:00::","synset_id":"court.n.10"},{"index":11,"sense_key":"court%1:04:00::","synset_id":"court.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"court%1:14:00::","call_ids":["semeval2013.d009.s018.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"court%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1181,"cached_input_tokens":0,"output_tokens":251,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004872,"input_uncached_usd":0.002362,"input_cached_usd":0.0,"output_usd":0.00251,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.0498974580550566}
{"item_id":"semeval2013.d009.s019.t000","gold_sense_keys":["appearance%1:04:00::"],"candidates":[{"index":1,"sense_key":"appearance%1:07:00::","synset_id":"appearance.n.01"},{"index":2,"sense_key":"appearance%1:11:00::","synset_id":"appearance.n.02"},{"index":3,"sense_key":"appearance%1:04:01::","synset_id":"appearance.n.03"},{"index":4,"sense_key":"appearance%1:09:00::","synset_id":"appearance.n.04"},{"index":5,"sense_key":"appearance%1:04:00::","synset_id":"appearance.n.05"},{"index":6,"sense_key":"appearance%1:04:02::","synset_id":"appearance.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"appearance%1:04:00::","call_ids":["semeval2013.d009.s019.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"appearance%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":915,"cached_input_tokens":0,"output_tokens":84,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0026699999999999996,"input_uncached_usd":0.0018299999999999998,"input_cached_usd":0.0,"output_usd":0.00084,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.5654376670718193}
{"item_id":"semeval2013.d009.s019.t001","gold_sense_keys":["assistance%1:04:00::"],"candidates":[{"index":1,"sense_key":"assistance%1:04:00::","synset_id":"aid.n.02"},{"index":2,"sense_key":"assistance%1:07:00::","synset_id":"aid.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"assistance%1:04:00::","call_ids":["semeval2013.d009.s019.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"assistance%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":735,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00159,"input_uncached_usd":0.00147,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.082397249992937}
{"item_id":"semeval2013.d009.s019.t002","gold_sense_keys":["event%1:03:00::"],"candidates":[{"index":1,"sense_key":"event%1:03:00::","synset_id":"event.n.01"},{"index":2,"sense_key":"event%1:26:00::","synset_id":"event.n.02"},{"index":3,"sense_key":"event%1:19:00::","synset_id":"event.n.03"},{"index":4,"sense_key":"event%1:19:01::","synset_id":"consequence.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"event%1:03:00::","call_ids":["semeval2013.d009.s019.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"event%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":851,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001822,"input_uncached_usd":0.001702,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.455073875025846}
{"item_id":"semeval2013.d009.s019.t003","gold_sense_keys":["claim%1:10:02::"],"candidates":[{"index":1,"sense_key":"claim%1:10:00::","synset_id":"claim.n.01"},{"index":2,"sense_key":"claim%1:10:02::","synset_id":"claim.n.02"},{"index":3,"sense_key":"claim%1:04:00::","synset_id":"claim.n.03"},{"index":4,"sense_key":"claim%1:07:02::","synset_id":"claim.n.04"},{"index":5,"sense_key":"claim%1:07:01::","synset_id":"title.n.07"},{"index":6,"sense_key":"claim%1:10:01::","synset_id":"call.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"claim%1:10:02::","call_ids":["semeval2013.d009.s019.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"claim%1:10:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":922,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002504,"input_uncached_usd":0.001844,"input_cached_usd":0.0,"output_usd":0.0006600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":5.131996833020821}
{"item_id":"semeval2013.d009.s020.t000","gold_sense_keys":["regard%1:09:01::"],"candidates":[{"index":1,"sense_key":"regard%1:09:00::","synset_id":"respect.n.01"},{"index":2,"sense_key":"regard%1:09:01::","synset_id":"attentiveness.n.01"},{"index":3,"sense_key":"regard%1:10:00::","synset_id":"regard.n.03"},{"index":4,"sense_key":"regard%1:04:00::","synset_id":"gaze.n.01"},{"index":5,"sense_key":"regard%1:26:00::","synset_id":"esteem.n.01"},{"index":6,"sense_key":"regard%1:12:02::","synset_id":"regard.n.06"},{"index":7,"sense_key":"regard%1:09:02::","synset_id":"respect.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"regard%1:09:01::","call_ids":["semeval2013.d009.s020.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"regard%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1052,"cached_input_tokens":0,"output_tokens":173,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003834,"input_uncached_usd":0.002104,"input_cached_usd":0.0,"output_usd":0.0017300000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.9328662080224603}
{"item_id":"semeval2013.d009.s020.t002","gold_sense_keys":["business%1:09:01::"],"candidates":[{"index":1,"sense_key":"business%1:14:00::","synset_id":"business.n.01"},{"index":2,"sense_key":"business%1:04:01::","synset_id":"commercial_enterprise.n.02"},{"index":3,"sense_key":"business%1:04:00::","synset_id":"occupation.n.01"},{"index":4,"sense_key":"business%1:09:01::","synset_id":"business.n.04"},{"index":5,"sense_key":"business%1:09:00::","synset_id":"business.n.05"},{"index":6,"sense_key":"business%1:04:02::","synset_id":"business.n.06"},{"index":7,"sense_key":"business%1:14:02::","synset_id":"business.n.07"},{"index":8,"sense_key":"business%1:14:01::","synset_id":"clientele.n.01"},{"index":9,"sense_key":"business%1:04:03::","synset_id":"business.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"business%1:09:01::","call_ids":["semeval2013.d009.s020.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"business%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1118,"cached_input_tokens":0,"output_tokens":121,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003446,"input_uncached_usd":0.0022359999999999997,"input_cached_usd":0.0,"output_usd":0.0012100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":5.544079833081923}
{"item_id":"semeval2013.d009.s020.t003","gold_sense_keys":["court%1:14:00::"],"candidates":[{"index":1,"sense_key":"court%1:14:00::","synset_id":"court.n.01"},{"index":2,"sense_key":"court%1:06:02::","synset_id":"court.n.02"},{"index":3,"sense_key":"court%1:14:01::","synset_id":"court.n.03"},{"index":4,"sense_key":"court%1:06:01::","synset_id":"court.n.04"},{"index":5,"sense_key":"court%1:18:00::","synset_id":"court.n.05"},{"index":6,"sense_key":"court%1:14:02::","synset_id":"court.n.06"},{"index":7,"sense_key":"court%1:06:03::","synset_id":"motor_hotel.n.01"},{"index":8,"sense_key":"court%1:06:05::","synset_id":"court.n.08"},{"index":9,"sense_key":"court%1:06:04::","synset_id":"court.n.09"},{"index":10,"sense_key":"court%1:06:00::","synset_id":"court.n.10"},{"index":11,"sense_key":"court%1:04:00::","synset_id":"court.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"court%1:14:00::","call_ids":["semeval2013.d009.s020.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"court%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1205,"cached_input_tokens":0,"output_tokens":175,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00416,"input_uncached_usd":0.00241,"input_cached_usd":0.0,"output_usd":0.00175,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8525057500228286}
{"item_id":"semeval2013.d009.s020.t004","gold_sense_keys":["group%1:03:00::"],"candidates":[{"index":1,"sense_key":"group%1:03:00::","synset_id":"group.n.01"},{"index":2,"sense_key":"group%1:27:00::","synset_id":"group.n.02"},{"index":3,"sense_key":"group%1:09:00::","synset_id":"group.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"group%1:03:00::","call_ids":["semeval2013.d009.s020.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"group%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":749,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001618,"input_uncached_usd":0.001498,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.132360624964349}
{"item_id":"semeval2013.d009.s022.t001","gold_sense_keys":["justice%1:18:00::"],"candidates":[{"index":1,"sense_key":"justice%1:07:00::","synset_id":"justice.n.01"},{"index":2,"sense_key":"justice%1:04:00::","synset_id":"justice.n.02"},{"index":3,"sense_key":"justice%1:18:00::","synset_id":"judge.n.01"},{"index":4,"sense_key":"justice%1:14:00::","synset_id":"department_of_justice.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"justice%1:18:00::","call_ids":["semeval2013.d009.s022.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"justice%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":755,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00163,"input_uncached_usd":0.0015099999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6116460419725627}
{"item_id":"semeval2013.d009.s022.t002","gold_sense_keys":["issue%1:09:01::"],"candidates":[{"index":1,"sense_key":"issue%1:09:01::","synset_id":"issue.n.01"},{"index":2,"sense_key":"issue%1:10:00::","synset_id":"issue.n.02"},{"index":3,"sense_key":"issue%1:09:00::","synset_id":"topic.n.02"},{"index":4,"sense_key":"issue%1:04:01::","synset_id":"issue.n.04"},{"index":5,"sense_key":"issue%1:21:01::","synset_id":"issue.n.05"},{"index":6,"sense_key":"issue%1:21:00::","synset_id":"return.n.06"},{"index":7,"sense_key":"issue%1:19:00::","synset_id":"consequence.n.01"},{"index":8,"sense_key":"issue%1:18:00::","synset_id":"offspring.n.01"},{"index":9,"sense_key":"issue%1:11:00::","synset_id":"emergence.n.02"},{"index":10,"sense_key":"issue%1:06:00::","synset_id":"exit.n.01"},{"index":11,"sense_key":"issue%1:04:00::","synset_id":"issue.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"issue%1:09:01::","call_ids":["semeval2013.d009.s022.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"issue%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1306,"cached_input_tokens":0,"output_tokens":163,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004242,"input_uncached_usd":0.0026119999999999997,"input_cached_usd":0.0,"output_usd":0.0016300000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8102301250910386}
{"item_id":"semeval2013.d009.s023.t001","gold_sense_keys":["administration%1:28:00::"],"candidates":[{"index":1,"sense_key":"administration%1:04:00::","synset_id":"administration.n.01"},{"index":2,"sense_key":"administration%1:14:00::","synset_id":"administration.n.02"},{"index":3,"sense_key":"administration%1:04:01::","synset_id":"administration.n.03"},{"index":4,"sense_key":"administration%1:28:00::","synset_id":"presidency.n.01"},{"index":5,"sense_key":"administration%1:04:04::","synset_id":"government.n.02"},{"index":6,"sense_key":"administration%1:04:02::","synset_id":"administration.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"administration%1:14:00::","call_ids":["semeval2013.d009.s023.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"administration%1:14:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":899,"cached_input_tokens":0,"output_tokens":222,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004018,"input_uncached_usd":0.001798,"input_cached_usd":0.0,"output_usd":0.00222,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.9719197920057923}
{"item_id":"semeval2013.d009.s023.t003","gold_sense_keys":["law%1:10:00::"],"candidates":[{"index":1,"sense_key":"law%1:14:00::","synset_id":"law.n.01"},{"index":2,"sense_key":"law%1:10:00::","synset_id":"law.n.02"},{"index":3,"sense_key":"law%1:09:00::","synset_id":"law.n.03"},{"index":4,"sense_key":"law%1:09:01::","synset_id":"law.n.04"},{"index":5,"sense_key":"law%1:09:02::","synset_id":"jurisprudence.n.01"},{"index":6,"sense_key":"law%1:04:00::","synset_id":"law.n.06"},{"index":7,"sense_key":"law%1:14:01::","synset_id":"police.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"law%1:10:00::","call_ids":["semeval2013.d009.s023.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"law%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":971,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002622,"input_uncached_usd":0.001942,"input_cached_usd":0.0,"output_usd":0.00068,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9592246669344604}
{"item_id":"semeval2013.d009.s023.t005","gold_sense_keys":["level%1:26:01::"],"candidates":[{"index":1,"sense_key":"level%1:07:00::","synset_id":"degree.n.01"},{"index":2,"sense_key":"level%1:26:01::","synset_id":"grade.n.02"},{"index":3,"sense_key":"level%1:26:00::","synset_id":"degree.n.02"},{"index":4,"sense_key":"level%1:07:01::","synset_id":"level.n.04"},{"index":5,"sense_key":"level%1:06:00::","synset_id":"level.n.05"},{"index":6,"sense_key":"level%1:06:03::","synset_id":"horizontal_surface.n.01"},{"index":7,"sense_key":"level%1:09:00::","synset_id":"level.n.07"},{"index":8,"sense_key":"level%1:06:01::","synset_id":"floor.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"level%1:26:01::","call_ids":["semeval2013.d009.s023.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"level%1:26:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1032,"cached_input_tokens":0,"output_tokens":112,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003184,"input_uncached_usd":0.002064,"input_cached_usd":0.0,"output_usd":0.0011200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9166670420672745}
{"item_id":"semeval2013.d009.s024.t001","gold_sense_keys":["case%1:04:00::"],"candidates":[{"index":1,"sense_key":"case%1:11:00::","synset_id":"case.n.01"},{"index":2,"sense_key":"case%1:26:00::","synset_id":"event.n.02"},{"index":3,"sense_key":"case%1:04:00::","synset_id":"lawsuit.n.01"},{"index":4,"sense_key":"case%1:09:00::","synset_id":"case.n.04"},{"index":5,"sense_key":"case%1:06:00::","synset_id":"case.n.05"},{"index":6,"sense_key":"case%1:18:03::","synset_id":"case.n.06"},{"index":7,"sense_key":"case%1:18:02::","synset_id":"subject.n.06"},{"index":8,"sense_key":"case%1:10:02::","synset_id":"case.n.08"},{"index":9,"sense_key":"case%1:10:01::","synset_id":"case.n.09"},{"index":10,"sense_key":"case%1:23:00::","synset_id":"case.n.10"},{"index":11,"sense_key":"case%1:10:00::","synset_id":"case.n.11"},{"index":12,"sense_key":"case%1:26:02::","synset_id":"case.n.12"},{"index":13,"sense_key":"case%1:18:04::","synset_id":"character.n.05"},{"index":14,"sense_key":"case%1:10:03::","synset_id":"font.n.01"},{"index":15,"sense_key":"case%1:08:00::","synset_id":"sheath.n.02"},{"index":16,"sense_key":"case%1:06:06::","synset_id":"shell.n.08"},{"index":17,"sense_key":"case%1:06:07::","synset_id":"casing.n.03"},{"index":18,"sense_key":"case%1:06:03::","synset_id":"case.n.18"},{"index":19,"sense_key":"case%1:06:02::","synset_id":"case.n.19"},{"index":20,"sense_key":"case%1:06:01::","synset_id":"case.n.20"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"case%1:04:00::","call_ids":["semeval2013.d009.s024.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"case%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1741,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0036019999999999997,"input_uncached_usd":0.003482,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.151779834064655}
{"item_id":"semeval2013.d009.s024.t002","gold_sense_keys":["law%1:10:00::"],"candidates":[{"index":1,"sense_key":"law%1:14:00::","synset_id":"law.n.01"},{"index":2,"sense_key":"law%1:10:00::","synset_id":"law.n.02"},{"index":3,"sense_key":"law%1:09:00::","synset_id":"law.n.03"},{"index":4,"sense_key":"law%1:09:01::","synset_id":"law.n.04"},{"index":5,"sense_key":"law%1:09:02::","synset_id":"jurisprudence.n.01"},{"index":6,"sense_key":"law%1:04:00::","synset_id":"law.n.06"},{"index":7,"sense_key":"law%1:14:01::","synset_id":"police.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"law%1:10:00::","call_ids":["semeval2013.d009.s024.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"law%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":949,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002618,"input_uncached_usd":0.001898,"input_cached_usd":0.0,"output_usd":0.00072,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9333379169693217}
{"item_id":"semeval2013.d009.s025.t000","gold_sense_keys":["indication%1:10:00::"],"candidates":[{"index":1,"sense_key":"indication%1:10:00::","synset_id":"indication.n.01"},{"index":2,"sense_key":"indication%1:10:01::","synset_id":"indication.n.02"},{"index":3,"sense_key":"indication%1:10:02::","synset_id":"indication.n.03"},{"index":4,"sense_key":"indication%1:10:03::","synset_id":"indication.n.04"},{"index":5,"sense_key":"indication%1:09:00::","synset_id":"reading.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"indication%1:10:00::","call_ids":["semeval2013.d009.s025.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"indication%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":777,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001674,"input_uncached_usd":0.0015539999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":10.479506250005215}
{"item_id":"semeval2013.d010.s000.t000","gold_sense_keys":["familiarity%1:09:00::"],"candidates":[{"index":1,"sense_key":"familiarity%1:09:00::","synset_id":"acquaintance.n.01"},{"index":2,"sense_key":"familiarity%1:07:01::","synset_id":"familiarity.n.02"},{"index":3,"sense_key":"familiarity%1:07:02::","synset_id":"familiarity.n.03"},{"index":4,"sense_key":"familiarity%1:07:03::","synset_id":"casualness.n.01"},{"index":5,"sense_key":"familiarity%1:04:00::","synset_id":"familiarity.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"familiarity%1:09:00::","call_ids":["semeval2013.d010.s000.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"familiarity%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":540,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0018,"input_uncached_usd":0.00108,"input_cached_usd":0.0,"output_usd":0.00072,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4331260420149192}
{"item_id":"semeval2013.d010.s001.t000","gold_sense_keys":["player%1:18:01::"],"candidates":[{"index":1,"sense_key":"player%1:18:01::","synset_id":"player.n.01"},{"index":2,"sense_key":"player%1:18:02::","synset_id":"musician.n.01"},{"index":3,"sense_key":"player%1:18:00::","synset_id":"actor.n.01"},{"index":4,"sense_key":"player%1:18:04::","synset_id":"player.n.04"},{"index":5,"sense_key":"player%1:18:03::","synset_id":"player.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"player%1:18:01::","call_ids":["semeval2013.d010.s001.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"player%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":535,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00119,"input_uncached_usd":0.00107,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9593820839654654}
{"item_id":"semeval2013.d010.s001.t002","gold_sense_keys":["game%1:04:03::"],"candidates":[{"index":1,"sense_key":"game%1:04:00::","synset_id":"game.n.01"},{"index":2,"sense_key":"game%1:04:03::","synset_id":"game.n.02"},{"index":3,"sense_key":"game%1:04:01::","synset_id":"game.n.03"},{"index":4,"sense_key":"game%1:05:00::","synset_id":"game.n.04"},{"index":5,"sense_key":"game%1:28:00::","synset_id":"game.n.05"},{"index":6,"sense_key":"game%1:23:00::","synset_id":"game.n.06"},{"index":7,"sense_key":"game%1:13:00::","synset_id":"game.n.07"},{"index":8,"sense_key":"game%1:09:00::","synset_id":"plot.n.01"},{"index":9,"sense_key":"game%1:06:00::","synset_id":"game.n.09"},{"index":10,"sense_key":"game%1:04:02::","synset_id":"game.n.10"},{"index":11,"sense_key":"game%1:04:04::","synset_id":"game.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"game%1:04:03::","call_ids":["semeval2013.d010.s001.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"game%1:04:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":853,"cached_input_tokens":0,"output_tokens":52,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0022259999999999997,"input_uncached_usd":0.0017059999999999998,"input_cached_usd":0.0,"output_usd":0.0005200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2946904168929905}
{"item_id":"semeval2013.d010.s002.t000","gold_sense_keys":["side%1:14:00::"],"candidates":[{"index":1,"sense_key":"side%1:15:02::","synset_id":"side.n.01"},{"index":2,"sense_key":"side%1:14:00::","synset_id":"side.n.02"},{"index":3,"sense_key":"side%1:08:00::","synset_id":"side.n.03"},{"index":4,"sense_key":"side%1:15:00::","synset_id":"side.n.04"},{"index":5,"sense_key":"side%1:06:00::","synset_id":"side.n.05"},{"index":6,"sense_key":"side%1:09:00::","synset_id":"side.n.06"},{"index":7,"sense_key":"side%1:15:01::","synset_id":"side.n.07"},{"index":8,"sense_key":"side%1:14:01::","synset_id":"side.n.08"},{"index":9,"sense_key":"side%1:13:00::","synset_id":"side.n.09"},{"index":10,"sense_key":"side%1:10:00::","synset_id":"side.n.10"},{"index":11,"sense_key":"side%1:17:01::","synset_id":"slope.n.01"},{"index":12,"sense_key":"side%1:11:00::","synset_id":"english.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"side%1:14:00::","call_ids":["semeval2013.d010.s002.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"side%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1043,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002856,"input_uncached_usd":0.0020859999999999997,"input_cached_usd":0.0,"output_usd":0.0007700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2608904170338064}
{"item_id":"semeval2013.d010.s002.t002","gold_sense_keys":["player%1:18:01::"],"candidates":[{"index":1,"sense_key":"player%1:18:01::","synset_id":"player.n.01"},{"index":2,"sense_key":"player%1:18:02::","synset_id":"musician.n.01"},{"index":3,"sense_key":"player%1:18:00::","synset_id":"actor.n.01"},{"index":4,"sense_key":"player%1:18:04::","synset_id":"player.n.04"},{"index":5,"sense_key":"player%1:18:03::","synset_id":"player.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"player%1:18:01::","call_ids":["semeval2013.d010.s002.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"player%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":591,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001302,"input_uncached_usd":0.0011819999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8232152500422671}
{"item_id":"semeval2013.d010.s002.t003","gold_sense_keys":["game%1:04:03::"],"candidates":[{"index":1,"sense_key":"game%1:04:00::","synset_id":"game.n.01"},{"index":2,"sense_key":"game%1:04:03::","synset_id":"game.n.02"},{"index":3,"sense_key":"game%1:04:01::","synset_id":"game.n.03"},{"index":4,"sense_key":"game%1:05:00::","synset_id":"game.n.04"},{"index":5,"sense_key":"game%1:28:00::","synset_id":"game.n.05"},{"index":6,"sense_key":"game%1:23:00::","synset_id":"game.n.06"},{"index":7,"sense_key":"game%1:13:00::","synset_id":"game.n.07"},{"index":8,"sense_key":"game%1:09:00::","synset_id":"plot.n.01"},{"index":9,"sense_key":"game%1:06:00::","synset_id":"game.n.09"},{"index":10,"sense_key":"game%1:04:02::","synset_id":"game.n.10"},{"index":11,"sense_key":"game%1:04:04::","synset_id":"game.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"game%1:04:03::","call_ids":["semeval2013.d010.s002.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"game%1:04:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":909,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002438,"input_uncached_usd":0.001818,"input_cached_usd":0.0,"output_usd":0.00062,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.780372708919458}
{"item_id":"semeval2013.d010.s003.t002","gold_sense_keys":["player%1:18:01::"],"candidates":[{"index":1,"sense_key":"player%1:18:01::","synset_id":"player.n.01"},{"index":2,"sense_key":"player%1:18:02::","synset_id":"musician.n.01"},{"index":3,"sense_key":"player%1:18:00::","synset_id":"actor.n.01"},{"index":4,"sense_key":"player%1:18:04::","synset_id":"player.n.04"},{"index":5,"sense_key":"player%1:18:03::","synset_id":"player.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"player%1:18:01::","call_ids":["semeval2013.d010.s003.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"player%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":658,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001436,"input_uncached_usd":0.001316,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.461598917027004}
{"item_id":"semeval2013.d010.s003.t004","gold_sense_keys":["game%1:04:03::"],"candidates":[{"index":1,"sense_key":"game%1:04:00::","synset_id":"game.n.01"},{"index":2,"sense_key":"game%1:04:03::","synset_id":"game.n.02"},{"index":3,"sense_key":"game%1:04:01::","synset_id":"game.n.03"},{"index":4,"sense_key":"game%1:05:00::","synset_id":"game.n.04"},{"index":5,"sense_key":"game%1:28:00::","synset_id":"game.n.05"},{"index":6,"sense_key":"game%1:23:00::","synset_id":"game.n.06"},{"index":7,"sense_key":"game%1:13:00::","synset_id":"game.n.07"},{"index":8,"sense_key":"game%1:09:00::","synset_id":"plot.n.01"},{"index":9,"sense_key":"game%1:06:00::","synset_id":"game.n.09"},{"index":10,"sense_key":"game%1:04:02::","synset_id":"game.n.10"},{"index":11,"sense_key":"game%1:04:04::","synset_id":"game.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"game%1:04:03::","call_ids":["semeval2013.d010.s003.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"game%1:04:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":976,"cached_input_tokens":0,"output_tokens":88,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002832,"input_uncached_usd":0.001952,"input_cached_usd":0.0,"output_usd":0.00088,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5122379590757191}
{"item_id":"semeval2013.d010.s003.t005","gold_sense_keys":["today%1:28:01::"],"candidates":[{"index":1,"sense_key":"today%1:28:00::","synset_id":"today.n.01"},{"index":2,"sense_key":"today%1:28:01::","synset_id":"today.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"today%1:28:01::","call_ids":["semeval2013.d010.s003.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"today%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":531,"cached_input_tokens":0,"output_tokens":74,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0018020000000000002,"input_uncached_usd":0.001062,"input_cached_usd":0.0,"output_usd":0.0007400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4620635829633102}
{"item_id":"semeval2013.d010.s003.t006","gold_sense_keys":["san_jose%1:15:00::"],"candidates":[{"index":1,"sense_key":"san_jose%1:15:01::","synset_id":"san_jose.n.01"},{"index":2,"sense_key":"san_jose%1:15:00::","synset_id":"san_jose.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"san_jose%1:15:00::","call_ids":["semeval2013.d010.s003.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"san_jose%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":546,"cached_input_tokens":0,"output_tokens":82,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001912,"input_uncached_usd":0.0010919999999999999,"input_cached_usd":0.0,"output_usd":0.0008200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8235277499770746}
{"item_id":"semeval2013.d010.s004.t001","gold_sense_keys":["number_one%1:24:00::"],"candidates":[{"index":1,"sense_key":"number_one%1:18:00::","synset_id":"number_one.n.01"},{"index":2,"sense_key":"number_one%1:24:00::","synset_id":"first.n.01"},{"index":3,"sense_key":"number_one%1:23:00::","synset_id":"first.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"number_one%1:24:00::","call_ids":["semeval2013.d010.s004.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"number_one%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":614,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001948,"input_uncached_usd":0.001228,"input_cached_usd":0.0,"output_usd":0.00072,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4433203330263495}
{"item_id":"semeval2013.d010.s004.t003","gold_sense_keys":["record%1:04:01::"],"candidates":[{"index":1,"sense_key":"record%1:10:03::","synset_id":"record.n.01"},{"index":2,"sense_key":"record%1:06:00::","synset_id":"phonograph_record.n.01"},{"index":3,"sense_key":"record%1:23:00::","synset_id":"record.n.03"},{"index":4,"sense_key":"record%1:04:01::","synset_id":"record.n.04"},{"index":5,"sense_key":"record%1:10:02::","synset_id":"record.n.05"},{"index":6,"sense_key":"record%1:04:00::","synset_id":"record.n.06"},{"index":7,"sense_key":"record%1:21:00::","synset_id":"record.n.07"},{"index":8,"sense_key":"record%1:10:00::","synset_id":"criminal_record.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"record%1:04:01::","call_ids":["semeval2013.d010.s004.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"record%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1039,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002748,"input_uncached_usd":0.002078,"input_cached_usd":0.0,"output_usd":0.00067,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1574362079845741}
{"item_id":"semeval2013.d010.s004.t005","gold_sense_keys":["game%1:04:03::"],"candidates":[{"index":1,"sense_key":"game%1:04:00::","synset_id":"game.n.01"},{"index":2,"sense_key":"game%1:04:03::","synset_id":"game.n.02"},{"index":3,"sense_key":"game%1:04:01::","synset_id":"game.n.03"},{"index":4,"sense_key":"game%1:05:00::","synset_id":"game.n.04"},{"index":5,"sense_key":"game%1:28:00::","synset_id":"game.n.05"},{"index":6,"sense_key":"game%1:23:00::","synset_id":"game.n.06"},{"index":7,"sense_key":"game%1:13:00::","synset_id":"game.n.07"},{"index":8,"sense_key":"game%1:09:00::","synset_id":"plot.n.01"},{"index":9,"sense_key":"game%1:06:00::","synset_id":"game.n.09"},{"index":10,"sense_key":"game%1:04:02::","synset_id":"game.n.10"},{"index":11,"sense_key":"game%1:04:04::","synset_id":"game.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"game%1:04:03::","call_ids":["semeval2013.d010.s004.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"game%1:04:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1007,"cached_input_tokens":0,"output_tokens":47,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0024839999999999997,"input_uncached_usd":0.0020139999999999997,"input_cached_usd":0.0,"output_usd":0.00047000000000000004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2821652919519693}
{"item_id":"semeval2013.d010.s005.t000","gold_sense_keys":["number%1:07:00::"],"candidates":[{"index":1,"sense_key":"number%1:07:00::","synset_id":"number.n.01"},{"index":2,"sense_key":"number%1:23:00::","synset_id":"number.n.02"},{"index":3,"sense_key":"number%1:10:04::","synset_id":"act.n.04"},{"index":4,"sense_key":"number%1:10:03::","synset_id":"phone_number.n.01"},{"index":5,"sense_key":"number%1:10:01::","synset_id":"numeral.n.01"},{"index":6,"sense_key":"number%1:10:00::","synset_id":"issue.n.02"},{"index":7,"sense_key":"number%1:14:00::","synset_id":"number.n.07"},{"index":8,"sense_key":"number%1:10:02::","synset_id":"number.n.08"},{"index":9,"sense_key":"number%1:07:01::","synset_id":"number.n.09"},{"index":10,"sense_key":"number%1:10:05::","synset_id":"number.n.10"},{"index":11,"sense_key":"number%1:06:00::","synset_id":"number.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"number%1:07:00::","call_ids":["semeval2013.d010.s005.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"number%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1191,"cached_input_tokens":0,"output_tokens":92,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0033020000000000002,"input_uncached_usd":0.002382,"input_cached_usd":0.0,"output_usd":0.00092,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5452483330154791}
{"item_id":"semeval2013.d010.s005.t002","gold_sense_keys":["game%1:04:03::"],"candidates":[{"index":1,"sense_key":"game%1:04:00::","synset_id":"game.n.01"},{"index":2,"sense_key":"game%1:04:03::","synset_id":"game.n.02"},{"index":3,"sense_key":"game%1:04:01::","synset_id":"game.n.03"},{"index":4,"sense_key":"game%1:05:00::","synset_id":"game.n.04"},{"index":5,"sense_key":"game%1:28:00::","synset_id":"game.n.05"},{"index":6,"sense_key":"game%1:23:00::","synset_id":"game.n.06"},{"index":7,"sense_key":"game%1:13:00::","synset_id":"game.n.07"},{"index":8,"sense_key":"game%1:09:00::","synset_id":"plot.n.01"},{"index":9,"sense_key":"game%1:06:00::","synset_id":"game.n.09"},{"index":10,"sense_key":"game%1:04:02::","synset_id":"game.n.10"},{"index":11,"sense_key":"game%1:04:04::","synset_id":"game.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"game%1:04:03::","call_ids":["semeval2013.d010.s005.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"game%1:04:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1075,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00278,"input_uncached_usd":0.00215,"input_cached_usd":0.0,"output_usd":0.00063,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":6.48052900005132}
{"item_id":"semeval2013.d010.s006.t001","gold_sense_keys":["league%1:14:01::"],"candidates":[{"index":1,"sense_key":"league%1:14:01::","synset_id":"league.n.01"},{"index":2,"sense_key":"league%1:14:00::","synset_id":"league.n.02"},{"index":3,"sense_key":"league%1:23:00::","synset_id":"league.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"league%1:14:01::","call_ids":["semeval2013.d010.s006.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"league%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":659,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001938,"input_uncached_usd":0.001318,"input_cached_usd":0.0,"output_usd":0.00062,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4992499169893563}
{"item_id":"semeval2013.d010.s006.t004","gold_sense_keys":["year%1:28:01::","year%1:28:02::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["semeval2013.d010.s006.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":794,"cached_input_tokens":0,"output_tokens":135,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002938,"input_uncached_usd":0.001588,"input_cached_usd":0.0,"output_usd":0.00135,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9310139999724925}
{"item_id":"semeval2013.d010.s006.t005","gold_sense_keys":["league%1:14:01::"],"candidates":[{"index":1,"sense_key":"league%1:14:01::","synset_id":"league.n.01"},{"index":2,"sense_key":"league%1:14:00::","synset_id":"league.n.02"},{"index":3,"sense_key":"league%1:23:00::","synset_id":"league.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"league%1:14:01::","call_ids":["semeval2013.d010.s006.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"league%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":659,"cached_input_tokens":0,"output_tokens":47,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001788,"input_uncached_usd":0.001318,"input_cached_usd":0.0,"output_usd":0.00047000000000000004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":8.812121416907758}
{"item_id":"semeval2013.d010.s006.t007","gold_sense_keys":["competition%1:11:00::"],"candidates":[{"index":1,"sense_key":"competition%1:24:01::","synset_id":"competition.n.01"},{"index":2,"sense_key":"competition%1:11:00::","synset_id":"contest.n.01"},{"index":3,"sense_key":"competition%1:04:00::","synset_id":"competition.n.03"},{"index":4,"sense_key":"competition%1:18:00::","synset_id":"rival.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"competition%1:11:00::","call_ids":["semeval2013.d010.s006.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"competition%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":784,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0021780000000000002,"input_uncached_usd":0.001568,"input_cached_usd":0.0,"output_usd":0.0006100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2155711660161614}
{"item_id":"semeval2013.d010.s007.t000","gold_sense_keys":["experience%1:09:01::"],"candidates":[{"index":1,"sense_key":"experience%1:09:01::","synset_id":"experience.n.01"},{"index":2,"sense_key":"experience%1:09:00::","synset_id":"experience.n.02"},{"index":3,"sense_key":"experience%1:11:00::","synset_id":"experience.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"experience%1:09:01::","call_ids":["semeval2013.d010.s007.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"experience%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":729,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0021279999999999997,"input_uncached_usd":0.0014579999999999999,"input_cached_usd":0.0,"output_usd":0.00067,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9321560830576345}
{"item_id":"semeval2013.d010.s008.t000","gold_sense_keys":["player%1:18:01::"],"candidates":[{"index":1,"sense_key":"player%1:18:01::","synset_id":"player.n.01"},{"index":2,"sense_key":"player%1:18:02::","synset_id":"musician.n.01"},{"index":3,"sense_key":"player%1:18:00::","synset_id":"actor.n.01"},{"index":4,"sense_key":"player%1:18:04::","synset_id":"player.n.04"},{"index":5,"sense_key":"player%1:18:03::","synset_id":"player.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"player%1:18:01::","call_ids":["semeval2013.d010.s008.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"player%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":850,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00182,"input_uncached_usd":0.0017,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1122671659104526}
{"item_id":"semeval2013.d010.s008.t001","gold_sense_keys":["experience%1:09:01::"],"candidates":[{"index":1,"sense_key":"experience%1:09:01::","synset_id":"experience.n.01"},{"index":2,"sense_key":"experience%1:09:00::","synset_id":"experience.n.02"},{"index":3,"sense_key":"experience%1:11:00::","synset_id":"experience.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"experience%1:09:01::","call_ids":["semeval2013.d010.s008.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"experience%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":790,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00212,"input_uncached_usd":0.00158,"input_cached_usd":0.0,"output_usd":0.00054,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3635234580142424}
{"item_id":"semeval2013.d010.s008.t002","gold_sense_keys":["area%1:15:01::"],"candidates":[{"index":1,"sense_key":"area%1:15:01::","synset_id":"area.n.01"},{"index":2,"sense_key":"area%1:09:00::","synset_id":"area.n.02"},{"index":3,"sense_key":"area%1:08:00::","synset_id":"area.n.03"},{"index":4,"sense_key":"area%1:26:00::","synset_id":"sphere.n.01"},{"index":5,"sense_key":"area%1:06:00::","synset_id":"area.n.05"},{"index":6,"sense_key":"area%1:07:00::","synset_id":"area.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"area%1:26:00::","call_ids":["semeval2013.d010.s008.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"area%1:26:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1023,"cached_input_tokens":0,"output_tokens":143,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003476,"input_uncached_usd":0.002046,"input_cached_usd":0.0,"output_usd":0.00143,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8236665410222486}
{"item_id":"semeval2013.d010.s008.t003","gold_sense_keys":["level%1:26:01::"],"candidates":[{"index":1,"sense_key":"level%1:07:00::","synset_id":"degree.n.01"},{"index":2,"sense_key":"level%1:26:01::","synset_id":"grade.n.02"},{"index":3,"sense_key":"level%1:26:00::","synset_id":"degree.n.02"},{"index":4,"sense_key":"level%1:07:01::","synset_id":"level.n.04"},{"index":5,"sense_key":"level%1:06:00::","synset_id":"level.n.05"},{"index":6,"sense_key":"level%1:06:03::","synset_id":"horizontal_surface.n.01"},{"index":7,"sense_key":"level%1:09:00::","synset_id":"level.n.07"},{"index":8,"sense_key":"level%1:06:01::","synset_id":"floor.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"level%1:26:01::","call_ids":["semeval2013.d010.s008.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"level%1:26:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1137,"cached_input_tokens":0,"output_tokens":219,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004464,"input_uncached_usd":0.002274,"input_cached_usd":0.0,"output_usd":0.00219,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":15.847263415926136}
{"item_id":"semeval2013.d010.s009.t000","gold_sense_keys":["peak%1:26:00::"],"candidates":[{"index":1,"sense_key":"peak%1:23:00::","synset_id":"extremum.n.02"},{"index":2,"sense_key":"peak%1:28:00::","synset_id":"flower.n.03"},{"index":3,"sense_key":"peak%1:26:00::","synset_id":"acme.n.01"},{"index":4,"sense_key":"peak%1:15:00::","synset_id":"peak.n.04"},{"index":5,"sense_key":"peak%1:25:01::","synset_id":"point.n.08"},{"index":6,"sense_key":"peak%1:15:01::","synset_id":"vertex.n.02"},{"index":7,"sense_key":"peak%1:06:00::","synset_id":"bill.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"peak%1:26:00::","call_ids":["semeval2013.d010.s009.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"peak%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1098,"cached_input_tokens":0,"output_tokens":145,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003646,"input_uncached_usd":0.002196,"input_cached_usd":0.0,"output_usd":0.0014500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.961091083008796}
{"item_id":"semeval2013.d010.s009.t002","gold_sense_keys":["goalkeeper%1:18:00::"],"candidates":[{"index":1,"sense_key":"goalkeeper%1:18:00::","synset_id":"goalkeeper.n.01"},{"index":2,"sense_key":"goalkeeper%1:04:00::","synset_id":"goalkeeper.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"goalkeeper%1:18:00::","call_ids":["semeval2013.d010.s009.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"goalkeeper%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":762,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002154,"input_uncached_usd":0.001524,"input_cached_usd":0.0,"output_usd":0.00063,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":14.487485875026323}
{"item_id":"semeval2013.d010.s009.t005","gold_sense_keys":["game%1:04:03::"],"candidates":[{"index":1,"sense_key":"game%1:04:00::","synset_id":"game.n.01"},{"index":2,"sense_key":"game%1:04:03::","synset_id":"game.n.02"},{"index":3,"sense_key":"game%1:04:01::","synset_id":"game.n.03"},{"index":4,"sense_key":"game%1:05:00::","synset_id":"game.n.04"},{"index":5,"sense_key":"game%1:28:00::","synset_id":"game.n.05"},{"index":6,"sense_key":"game%1:23:00::","synset_id":"game.n.06"},{"index":7,"sense_key":"game%1:13:00::","synset_id":"game.n.07"},{"index":8,"sense_key":"game%1:09:00::","synset_id":"plot.n.01"},{"index":9,"sense_key":"game%1:06:00::","synset_id":"game.n.09"},{"index":10,"sense_key":"game%1:04:02::","synset_id":"game.n.10"},{"index":11,"sense_key":"game%1:04:04::","synset_id":"game.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"game%1:04:03::","call_ids":["semeval2013.d010.s009.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"game%1:04:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1145,"cached_input_tokens":0,"output_tokens":75,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00304,"input_uncached_usd":0.00229,"input_cached_usd":0.0,"output_usd":0.00075,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8430837499909103}
{"item_id":"semeval2013.d010.s009.t006","gold_sense_keys":["london%1:15:00::"],"candidates":[{"index":1,"sense_key":"london%1:15:00::","synset_id":"london.n.01"},{"index":2,"sense_key":"london%1:18:00::","synset_id":"london.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"london%1:15:00::","call_ids":["semeval2013.d010.s009.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"london%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":741,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0016020000000000001,"input_uncached_usd":0.001482,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5206586250569671}
{"item_id":"semeval2013.d010.s009.t007","gold_sense_keys":["record%1:04:00::"],"candidates":[{"index":1,"sense_key":"record%1:10:03::","synset_id":"record.n.01"},{"index":2,"sense_key":"record%1:06:00::","synset_id":"phonograph_record.n.01"},{"index":3,"sense_key":"record%1:23:00::","synset_id":"record.n.03"},{"index":4,"sense_key":"record%1:04:01::","synset_id":"record.n.04"},{"index":5,"sense_key":"record%1:10:02::","synset_id":"record.n.05"},{"index":6,"sense_key":"record%1:04:00::","synset_id":"record.n.06"},{"index":7,"sense_key":"record%1:21:00::","synset_id":"record.n.07"},{"index":8,"sense_key":"record%1:10:00::","synset_id":"criminal_record.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"record%1:04:00::","call_ids":["semeval2013.d010.s009.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"record%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1177,"cached_input_tokens":0,"output_tokens":108,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0034339999999999996,"input_uncached_usd":0.0023539999999999998,"input_cached_usd":0.0,"output_usd":0.00108,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4557944170664996}
{"item_id":"semeval2013.d010.s009.t008","gold_sense_keys":["goalkeeper%1:18:00::"],"candidates":[{"index":1,"sense_key":"goalkeeper%1:18:00::","synset_id":"goalkeeper.n.01"},{"index":2,"sense_key":"goalkeeper%1:04:00::","synset_id":"goalkeeper.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"goalkeeper%1:18:00::","call_ids":["semeval2013.d010.s009.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"goalkeeper%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":762,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002064,"input_uncached_usd":0.001524,"input_cached_usd":0.0,"output_usd":0.00054,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":11.170397292007692}
{"item_id":"semeval2013.d010.s009.t009","gold_sense_keys":["match%1:11:00::"],"candidates":[{"index":1,"sense_key":"match%1:06:00::","synset_id":"match.n.01"},{"index":2,"sense_key":"match%1:11:00::","synset_id":"match.n.02"},{"index":3,"sense_key":"match%1:06:02::","synset_id":"match.n.03"},{"index":4,"sense_key":"match%1:06:01::","synset_id":"match.n.04"},{"index":5,"sense_key":"match%1:23:00::","synset_id":"match.n.05"},{"index":6,"sense_key":"match%1:18:01::","synset_id":"catch.n.03"},{"index":7,"sense_key":"match%1:18:00::","synset_id":"peer.n.01"},{"index":8,"sense_key":"match%1:14:00::","synset_id":"couple.n.02"},{"index":9,"sense_key":"match%1:09:00::","synset_id":"match.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"match%1:11:00::","call_ids":["semeval2013.d010.s009.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"match%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1068,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0022559999999999998,"input_uncached_usd":0.002136,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.462208874989301}
{"item_id":"semeval2013.d010.s010.t000","gold_sense_keys":["today%1:28:01::"],"candidates":[{"index":1,"sense_key":"today%1:28:00::","synset_id":"today.n.01"},{"index":2,"sense_key":"today%1:28:01::","synset_id":"today.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"today%1:28:01::","call_ids":["semeval2013.d010.s010.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"today%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":659,"cached_input_tokens":0,"output_tokens":43,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001748,"input_uncached_usd":0.001318,"input_cached_usd":0.0,"output_usd":0.00043000000000000004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3835502500878647}
{"item_id":"semeval2013.d010.s010.t001","gold_sense_keys":["record%1:04:00::"],"candidates":[{"index":1,"sense_key":"record%1:10:03::","synset_id":"record.n.01"},{"index":2,"sense_key":"record%1:06:00::","synset_id":"phonograph_record.n.01"},{"index":3,"sense_key":"record%1:23:00::","synset_id":"record.n.03"},{"index":4,"sense_key":"record%1:04:01::","synset_id":"record.n.04"},{"index":5,"sense_key":"record%1:10:02::","synset_id":"record.n.05"},{"index":6,"sense_key":"record%1:04:00::","synset_id":"record.n.06"},{"index":7,"sense_key":"record%1:21:00::","synset_id":"record.n.07"},{"index":8,"sense_key":"record%1:10:00::","synset_id":"criminal_record.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"record%1:04:00::","call_ids":["semeval2013.d010.s010.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"record%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1136,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0023919999999999996,"input_uncached_usd":0.0022719999999999997,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1789154580328614}
{"item_id":"semeval2013.d010.s010.t002","gold_sense_keys":["match%1:11:00::"],"candidates":[{"index":1,"sense_key":"match%1:06:00::","synset_id":"match.n.01"},{"index":2,"sense_key":"match%1:11:00::","synset_id":"match.n.02"},{"index":3,"sense_key":"match%1:06:02::","synset_id":"match.n.03"},{"index":4,"sense_key":"match%1:06:01::","synset_id":"match.n.04"},{"index":5,"sense_key":"match%1:23:00::","synset_id":"match.n.05"},{"index":6,"sense_key":"match%1:18:01::","synset_id":"catch.n.03"},{"index":7,"sense_key":"match%1:18:00::","synset_id":"peer.n.01"},{"index":8,"sense_key":"match%1:14:00::","synset_id":"couple.n.02"},{"index":9,"sense_key":"match%1:09:00::","synset_id":"match.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"match%1:11:00::","call_ids":["semeval2013.d010.s010.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"match%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1027,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0021739999999999997,"input_uncached_usd":0.002054,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":14.206831208080985}
{"item_id":"semeval2013.d010.s011.t000","gold_sense_keys":["player%1:18:01::"],"candidates":[{"index":1,"sense_key":"player%1:18:01::","synset_id":"player.n.01"},{"index":2,"sense_key":"player%1:18:02::","synset_id":"musician.n.01"},{"index":3,"sense_key":"player%1:18:00::","synset_id":"actor.n.01"},{"index":4,"sense_key":"player%1:18:04::","synset_id":"player.n.04"},{"index":5,"sense_key":"player%1:18:03::","synset_id":"player.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"player%1:18:01::","call_ids":["semeval2013.d010.s011.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"player%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":821,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0017620000000000001,"input_uncached_usd":0.001642,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4023564170347527}
{"item_id":"semeval2013.d010.s011.t001","gold_sense_keys":["match%1:11:00::"],"candidates":[{"index":1,"sense_key":"match%1:06:00::","synset_id":"match.n.01"},{"index":2,"sense_key":"match%1:11:00::","synset_id":"match.n.02"},{"index":3,"sense_key":"match%1:06:02::","synset_id":"match.n.03"},{"index":4,"sense_key":"match%1:06:01::","synset_id":"match.n.04"},{"index":5,"sense_key":"match%1:23:00::","synset_id":"match.n.05"},{"index":6,"sense_key":"match%1:18:01::","synset_id":"catch.n.03"},{"index":7,"sense_key":"match%1:18:00::","synset_id":"peer.n.01"},{"index":8,"sense_key":"match%1:14:00::","synset_id":"couple.n.02"},{"index":9,"sense_key":"match%1:09:00::","synset_id":"match.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"match%1:11:00::","call_ids":["semeval2013.d010.s011.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"match%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1062,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002244,"input_uncached_usd":0.002124,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0802705420646816}
{"item_id":"semeval2013.d010.s012.t000","gold_sense_keys":["week%1:28:02::"],"candidates":[{"index":1,"sense_key":"week%1:28:00::","synset_id":"week.n.01"},{"index":2,"sense_key":"week%1:28:01::","synset_id":"workweek.n.01"},{"index":3,"sense_key":"week%1:28:02::","synset_id":"week.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"week%1:28:00::","call_ids":["semeval2013.d010.s012.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"week%1:28:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":697,"cached_input_tokens":0,"output_tokens":133,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002724,"input_uncached_usd":0.0013939999999999998,"input_cached_usd":0.0,"output_usd":0.00133,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.4152713329531252}
{"item_id":"semeval2013.d010.s012.t001","gold_sense_keys":["player%1:18:01::"],"candidates":[{"index":1,"sense_key":"player%1:18:01::","synset_id":"player.n.01"},{"index":2,"sense_key":"player%1:18:02::","synset_id":"musician.n.01"},{"index":3,"sense_key":"player%1:18:00::","synset_id":"actor.n.01"},{"index":4,"sense_key":"player%1:18:04::","synset_id":"player.n.04"},{"index":5,"sense_key":"player%1:18:03::","synset_id":"player.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"player%1:18:01::","call_ids":["semeval2013.d010.s012.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"player%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":777,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001674,"input_uncached_usd":0.0015539999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8804357079789042}
{"item_id":"semeval2013.d010.s012.t002","gold_sense_keys":["team%1:14:00::"],"candidates":[{"index":1,"sense_key":"team%1:14:00::","synset_id":"team.n.01"},{"index":2,"sense_key":"team%1:14:01::","synset_id":"team.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"team%1:14:00::","call_ids":["semeval2013.d010.s012.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"team%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":619,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001358,"input_uncached_usd":0.001238,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9511364579666406}
{"item_id":"semeval2013.d010.s012.t003","gold_sense_keys":["match%1:11:00::"],"candidates":[{"index":1,"sense_key":"match%1:06:00::","synset_id":"match.n.01"},{"index":2,"sense_key":"match%1:11:00::","synset_id":"match.n.02"},{"index":3,"sense_key":"match%1:06:02::","synset_id":"match.n.03"},{"index":4,"sense_key":"match%1:06:01::","synset_id":"match.n.04"},{"index":5,"sense_key":"match%1:23:00::","synset_id":"match.n.05"},{"index":6,"sense_key":"match%1:18:01::","synset_id":"catch.n.03"},{"index":7,"sense_key":"match%1:18:00::","synset_id":"peer.n.01"},{"index":8,"sense_key":"match%1:14:00::","synset_id":"couple.n.02"},{"index":9,"sense_key":"match%1:09:00::","synset_id":"match.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"match%1:11:00::","call_ids":["semeval2013.d010.s012.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"match%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1018,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002156,"input_uncached_usd":0.002036,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9637632499216124}
{"item_id":"semeval2013.d010.s012.t004","gold_sense_keys":["striker%1:18:02::"],"candidates":[{"index":1,"sense_key":"striker%1:18:02::","synset_id":"striker.n.01"},{"index":2,"sense_key":"striker%1:18:01::","synset_id":"striker.n.02"},{"index":3,"sense_key":"striker%1:18:00::","synset_id":"striker.n.03"},{"index":4,"sense_key":"striker%1:18:03::","synset_id":"hitter.n.02"},{"index":5,"sense_key":"striker%1:06:00::","synset_id":"striker.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"striker%1:18:02::","call_ids":["semeval2013.d010.s012.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"striker%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":737,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001594,"input_uncached_usd":0.0014739999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.992063874960877}
{"item_id":"semeval2013.d010.s013.t000","gold_sense_keys":["number%1:07:00::"],"candidates":[{"index":1,"sense_key":"number%1:07:00::","synset_id":"number.n.01"},{"index":2,"sense_key":"number%1:23:00::","synset_id":"number.n.02"},{"index":3,"sense_key":"number%1:10:04::","synset_id":"act.n.04"},{"index":4,"sense_key":"number%1:10:03::","synset_id":"phone_number.n.01"},{"index":5,"sense_key":"number%1:10:01::","synset_id":"numeral.n.01"},{"index":6,"sense_key":"number%1:10:00::","synset_id":"issue.n.02"},{"index":7,"sense_key":"number%1:14:00::","synset_id":"number.n.07"},{"index":8,"sense_key":"number%1:10:02::","synset_id":"number.n.08"},{"index":9,"sense_key":"number%1:07:01::","synset_id":"number.n.09"},{"index":10,"sense_key":"number%1:10:05::","synset_id":"number.n.10"},{"index":11,"sense_key":"number%1:06:00::","synset_id":"number.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"number%1:23:00::","call_ids":["semeval2013.d010.s013.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"number%1:23:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1259,"cached_input_tokens":0,"output_tokens":224,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004758,"input_uncached_usd":0.002518,"input_cached_usd":0.0,"output_usd":0.0022400000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.444716583006084}
{"item_id":"semeval2013.d010.s013.t001","gold_sense_keys":["game%1:04:03::"],"candidates":[{"index":1,"sense_key":"game%1:04:00::","synset_id":"game.n.01"},{"index":2,"sense_key":"game%1:04:03::","synset_id":"game.n.02"},{"index":3,"sense_key":"game%1:04:01::","synset_id":"game.n.03"},{"index":4,"sense_key":"game%1:05:00::","synset_id":"game.n.04"},{"index":5,"sense_key":"game%1:28:00::","synset_id":"game.n.05"},{"index":6,"sense_key":"game%1:23:00::","synset_id":"game.n.06"},{"index":7,"sense_key":"game%1:13:00::","synset_id":"game.n.07"},{"index":8,"sense_key":"game%1:09:00::","synset_id":"plot.n.01"},{"index":9,"sense_key":"game%1:06:00::","synset_id":"game.n.09"},{"index":10,"sense_key":"game%1:04:02::","synset_id":"game.n.10"},{"index":11,"sense_key":"game%1:04:04::","synset_id":"game.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"game%1:04:03::","call_ids":["semeval2013.d010.s013.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"game%1:04:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1143,"cached_input_tokens":0,"output_tokens":80,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0030859999999999998,"input_uncached_usd":0.002286,"input_cached_usd":0.0,"output_usd":0.0008,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0377547499956563}
{"item_id":"semeval2013.d010.s015.t000","gold_sense_keys":["side%1:14:00::"],"candidates":[{"index":1,"sense_key":"side%1:15:02::","synset_id":"side.n.01"},{"index":2,"sense_key":"side%1:14:00::","synset_id":"side.n.02"},{"index":3,"sense_key":"side%1:08:00::","synset_id":"side.n.03"},{"index":4,"sense_key":"side%1:15:00::","synset_id":"side.n.04"},{"index":5,"sense_key":"side%1:06:00::","synset_id":"side.n.05"},{"index":6,"sense_key":"side%1:09:00::","synset_id":"side.n.06"},{"index":7,"sense_key":"side%1:15:01::","synset_id":"side.n.07"},{"index":8,"sense_key":"side%1:14:01::","synset_id":"side.n.08"},{"index":9,"sense_key":"side%1:13:00::","synset_id":"side.n.09"},{"index":10,"sense_key":"side%1:10:00::","synset_id":"side.n.10"},{"index":11,"sense_key":"side%1:17:01::","synset_id":"slope.n.01"},{"index":12,"sense_key":"side%1:11:00::","synset_id":"english.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"side%1:14:00::","call_ids":["semeval2013.d010.s015.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"side%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1286,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003232,"input_uncached_usd":0.002572,"input_cached_usd":0.0,"output_usd":0.0006600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.23593104199972}
{"item_id":"semeval2013.d010.s015.t001","gold_sense_keys":["match%1:11:00::"],"candidates":[{"index":1,"sense_key":"match%1:06:00::","synset_id":"match.n.01"},{"index":2,"sense_key":"match%1:11:00::","synset_id":"match.n.02"},{"index":3,"sense_key":"match%1:06:02::","synset_id":"match.n.03"},{"index":4,"sense_key":"match%1:06:01::","synset_id":"match.n.04"},{"index":5,"sense_key":"match%1:23:00::","synset_id":"match.n.05"},{"index":6,"sense_key":"match%1:18:01::","synset_id":"catch.n.03"},{"index":7,"sense_key":"match%1:18:00::","synset_id":"peer.n.01"},{"index":8,"sense_key":"match%1:14:00::","synset_id":"couple.n.02"},{"index":9,"sense_key":"match%1:09:00::","synset_id":"match.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"match%1:11:00::","call_ids":["semeval2013.d010.s015.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"match%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1075,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00227,"input_uncached_usd":0.00215,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0637472920352593}
{"item_id":"semeval2013.d010.s015.t002","gold_sense_keys":["striker%1:18:02::"],"candidates":[{"index":1,"sense_key":"striker%1:18:02::","synset_id":"striker.n.01"},{"index":2,"sense_key":"striker%1:18:01::","synset_id":"striker.n.02"},{"index":3,"sense_key":"striker%1:18:00::","synset_id":"striker.n.03"},{"index":4,"sense_key":"striker%1:18:03::","synset_id":"hitter.n.02"},{"index":5,"sense_key":"striker%1:06:00::","synset_id":"striker.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"striker%1:18:02::","call_ids":["semeval2013.d010.s015.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"striker%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":794,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001708,"input_uncached_usd":0.001588,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":14.137690291972831}
{"item_id":"semeval2013.d010.s016.t000","gold_sense_keys":["game%1:04:03::"],"candidates":[{"index":1,"sense_key":"game%1:04:00::","synset_id":"game.n.01"},{"index":2,"sense_key":"game%1:04:03::","synset_id":"game.n.02"},{"index":3,"sense_key":"game%1:04:01::","synset_id":"game.n.03"},{"index":4,"sense_key":"game%1:05:00::","synset_id":"game.n.04"},{"index":5,"sense_key":"game%1:28:00::","synset_id":"game.n.05"},{"index":6,"sense_key":"game%1:23:00::","synset_id":"game.n.06"},{"index":7,"sense_key":"game%1:13:00::","synset_id":"game.n.07"},{"index":8,"sense_key":"game%1:09:00::","synset_id":"plot.n.01"},{"index":9,"sense_key":"game%1:06:00::","synset_id":"game.n.09"},{"index":10,"sense_key":"game%1:04:02::","synset_id":"game.n.10"},{"index":11,"sense_key":"game%1:04:04::","synset_id":"game.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"game%1:04:03::","call_ids":["semeval2013.d010.s016.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"game%1:04:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1122,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002864,"input_uncached_usd":0.002244,"input_cached_usd":0.0,"output_usd":0.00062,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5297701250528917}
{"item_id":"semeval2013.d010.s018.t000","gold_sense_keys":["match%1:11:00::"],"candidates":[{"index":1,"sense_key":"match%1:06:00::","synset_id":"match.n.01"},{"index":2,"sense_key":"match%1:11:00::","synset_id":"match.n.02"},{"index":3,"sense_key":"match%1:06:02::","synset_id":"match.n.03"},{"index":4,"sense_key":"match%1:06:01::","synset_id":"match.n.04"},{"index":5,"sense_key":"match%1:23:00::","synset_id":"match.n.05"},{"index":6,"sense_key":"match%1:18:01::","synset_id":"catch.n.03"},{"index":7,"sense_key":"match%1:18:00::","synset_id":"peer.n.01"},{"index":8,"sense_key":"match%1:14:00::","synset_id":"couple.n.02"},{"index":9,"sense_key":"match%1:09:00::","synset_id":"match.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"match%1:11:00::","call_ids":["semeval2013.d010.s018.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"match%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1088,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002296,"input_uncached_usd":0.002176,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4723075419897214}
{"item_id":"semeval2013.d010.s019.t000","gold_sense_keys":["side%1:14:00::"],"candidates":[{"index":1,"sense_key":"side%1:15:02::","synset_id":"side.n.01"},{"index":2,"sense_key":"side%1:14:00::","synset_id":"side.n.02"},{"index":3,"sense_key":"side%1:08:00::","synset_id":"side.n.03"},{"index":4,"sense_key":"side%1:15:00::","synset_id":"side.n.04"},{"index":5,"sense_key":"side%1:06:00::","synset_id":"side.n.05"},{"index":6,"sense_key":"side%1:09:00::","synset_id":"side.n.06"},{"index":7,"sense_key":"side%1:15:01::","synset_id":"side.n.07"},{"index":8,"sense_key":"side%1:14:01::","synset_id":"side.n.08"},{"index":9,"sense_key":"side%1:13:00::","synset_id":"side.n.09"},{"index":10,"sense_key":"side%1:10:00::","synset_id":"side.n.10"},{"index":11,"sense_key":"side%1:17:01::","synset_id":"slope.n.01"},{"index":12,"sense_key":"side%1:11:00::","synset_id":"english.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"side%1:14:00::","call_ids":["semeval2013.d010.s019.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"side%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1288,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003146,"input_uncached_usd":0.0025759999999999997,"input_cached_usd":0.0,"output_usd":0.0005700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":10.825888207997195}
{"item_id":"semeval2013.d010.s019.t001","gold_sense_keys":["presence%1:04:00::"],"candidates":[{"index":1,"sense_key":"presence%1:26:00::","synset_id":"presence.n.01"},{"index":2,"sense_key":"presence%1:15:00::","synset_id":"presence.n.02"},{"index":3,"sense_key":"presence%1:18:00::","synset_id":"presence.n.03"},{"index":4,"sense_key":"presence%1:09:00::","synset_id":"presence.n.04"},{"index":5,"sense_key":"presence%1:07:00::","synset_id":"bearing.n.03"},{"index":6,"sense_key":"presence%1:04:00::","synset_id":"presence.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"presence%1:04:00::","call_ids":["semeval2013.d010.s019.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"presence%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":867,"cached_input_tokens":0,"output_tokens":123,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002964,"input_uncached_usd":0.0017339999999999999,"input_cached_usd":0.0,"output_usd":0.0012300000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9255700419889763}
{"item_id":"semeval2013.d010.s020.t000","gold_sense_keys":["rest%1:24:00::"],"candidates":[{"index":1,"sense_key":"rest%1:24:00::","synset_id":"remainder.n.01"},{"index":2,"sense_key":"rest%1:04:00::","synset_id":"rest.n.02"},{"index":3,"sense_key":"rest%1:28:00::","synset_id":"respite.n.04"},{"index":4,"sense_key":"rest%1:26:00::","synset_id":"rest.n.04"},{"index":5,"sense_key":"rest%1:26:01::","synset_id":"rest.n.05"},{"index":6,"sense_key":"rest%1:06:00::","synset_id":"rest.n.06"},{"index":7,"sense_key":"rest%1:10:00::","synset_id":"rest.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"rest%1:24:00::","call_ids":["semeval2013.d010.s020.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"rest%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1028,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002176,"input_uncached_usd":0.002056,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0528709590435028}
{"item_id":"semeval2013.d010.s021.t000","gold_sense_keys":["game%1:04:03::"],"candidates":[{"index":1,"sense_key":"game%1:04:00::","synset_id":"game.n.01"},{"index":2,"sense_key":"game%1:04:03::","synset_id":"game.n.02"},{"index":3,"sense_key":"game%1:04:01::","synset_id":"game.n.03"},{"index":4,"sense_key":"game%1:05:00::","synset_id":"game.n.04"},{"index":5,"sense_key":"game%1:28:00::","synset_id":"game.n.05"},{"index":6,"sense_key":"game%1:23:00::","synset_id":"game.n.06"},{"index":7,"sense_key":"game%1:13:00::","synset_id":"game.n.07"},{"index":8,"sense_key":"game%1:09:00::","synset_id":"plot.n.01"},{"index":9,"sense_key":"game%1:06:00::","synset_id":"game.n.09"},{"index":10,"sense_key":"game%1:04:02::","synset_id":"game.n.10"},{"index":11,"sense_key":"game%1:04:04::","synset_id":"game.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"game%1:04:03::","call_ids":["semeval2013.d010.s021.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"game%1:04:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1114,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002828,"input_uncached_usd":0.002228,"input_cached_usd":0.0,"output_usd":0.0006000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2354744169861078}
{"item_id":"semeval2013.d010.s022.t000","gold_sense_keys":["scorer%1:18:00::"],"candidates":[{"index":1,"sense_key":"scorer%1:18:00::","synset_id":"scorer.n.01"},{"index":2,"sense_key":"scorer%1:18:01::","synset_id":"scorer.n.02"},{"index":3,"sense_key":"scorer%1:18:02::","synset_id":"scorekeeper.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"scorer%1:18:00::","call_ids":["semeval2013.d010.s022.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"scorer%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":658,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001436,"input_uncached_usd":0.001316,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4977052500471473}
{"item_id":"semeval2013.d010.s022.t001","gold_sense_keys":["goal%1:04:00::"],"candidates":[{"index":1,"sense_key":"goal%1:09:00::","synset_id":"goal.n.01"},{"index":2,"sense_key":"goal%1:15:00::","synset_id":"finish.n.04"},{"index":3,"sense_key":"goal%1:06:00::","synset_id":"goal.n.03"},{"index":4,"sense_key":"goal%1:04:00::","synset_id":"goal.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"goal%1:04:00::","call_ids":["semeval2013.d010.s022.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"goal%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":801,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001722,"input_uncached_usd":0.001602,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2936814170097932}
{"item_id":"semeval2013.d010.s023.t000","gold_sense_keys":["point%1:23:01::"],"candidates":[{"index":1,"sense_key":"point%1:09:00::","synset_id":"point.n.01"},{"index":2,"sense_key":"point%1:15:00::","synset_id":"point.n.02"},{"index":3,"sense_key":"point%1:10:01::","synset_id":"point.n.03"},{"index":4,"sense_key":"point%1:09:01::","synset_id":"detail.n.01"},{"index":5,"sense_key":"point%1:26:00::","synset_id":"degree.n.02"},{"index":6,"sense_key":"point%1:28:00::","synset_id":"point.n.06"},{"index":7,"sense_key":"point%1:09:02::","synset_id":"point.n.07"},{"index":8,"sense_key":"point%1:25:00::","synset_id":"point.n.08"},{"index":9,"sense_key":"point%1:25:02::","synset_id":"point.n.09"},{"index":10,"sense_key":"point%1:23:01::","synset_id":"point.n.10"},{"index":11,"sense_key":"point%1:17:02::","synset_id":"point.n.11"},{"index":12,"sense_key":"point%1:10:03::","synset_id":"item.n.01"},{"index":13,"sense_key":"point%1:24:04::","synset_id":"point.n.13"},{"index":14,"sense_key":"point%1:07:01::","synset_id":"point.n.14"},{"index":15,"sense_key":"point%1:06:00::","synset_id":"point.n.15"},{"index":16,"sense_key":"point%1:24:00::","synset_id":"compass_point.n.01"},{"index":17,"sense_key":"point%1:23:00::","synset_id":"point.n.17"},{"index":18,"sense_key":"point%1:21:00::","synset_id":"point.n.18"},{"index":19,"sense_key":"point%1:10:00::","synset_id":"period.n.07"},{"index":20,"sense_key":"point%1:10:02::","synset_id":"point.n.20"},{"index":21,"sense_key":"point%1:10:04::","synset_id":"decimal_point.n.01"},{"index":22,"sense_key":"point%1:07:00::","synset_id":"point.n.22"},{"index":23,"sense_key":"point%1:07:02::","synset_id":"point.n.23"},{"index":24,"sense_key":"point%1:06:02::","synset_id":"point.n.24"},{"index":25,"sense_key":"point%1:06:01::","synset_id":"point.n.25"},{"index":26,"sense_key":"point%1:06:03::","synset_id":"distributor_point.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"point%1:23:01::","call_ids":["semeval2013.d010.s023.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"point%1:23:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2011,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004621999999999999,"input_uncached_usd":0.0040219999999999995,"input_cached_usd":0.0,"output_usd":0.0006000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":10.599668667069636}
{"item_id":"semeval2013.d011.s000.t001","gold_sense_keys":["burden%1:09:01::"],"candidates":[{"index":1,"sense_key":"burden%1:09:01::","synset_id":"burden.n.01"},{"index":2,"sense_key":"burden%1:06:01::","synset_id":"load.n.01"},{"index":3,"sense_key":"burden%1:10:00::","synset_id":"effect.n.04"},{"index":4,"sense_key":"burden%1:09:00::","synset_id":"burden.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"burden%1:09:01::","call_ids":["semeval2013.d011.s000.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"burden%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":544,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001208,"input_uncached_usd":0.001088,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.7572613339871168}
{"item_id":"semeval2013.d011.s000.t003","gold_sense_keys":["economy%1:14:00::"],"candidates":[{"index":1,"sense_key":"economy%1:14:00::","synset_id":"economy.n.01"},{"index":2,"sense_key":"economy%1:09:01::","synset_id":"economy.n.02"},{"index":3,"sense_key":"economy%1:07:00::","synset_id":"economy.n.03"},{"index":4,"sense_key":"economy%1:04:00::","synset_id":"economy.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"economy%1:14:00::","call_ids":["semeval2013.d011.s000.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"economy%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":550,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00122,"input_uncached_usd":0.0010999999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.7354132919572294}
{"item_id":"semeval2013.d011.s001.t000","gold_sense_keys":["immigration%1:04:00::"],"candidates":[{"index":1,"sense_key":"immigration%1:04:00::","synset_id":"immigration.n.01"},{"index":2,"sense_key":"immigration%1:14:00::","synset_id":"immigration.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"immigration%1:04:00::","call_ids":["semeval2013.d011.s001.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"immigration%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":521,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001162,"input_uncached_usd":0.001042,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1955019999295473}
{"item_id":"semeval2013.d011.s001.t001","gold_sense_keys":["policy%1:09:00::"],"candidates":[{"index":1,"sense_key":"policy%1:09:00::","synset_id":"policy.n.01"},{"index":2,"sense_key":"policy%1:10:00::","synset_id":"policy.n.02"},{"index":3,"sense_key":"policy%1:10:02::","synset_id":"policy.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"policy%1:09:00::","call_ids":["semeval2013.d011.s001.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"policy%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":589,"cached_input_tokens":0,"output_tokens":106,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0022380000000000004,"input_uncached_usd":0.001178,"input_cached_usd":0.0,"output_usd":0.0010600000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":15.043231667019427}
{"item_id":"semeval2013.d011.s001.t002","gold_sense_keys":["aspect%1:09:00::"],"candidates":[{"index":1,"sense_key":"aspect%1:09:00::","synset_id":"aspect.n.01"},{"index":2,"sense_key":"aspect%1:07:02::","synset_id":"aspect.n.02"},{"index":3,"sense_key":"aspect%1:09:01::","synset_id":"view.n.02"},{"index":4,"sense_key":"aspect%1:24:00::","synset_id":"aspect.n.04"},{"index":5,"sense_key":"aspect%1:07:01::","synset_id":"expression.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"aspect%1:09:00::","call_ids":["semeval2013.d011.s001.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"aspect%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":677,"cached_input_tokens":0,"output_tokens":123,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0025840000000000004,"input_uncached_usd":0.001354,"input_cached_usd":0.0,"output_usd":0.0012300000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2766726249828935}
{"item_id":"semeval2013.d011.s001.t003","gold_sense_keys":["congestion%1:26:01::"],"candidates":[{"index":1,"sense_key":"congestion%1:26:00::","synset_id":"congestion.n.01"},{"index":2,"sense_key":"congestion%1:26:01::","synset_id":"congestion.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"congestion%1:26:01::","call_ids":["semeval2013.d011.s001.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"congestion%1:26:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":496,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0011120000000000001,"input_uncached_usd":0.000992,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.7543728750897571}
{"item_id":"semeval2013.d011.s001.t006","gold_sense_keys":["policy%1:09:00::"],"candidates":[{"index":1,"sense_key":"policy%1:09:00::","synset_id":"policy.n.01"},{"index":2,"sense_key":"policy%1:10:00::","synset_id":"policy.n.02"},{"index":3,"sense_key":"policy%1:10:02::","synset_id":"policy.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"policy%1:09:00::","call_ids":["semeval2013.d011.s001.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"policy%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":589,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0012980000000000001,"input_uncached_usd":0.001178,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.050644624978304}
{"item_id":"semeval2013.d011.s001.t007","gold_sense_keys":["number%1:07:00::"],"candidates":[{"index":1,"sense_key":"number%1:07:00::","synset_id":"number.n.01"},{"index":2,"sense_key":"number%1:23:00::","synset_id":"number.n.02"},{"index":3,"sense_key":"number%1:10:04::","synset_id":"act.n.04"},{"index":4,"sense_key":"number%1:10:03::","synset_id":"phone_number.n.01"},{"index":5,"sense_key":"number%1:10:01::","synset_id":"numeral.n.01"},{"index":6,"sense_key":"number%1:10:00::","synset_id":"issue.n.02"},{"index":7,"sense_key":"number%1:14:00::","synset_id":"number.n.07"},{"index":8,"sense_key":"number%1:10:02::","synset_id":"number.n.08"},{"index":9,"sense_key":"number%1:07:01::","synset_id":"number.n.09"},{"index":10,"sense_key":"number%1:10:05::","synset_id":"number.n.10"},{"index":11,"sense_key":"number%1:06:00::","synset_id":"number.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"number%1:07:00::","call_ids":["semeval2013.d011.s001.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"number%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1072,"cached_input_tokens":0,"output_tokens":78,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0029240000000000004,"input_uncached_usd":0.002144,"input_cached_usd":0.0,"output_usd":0.0007800000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6081742909736931}
{"item_id":"semeval2013.d011.s001.t008","gold_sense_keys":["compatibility%1:07:00::"],"candidates":[{"index":1,"sense_key":"compatibility%1:12:00::","synset_id":"compatibility.n.01"},{"index":2,"sense_key":"compatibility%1:07:00::","synset_id":"compatibility.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"compatibility%1:07:00::","call_ids":["semeval2013.d011.s001.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"compatibility%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":486,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001092,"input_uncached_usd":0.000972,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4042762910248712}
{"item_id":"semeval2013.d011.s001.t009","gold_sense_keys":["policy%1:09:00::"],"candidates":[{"index":1,"sense_key":"policy%1:09:00::","synset_id":"policy.n.01"},{"index":2,"sense_key":"policy%1:10:00::","synset_id":"policy.n.02"},{"index":3,"sense_key":"policy%1:10:02::","synset_id":"policy.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"policy%1:09:00::","call_ids":["semeval2013.d011.s001.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"policy%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":589,"cached_input_tokens":0,"output_tokens":104,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0022180000000000004,"input_uncached_usd":0.001178,"input_cached_usd":0.0,"output_usd":0.0010400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.938964999979362}
{"item_id":"semeval2013.d011.s001.t010","gold_sense_keys":["status%1:26:00::"],"candidates":[{"index":1,"sense_key":"status%1:26:00::","synset_id":"status.n.01"},{"index":2,"sense_key":"status%1:26:01::","synset_id":"condition.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"status%1:26:00::","call_ids":["semeval2013.d011.s001.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"status%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":533,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001186,"input_uncached_usd":0.0010659999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9622284579090774}
{"item_id":"semeval2013.d011.s001.t011","gold_sense_keys":["country%1:14:00::"],"candidates":[{"index":1,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":2,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":3,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":4,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":5,"sense_key":"country%1:15:02::","synset_id":"area.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"country%1:14:00::","call_ids":["semeval2013.d011.s001.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"country%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":752,"cached_input_tokens":0,"output_tokens":138,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0028840000000000003,"input_uncached_usd":0.001504,"input_cached_usd":0.0,"output_usd":0.0013800000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8563004169845954}
{"item_id":"semeval2013.d011.s001.t012","gold_sense_keys":["country%1:14:00::"],"candidates":[{"index":1,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":2,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":3,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":4,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":5,"sense_key":"country%1:15:02::","synset_id":"area.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"country%1:14:00::","call_ids":["semeval2013.d011.s001.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"country%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":752,"cached_input_tokens":0,"output_tokens":106,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0025640000000000003,"input_uncached_usd":0.001504,"input_cached_usd":0.0,"output_usd":0.0010600000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.019813416991383}
{"item_id":"semeval2013.d011.s002.t000","gold_sense_keys":["harassment%1:04:00::"],"candidates":[{"index":1,"sense_key":"harassment%1:12:00::","synset_id":"harassment.n.01"},{"index":2,"sense_key":"harassment%1:04:00::","synset_id":"harassment.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"harassment%1:04:00::","call_ids":["semeval2013.d011.s002.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"harassment%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":551,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001742,"input_uncached_usd":0.001102,"input_cached_usd":0.0,"output_usd":0.00064,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.426015374949202}
{"item_id":"semeval2013.d011.s002.t001","gold_sense_keys":["student%1:18:00::"],"candidates":[{"index":1,"sense_key":"student%1:18:00::","synset_id":"student.n.01"},{"index":2,"sense_key":"student%1:18:01::","synset_id":"scholar.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"student%1:18:00::","call_ids":["semeval2013.d011.s002.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"student%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":543,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001206,"input_uncached_usd":0.001086,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8648506249301136}
{"item_id":"semeval2013.d011.s002.t002","gold_sense_keys":["france%1:15:00::"],"candidates":[{"index":1,"sense_key":"france%1:15:00::","synset_id":"france.n.01"},{"index":2,"sense_key":"france%1:18:00::","synset_id":"france.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"france%1:15:00::","call_ids":["semeval2013.d011.s002.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"france%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":542,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001204,"input_uncached_usd":0.001084,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3637187919812277}
{"item_id":"semeval2013.d011.s002.t003","gold_sense_keys":["talent%1:18:00::"],"candidates":[{"index":1,"sense_key":"talent%1:09:00::","synset_id":"endowment.n.01"},{"index":2,"sense_key":"talent%1:18:00::","synset_id":"talent.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"talent%1:18:00::","call_ids":["semeval2013.d011.s002.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"talent%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":513,"cached_input_tokens":0,"output_tokens":76,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001786,"input_uncached_usd":0.001026,"input_cached_usd":0.0,"output_usd":0.00076,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4600395420566201}
{"item_id":"semeval2013.d011.s002.t005","gold_sense_keys":["waiter%1:18:00::"],"candidates":[{"index":1,"sense_key":"waiter%1:18:00::","synset_id":"waiter.n.01"},{"index":2,"sense_key":"waiter%1:18:01::","synset_id":"waiter.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"waiter%1:18:00::","call_ids":["semeval2013.d011.s002.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"waiter%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":509,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0011380000000000001,"input_uncached_usd":0.001018,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.698778000078164}
{"item_id":"semeval2013.d011.s003.t000","gold_sense_keys":["viewpoint%1:09:00::"],"candidates":[{"index":1,"sense_key":"viewpoint%1:09:00::","synset_id":"point_of_view.n.01"},{"index":2,"sense_key":"viewpoint%1:15:00::","synset_id":"vantage_point.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"viewpoint%1:09:00::","call_ids":["semeval2013.d011.s003.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"viewpoint%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":608,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001336,"input_uncached_usd":0.0012159999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0648132920032367}
{"item_id":"semeval2013.d011.s004.t000","gold_sense_keys":["lot%1:23:00::"],"candidates":[{"index":1,"sense_key":"lot%1:23:00::","synset_id":"batch.n.02"},{"index":2,"sense_key":"lot%1:15:00::","synset_id":"lot.n.02"},{"index":3,"sense_key":"lot%1:14:00::","synset_id":"set.n.05"},{"index":4,"sense_key":"lot%1:26:00::","synset_id":"fortune.n.04"},{"index":5,"sense_key":"lot%1:06:00::","synset_id":"draw.n.04"},{"index":6,"sense_key":"lot%1:14:01::","synset_id":"bunch.n.03"},{"index":7,"sense_key":"lot%1:18:00::","synset_id":"lot.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"lot%1:23:00::","call_ids":["semeval2013.d011.s004.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"lot%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1040,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0021999999999999997,"input_uncached_usd":0.00208,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9740879579912871}
{"item_id":"semeval2013.d011.s004.t002","gold_sense_keys":["left%1:14:00::"],"candidates":[{"index":1,"sense_key":"left%1:15:00::","synset_id":"left.n.01"},{"index":2,"sense_key":"left%1:14:00::","synset_id":"left.n.02"},{"index":3,"sense_key":"left%1:08:00::","synset_id":"left.n.03"},{"index":4,"sense_key":"left%1:06:00::","synset_id":"left_field.n.01"},{"index":5,"sense_key":"left%1:04:00::","synset_id":"left.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"left%1:14:00::","call_ids":["semeval2013.d011.s004.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"left%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":819,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001758,"input_uncached_usd":0.001638,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0470817079767585}
{"item_id":"semeval2013.d011.s004.t003","gold_sense_keys":["right%1:14:00::"],"candidates":[{"index":1,"sense_key":"right%1:07:00::","synset_id":"right.n.01"},{"index":2,"sense_key":"right%1:15:00::","synset_id":"right.n.02"},{"index":3,"sense_key":"right%1:06:00::","synset_id":"right_field.n.01"},{"index":4,"sense_key":"right%1:14:00::","synset_id":"right.n.04"},{"index":5,"sense_key":"right%1:08:00::","synset_id":"right.n.05"},{"index":6,"sense_key":"right%1:04:00::","synset_id":"right.n.06"},{"index":7,"sense_key":"right%1:07:01::","synset_id":"right.n.07"},{"index":8,"sense_key":"right%1:21:00::","synset_id":"right.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"right%1:14:00::","call_ids":["semeval2013.d011.s004.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"right%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1069,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0022579999999999996,"input_uncached_usd":0.0021379999999999997,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":4.714249084005132}
{"item_id":"semeval2013.d011.s004.t004","gold_sense_keys":["subject%1:09:01::","subject%1:10:00::"],"candidates":[{"index":1,"sense_key":"subject%1:10:00::","synset_id":"subject.n.01"},{"index":2,"sense_key":"subject%1:06:00::","synset_id":"subject.n.02"},{"index":3,"sense_key":"subject%1:09:00::","synset_id":"discipline.n.01"},{"index":4,"sense_key":"subject%1:09:01::","synset_id":"topic.n.02"},{"index":5,"sense_key":"subject%1:10:01::","synset_id":"subject.n.05"},{"index":6,"sense_key":"subject%1:18:01::","synset_id":"subject.n.06"},{"index":7,"sense_key":"subject%1:18:00::","synset_id":"national.n.01"},{"index":8,"sense_key":"subject%1:10:02::","synset_id":"subject.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"subject%1:10:00::","call_ids":["semeval2013.d011.s004.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"subject%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1110,"cached_input_tokens":0,"output_tokens":167,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00389,"input_uncached_usd":0.0022199999999999998,"input_cached_usd":0.0,"output_usd":0.00167,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.614249833044596}
{"item_id":"semeval2013.d011.s005.t000","gold_sense_keys":["right%1:14:00::"],"candidates":[{"index":1,"sense_key":"right%1:07:00::","synset_id":"right.n.01"},{"index":2,"sense_key":"right%1:15:00::","synset_id":"right.n.02"},{"index":3,"sense_key":"right%1:06:00::","synset_id":"right_field.n.01"},{"index":4,"sense_key":"right%1:14:00::","synset_id":"right.n.04"},{"index":5,"sense_key":"right%1:08:00::","synset_id":"right.n.05"},{"index":6,"sense_key":"right%1:04:00::","synset_id":"right.n.06"},{"index":7,"sense_key":"right%1:07:01::","synset_id":"right.n.07"},{"index":8,"sense_key":"right%1:21:00::","synset_id":"right.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"right%1:14:00::","call_ids":["semeval2013.d011.s005.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"right%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1081,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0022819999999999997,"input_uncached_usd":0.002162,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0493637919425964}
{"item_id":"semeval2013.d011.s005.t001","gold_sense_keys":["tone%1:07:03::"],"candidates":[{"index":1,"sense_key":"tone%1:10:01::","synset_id":"tone.n.01"},{"index":2,"sense_key":"tone%1:07:02::","synset_id":"tone.n.02"},{"index":3,"sense_key":"tone%1:07:01::","synset_id":"timbre.n.01"},{"index":4,"sense_key":"tone%1:26:01::","synset_id":"spirit.n.02"},{"index":5,"sense_key":"tone%1:07:00::","synset_id":"shade.n.02"},{"index":6,"sense_key":"tone%1:10:02::","synset_id":"note.n.03"},{"index":7,"sense_key":"tone%1:09:00::","synset_id":"tone.n.07"},{"index":8,"sense_key":"tone%1:26:02::","synset_id":"tonicity.n.01"},{"index":9,"sense_key":"tone%1:10:00::","synset_id":"tone.n.09"},{"index":10,"sense_key":"tone%1:07:03::","synset_id":"tone.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"tone%1:26:01::","call_ids":["semeval2013.d011.s005.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"tone%1:26:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1342,"cached_input_tokens":0,"output_tokens":153,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004214,"input_uncached_usd":0.0026839999999999998,"input_cached_usd":0.0,"output_usd":0.0015300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8949349159374833}
{"item_id":"semeval2013.d011.s005.t003","gold_sense_keys":["problem%1:09:00::","problem%1:26:00::"],"candidates":[{"index":1,"sense_key":"problem%1:26:00::","synset_id":"problem.n.01"},{"index":2,"sense_key":"problem%1:10:00::","synset_id":"problem.n.02"},{"index":3,"sense_key":"problem%1:09:00::","synset_id":"trouble.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"problem%1:09:00::","call_ids":["semeval2013.d011.s005.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"problem%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":731,"cached_input_tokens":0,"output_tokens":115,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002612,"input_uncached_usd":0.001462,"input_cached_usd":0.0,"output_usd":0.0011500000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":5.143583458964713}
{"item_id":"semeval2013.d011.s005.t004","gold_sense_keys":["country%1:14:00::","country%1:15:00::"],"candidates":[{"index":1,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":2,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":3,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":4,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":5,"sense_key":"country%1:15:02::","synset_id":"area.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"country%1:14:00::","call_ids":["semeval2013.d011.s005.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"country%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":903,"cached_input_tokens":0,"output_tokens":96,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0027660000000000002,"input_uncached_usd":0.0018059999999999999,"input_cached_usd":0.0,"output_usd":0.0009600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4862262500682846}
{"item_id":"semeval2013.d011.s005.t005","gold_sense_keys":["worker%1:18:00::"],"candidates":[{"index":1,"sense_key":"worker%1:18:00::","synset_id":"worker.n.01"},{"index":2,"sense_key":"worker%1:18:01::","synset_id":"proletarian.n.01"},{"index":3,"sense_key":"worker%1:05:01::","synset_id":"worker.n.03"},{"index":4,"sense_key":"worker%1:18:02::","synset_id":"actor.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"worker%1:18:00::","call_ids":["semeval2013.d011.s005.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"worker%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":774,"cached_input_tokens":0,"output_tokens":109,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0026379999999999997,"input_uncached_usd":0.001548,"input_cached_usd":0.0,"output_usd":0.00109,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.836662375018932}
{"item_id":"semeval2013.d011.s005.t006","gold_sense_keys":["left%1:14:00::"],"candidates":[{"index":1,"sense_key":"left%1:15:00::","synset_id":"left.n.01"},{"index":2,"sense_key":"left%1:14:00::","synset_id":"left.n.02"},{"index":3,"sense_key":"left%1:08:00::","synset_id":"left.n.03"},{"index":4,"sense_key":"left%1:06:00::","synset_id":"left_field.n.01"},{"index":5,"sense_key":"left%1:04:00::","synset_id":"left.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"left%1:14:00::","call_ids":["semeval2013.d011.s005.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"left%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":831,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001782,"input_uncached_usd":0.0016619999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5021960840094835}
{"item_id":"semeval2013.d011.s005.t007","gold_sense_keys":["france%1:15:00::"],"candidates":[{"index":1,"sense_key":"france%1:15:00::","synset_id":"france.n.01"},{"index":2,"sense_key":"france%1:18:00::","synset_id":"france.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"france%1:15:00::","call_ids":["semeval2013.d011.s005.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"france%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":672,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001464,"input_uncached_usd":0.001344,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4955698340199888}
{"item_id":"semeval2013.d011.s005.t009","gold_sense_keys":["misery%1:26:00::"],"candidates":[{"index":1,"sense_key":"misery%1:26:00::","synset_id":"misery.n.01"},{"index":2,"sense_key":"misery%1:12:00::","synset_id":"misery.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"misery%1:26:00::","call_ids":["semeval2013.d011.s005.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"misery%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":681,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002022,"input_uncached_usd":0.001362,"input_cached_usd":0.0,"output_usd":0.0006600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":5.069295875029638}
{"item_id":"semeval2013.d011.s006.t000","gold_sense_keys":["difference%1:07:00::"],"candidates":[{"index":1,"sense_key":"difference%1:07:00::","synset_id":"difference.n.01"},{"index":2,"sense_key":"difference%1:11:00::","synset_id":"deviation.n.01"},{"index":3,"sense_key":"difference%1:10:00::","synset_id":"dispute.n.01"},{"index":4,"sense_key":"difference%1:24:00::","synset_id":"difference.n.04"},{"index":5,"sense_key":"difference%1:23:00::","synset_id":"remainder.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"difference%1:07:00::","call_ids":["semeval2013.d011.s006.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"difference%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":878,"cached_input_tokens":0,"output_tokens":89,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002646,"input_uncached_usd":0.001756,"input_cached_usd":0.0,"output_usd":0.0008900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":14.232873750035651}
{"item_id":"semeval2013.d011.s006.t001","gold_sense_keys":["degree%1:07:00::","degree%1:26:01::"],"candidates":[{"index":1,"sense_key":"degree%1:07:00::","synset_id":"degree.n.01"},{"index":2,"sense_key":"degree%1:26:01::","synset_id":"degree.n.02"},{"index":3,"sense_key":"degree%1:10:00::","synset_id":"academic_degree.n.01"},{"index":4,"sense_key":"degree%1:23:00::","synset_id":"degree.n.04"},{"index":5,"sense_key":"degree%1:09:00::","synset_id":"degree.n.05"},{"index":6,"sense_key":"degree%1:23:03::","synset_id":"degree.n.06"},{"index":7,"sense_key":"degree%1:07:01::","synset_id":"degree.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"degree%1:26:01::","call_ids":["semeval2013.d011.s006.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"degree%1:26:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1020,"cached_input_tokens":0,"output_tokens":205,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00409,"input_uncached_usd":0.0020399999999999997,"input_cached_usd":0.0,"output_usd":0.00205,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.122943750000559}
{"item_id":"semeval2013.d011.s006.t002","gold_sense_keys":["generosity%1:07:00::"],"candidates":[{"index":1,"sense_key":"generosity%1:07:00::","synset_id":"generosity.n.01"},{"index":2,"sense_key":"generosity%1:04:00::","synset_id":"generosity.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"generosity%1:07:00::","call_ids":["semeval2013.d011.s006.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"generosity%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":662,"cached_input_tokens":0,"output_tokens":116,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002484,"input_uncached_usd":0.0013239999999999999,"input_cached_usd":0.0,"output_usd":0.00116,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2834876249544322}
{"item_id":"semeval2013.d011.s007.t000","gold_sense_keys":["fear%1:12:01::"],"candidates":[{"index":1,"sense_key":"fear%1:12:00::","synset_id":"fear.n.01"},{"index":2,"sense_key":"fear%1:12:01::","synset_id":"concern.n.02"},{"index":3,"sense_key":"fear%1:12:02::","synset_id":"fear.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"fear%1:12:00::","call_ids":["semeval2013.d011.s007.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"fear%1:12:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":729,"cached_input_tokens":0,"output_tokens":111,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002568,"input_uncached_usd":0.0014579999999999999,"input_cached_usd":0.0,"output_usd":0.00111,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7934252920094877}
{"item_id":"semeval2013.d011.s007.t001","gold_sense_keys":["invasion%1:11:00::"],"candidates":[{"index":1,"sense_key":"invasion%1:04:00::","synset_id":"invasion.n.01"},{"index":2,"sense_key":"invasion%1:11:00::","synset_id":"invasion.n.02"},{"index":3,"sense_key":"invasion%1:11:01::","synset_id":"invasion.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"invasion%1:11:00::","call_ids":["semeval2013.d011.s007.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"invasion%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":708,"cached_input_tokens":0,"output_tokens":89,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002306,"input_uncached_usd":0.001416,"input_cached_usd":0.0,"output_usd":0.0008900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9182292920304462}
{"item_id":"semeval2013.d011.s007.t002","gold_sense_keys":["territory%1:15:01::"],"candidates":[{"index":1,"sense_key":"territory%1:15:00::","synset_id":"district.n.01"},{"index":2,"sense_key":"territory%1:09:00::","synset_id":"territory.n.02"},{"index":3,"sense_key":"territory%1:15:01::","synset_id":"territory.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"territory%1:15:01::","call_ids":["semeval2013.d011.s007.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"territory%1:15:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":685,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0019,"input_uncached_usd":0.00137,"input_cached_usd":0.0,"output_usd":0.0005300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":14.465902625001036}
{"item_id":"semeval2013.d011.s007.t003","gold_sense_keys":["horde%1:14:00::"],"candidates":[{"index":1,"sense_key":"horde%1:14:00::","synset_id":"horde.n.01"},{"index":2,"sense_key":"horde%1:14:02::","synset_id":"horde.n.02"},{"index":3,"sense_key":"horde%1:14:01::","synset_id":"drove.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"horde%1:14:01::","call_ids":["semeval2013.d011.s007.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"horde%1:14:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":623,"cached_input_tokens":0,"output_tokens":127,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002516,"input_uncached_usd":0.001246,"input_cached_usd":0.0,"output_usd":0.00127,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.850637499941513}
{"item_id":"semeval2013.d011.s007.t005","gold_sense_keys":["left%1:14:00::"],"candidates":[{"index":1,"sense_key":"left%1:15:00::","synset_id":"left.n.01"},{"index":2,"sense_key":"left%1:14:00::","synset_id":"left.n.02"},{"index":3,"sense_key":"left%1:08:00::","synset_id":"left.n.03"},{"index":4,"sense_key":"left%1:06:00::","synset_id":"left_field.n.01"},{"index":5,"sense_key":"left%1:04:00::","synset_id":"left.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"left%1:14:00::","call_ids":["semeval2013.d011.s007.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"left%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":794,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001708,"input_uncached_usd":0.001588,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5341076659969985}
{"item_id":"semeval2013.d011.s007.t006","gold_sense_keys":["right%1:14:00::"],"candidates":[{"index":1,"sense_key":"right%1:07:00::","synset_id":"right.n.01"},{"index":2,"sense_key":"right%1:15:00::","synset_id":"right.n.02"},{"index":3,"sense_key":"right%1:06:00::","synset_id":"right_field.n.01"},{"index":4,"sense_key":"right%1:14:00::","synset_id":"right.n.04"},{"index":5,"sense_key":"right%1:08:00::","synset_id":"right.n.05"},{"index":6,"sense_key":"right%1:04:00::","synset_id":"right.n.06"},{"index":7,"sense_key":"right%1:07:01::","synset_id":"right.n.07"},{"index":8,"sense_key":"right%1:21:00::","synset_id":"right.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"right%1:14:00::","call_ids":["semeval2013.d011.s007.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"right%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1044,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002208,"input_uncached_usd":0.002088,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.131512583931908}
{"item_id":"semeval2013.d011.s008.t000","gold_sense_keys":["side%1:14:00::"],"candidates":[{"index":1,"sense_key":"side%1:15:02::","synset_id":"side.n.01"},{"index":2,"sense_key":"side%1:14:00::","synset_id":"side.n.02"},{"index":3,"sense_key":"side%1:08:00::","synset_id":"side.n.03"},{"index":4,"sense_key":"side%1:15:00::","synset_id":"side.n.04"},{"index":5,"sense_key":"side%1:06:00::","synset_id":"side.n.05"},{"index":6,"sense_key":"side%1:09:00::","synset_id":"side.n.06"},{"index":7,"sense_key":"side%1:15:01::","synset_id":"side.n.07"},{"index":8,"sense_key":"side%1:14:01::","synset_id":"side.n.08"},{"index":9,"sense_key":"side%1:13:00::","synset_id":"side.n.09"},{"index":10,"sense_key":"side%1:10:00::","synset_id":"side.n.10"},{"index":11,"sense_key":"side%1:17:01::","synset_id":"slope.n.01"},{"index":12,"sense_key":"side%1:11:00::","synset_id":"english.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"side%1:14:00::","call_ids":["semeval2013.d011.s008.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"side%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1164,"cached_input_tokens":0,"output_tokens":106,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003388,"input_uncached_usd":0.0023279999999999998,"input_cached_usd":0.0,"output_usd":0.0010600000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7777850419515744}
{"item_id":"semeval2013.d011.s008.t002","gold_sense_keys":["burden%1:09:01::"],"candidates":[{"index":1,"sense_key":"burden%1:09:01::","synset_id":"burden.n.01"},{"index":2,"sense_key":"burden%1:06:01::","synset_id":"load.n.01"},{"index":3,"sense_key":"burden%1:10:00::","synset_id":"effect.n.04"},{"index":4,"sense_key":"burden%1:09:00::","synset_id":"burden.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"burden%1:09:01::","call_ids":["semeval2013.d011.s008.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"burden%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":687,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001494,"input_uncached_usd":0.001374,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3746637500589713}
{"item_id":"semeval2013.d011.s008.t003","gold_sense_keys":["economy%1:14:00::"],"candidates":[{"index":1,"sense_key":"economy%1:14:00::","synset_id":"economy.n.01"},{"index":2,"sense_key":"economy%1:09:01::","synset_id":"economy.n.02"},{"index":3,"sense_key":"economy%1:07:00::","synset_id":"economy.n.03"},{"index":4,"sense_key":"economy%1:04:00::","synset_id":"economy.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"economy%1:14:00::","call_ids":["semeval2013.d011.s008.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"economy%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":693,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001506,"input_uncached_usd":0.0013859999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1342624159296975}
{"item_id":"semeval2013.d011.s008.t004","gold_sense_keys":["society%1:14:00::"],"candidates":[{"index":1,"sense_key":"society%1:14:00::","synset_id":"society.n.01"},{"index":2,"sense_key":"society%1:14:01::","synset_id":"club.n.02"},{"index":3,"sense_key":"society%1:26:00::","synset_id":"company.n.03"},{"index":4,"sense_key":"society%1:14:02::","synset_id":"society.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"society%1:14:00::","call_ids":["semeval2013.d011.s008.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"society%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":706,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001532,"input_uncached_usd":0.0014119999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1737932089017704}
{"item_id":"semeval2013.d011.s009.t000","gold_sense_keys":["difference%1:07:00::"],"candidates":[{"index":1,"sense_key":"difference%1:07:00::","synset_id":"difference.n.01"},{"index":2,"sense_key":"difference%1:11:00::","synset_id":"deviation.n.01"},{"index":3,"sense_key":"difference%1:10:00::","synset_id":"dispute.n.01"},{"index":4,"sense_key":"difference%1:24:00::","synset_id":"difference.n.04"},{"index":5,"sense_key":"difference%1:23:00::","synset_id":"remainder.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"difference%1:07:00::","call_ids":["semeval2013.d011.s009.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"difference%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":795,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0022099999999999997,"input_uncached_usd":0.0015899999999999998,"input_cached_usd":0.0,"output_usd":0.00062,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":14.614294084021822}
{"item_id":"semeval2013.d011.s009.t001","gold_sense_keys":["degree%1:07:00::","degree%1:26:01::"],"candidates":[{"index":1,"sense_key":"degree%1:07:00::","synset_id":"degree.n.01"},{"index":2,"sense_key":"degree%1:26:01::","synset_id":"degree.n.02"},{"index":3,"sense_key":"degree%1:10:00::","synset_id":"academic_degree.n.01"},{"index":4,"sense_key":"degree%1:23:00::","synset_id":"degree.n.04"},{"index":5,"sense_key":"degree%1:09:00::","synset_id":"degree.n.05"},{"index":6,"sense_key":"degree%1:23:03::","synset_id":"degree.n.06"},{"index":7,"sense_key":"degree%1:07:01::","synset_id":"degree.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"degree%1:26:01::","call_ids":["semeval2013.d011.s009.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"degree%1:26:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":937,"cached_input_tokens":0,"output_tokens":119,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003064,"input_uncached_usd":0.001874,"input_cached_usd":0.0,"output_usd":0.00119,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5498196659609675}
{"item_id":"semeval2013.d011.s009.t002","gold_sense_keys":["generosity%1:07:00::"],"candidates":[{"index":1,"sense_key":"generosity%1:07:00::","synset_id":"generosity.n.01"},{"index":2,"sense_key":"generosity%1:04:00::","synset_id":"generosity.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"generosity%1:07:00::","call_ids":["semeval2013.d011.s009.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"generosity%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":579,"cached_input_tokens":0,"output_tokens":108,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002238,"input_uncached_usd":0.001158,"input_cached_usd":0.0,"output_usd":0.00108,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7812746670097113}
{"item_id":"semeval2013.d011.s010.t000","gold_sense_keys":["program%1:10:04::"],"candidates":[{"index":1,"sense_key":"program%1:09:00::","synset_id":"plan.n.01"},{"index":2,"sense_key":"program%1:09:01::","synset_id":"program.n.02"},{"index":3,"sense_key":"program%1:10:01::","synset_id":"broadcast.n.02"},{"index":4,"sense_key":"program%1:10:04::","synset_id":"platform.n.02"},{"index":5,"sense_key":"program%1:10:00::","synset_id":"program.n.05"},{"index":6,"sense_key":"program%1:10:05::","synset_id":"course_of_study.n.01"},{"index":7,"sense_key":"program%1:10:02::","synset_id":"program.n.07"},{"index":8,"sense_key":"program%1:04:00::","synset_id":"program.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"program%1:10:04::","call_ids":["semeval2013.d011.s010.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"program%1:10:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1061,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002802,"input_uncached_usd":0.0021219999999999998,"input_cached_usd":0.0,"output_usd":0.00068,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3698095419676974}
{"item_id":"semeval2013.d011.s010.t003","gold_sense_keys":["problem%1:09:00::","problem%1:26:00::"],"candidates":[{"index":1,"sense_key":"problem%1:26:00::","synset_id":"problem.n.01"},{"index":2,"sense_key":"problem%1:10:00::","synset_id":"problem.n.02"},{"index":3,"sense_key":"problem%1:09:00::","synset_id":"trouble.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"problem%1:09:00::","call_ids":["semeval2013.d011.s010.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"problem%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":684,"cached_input_tokens":0,"output_tokens":148,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002848,"input_uncached_usd":0.0013679999999999999,"input_cached_usd":0.0,"output_usd":0.0014800000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9287391250254586}
{"item_id":"semeval2013.d011.s011.t000","gold_sense_keys":["notion%1:09:02::"],"candidates":[{"index":1,"sense_key":"notion%1:09:02::","synset_id":"impression.n.01"},{"index":2,"sense_key":"notion%1:09:00::","synset_id":"notion.n.02"},{"index":3,"sense_key":"notion%1:09:01::","synset_id":"notion.n.03"},{"index":4,"sense_key":"notion%1:06:00::","synset_id":"notion.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"notion%1:09:02::","call_ids":["semeval2013.d011.s011.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"notion%1:09:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":812,"cached_input_tokens":0,"output_tokens":140,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003024,"input_uncached_usd":0.001624,"input_cached_usd":0.0,"output_usd":0.0014000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.98529583401978}
{"item_id":"semeval2013.d011.s011.t002","gold_sense_keys":["burden%1:09:01::"],"candidates":[{"index":1,"sense_key":"burden%1:09:01::","synset_id":"burden.n.01"},{"index":2,"sense_key":"burden%1:06:01::","synset_id":"load.n.01"},{"index":3,"sense_key":"burden%1:10:00::","synset_id":"effect.n.04"},{"index":4,"sense_key":"burden%1:09:00::","synset_id":"burden.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"burden%1:09:01::","call_ids":["semeval2013.d011.s011.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"burden%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":761,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001642,"input_uncached_usd":0.001522,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0330062500433996}
{"item_id":"semeval2013.d011.s011.t003","gold_sense_keys":["problem%1:09:00::","problem%1:26:00::"],"candidates":[{"index":1,"sense_key":"problem%1:26:00::","synset_id":"problem.n.01"},{"index":2,"sense_key":"problem%1:10:00::","synset_id":"problem.n.02"},{"index":3,"sense_key":"problem%1:09:00::","synset_id":"trouble.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"problem%1:09:00::","call_ids":["semeval2013.d011.s011.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"problem%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":729,"cached_input_tokens":0,"output_tokens":118,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0026379999999999997,"input_uncached_usd":0.0014579999999999999,"input_cached_usd":0.0,"output_usd":0.00118,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7111945420037955}
{"item_id":"semeval2013.d011.s012.t001","gold_sense_keys":["book%1:10:00::"],"candidates":[{"index":1,"sense_key":"book%1:10:00::","synset_id":"book.n.01"},{"index":2,"sense_key":"book%1:06:00::","synset_id":"book.n.02"},{"index":3,"sense_key":"book%1:10:02::","synset_id":"record.n.05"},{"index":4,"sense_key":"book%1:10:01::","synset_id":"script.n.01"},{"index":5,"sense_key":"book%1:21:00::","synset_id":"ledger.n.01"},{"index":6,"sense_key":"book%1:14:01::","synset_id":"book.n.06"},{"index":7,"sense_key":"book%1:14:00::","synset_id":"book.n.07"},{"index":8,"sense_key":"book%1:10:05::","synset_id":"koran.n.01"},{"index":9,"sense_key":"book%1:10:04::","synset_id":"bible.n.01"},{"index":10,"sense_key":"book%1:10:03::","synset_id":"book.n.10"},{"index":11,"sense_key":"book%1:06:02::","synset_id":"book.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"book%1:10:00::","call_ids":["semeval2013.d011.s012.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"book%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1256,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002632,"input_uncached_usd":0.002512,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":4.330321542103775}
{"item_id":"semeval2013.d011.s012.t003","gold_sense_keys":["immigration%1:04:00::"],"candidates":[{"index":1,"sense_key":"immigration%1:04:00::","synset_id":"immigration.n.01"},{"index":2,"sense_key":"immigration%1:14:00::","synset_id":"immigration.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"immigration%1:04:00::","call_ids":["semeval2013.d011.s012.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"immigration%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":679,"cached_input_tokens":0,"output_tokens":83,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002188,"input_uncached_usd":0.0013579999999999998,"input_cached_usd":0.0,"output_usd":0.0008300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.49876912496984}
{"item_id":"semeval2013.d011.s012.t004","gold_sense_keys":["people%1:14:00::"],"candidates":[{"index":1,"sense_key":"people%1:14:00::","synset_id":"people.n.01"},{"index":2,"sense_key":"people%1:14:01::","synset_id":"citizenry.n.01"},{"index":3,"sense_key":"people%1:14:02::","synset_id":"people.n.03"},{"index":4,"sense_key":"people%1:14:03::","synset_id":"multitude.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"people%1:14:00::","call_ids":["semeval2013.d011.s012.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"people%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":793,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002206,"input_uncached_usd":0.001586,"input_cached_usd":0.0,"output_usd":0.00062,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5959902920294553}
{"item_id":"semeval2013.d011.s012.t005","gold_sense_keys":["migration%1:04:00::"],"candidates":[{"index":1,"sense_key":"migration%1:04:00::","synset_id":"migration.n.01"},{"index":2,"sense_key":"migration%1:14:00::","synset_id":"migration.n.02"},{"index":3,"sense_key":"migration%1:11:01::","synset_id":"migration.n.03"},{"index":4,"sense_key":"migration%1:11:00::","synset_id":"migration.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"migration%1:04:00::","call_ids":["semeval2013.d011.s012.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"migration%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":747,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001614,"input_uncached_usd":0.0014939999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.216594250057824}
{"item_id":"semeval2013.d011.s012.t007","gold_sense_keys":["future%1:28:00::"],"candidates":[{"index":1,"sense_key":"future%1:28:00::","synset_id":"future.n.01"},{"index":2,"sense_key":"future%1:10:00::","synset_id":"future.n.02"},{"index":3,"sense_key":"future%1:06:00::","synset_id":"future.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"future%1:28:00::","call_ids":["semeval2013.d011.s012.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"future%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":699,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001518,"input_uncached_usd":0.001398,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.058999374974519}
{"item_id":"semeval2013.d011.s012.t009","gold_sense_keys":["specialist%1:18:00::"],"candidates":[{"index":1,"sense_key":"specialist%1:18:00::","synset_id":"specialist.n.01"},{"index":2,"sense_key":"specialist%1:18:01::","synset_id":"specialist.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"specialist%1:18:00::","call_ids":["semeval2013.d011.s012.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"specialist%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":662,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001444,"input_uncached_usd":0.0013239999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1578809580532834}
{"item_id":"semeval2013.d011.s012.t010","gold_sense_keys":["impact%1:19:00::"],"candidates":[{"index":1,"sense_key":"impact%1:11:00::","synset_id":"impact.n.01"},{"index":2,"sense_key":"impact%1:19:00::","synset_id":"impact.n.02"},{"index":3,"sense_key":"impact%1:04:00::","synset_id":"impingement.n.01"},{"index":4,"sense_key":"impact%1:04:01::","synset_id":"shock.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"impact%1:19:00::","call_ids":["semeval2013.d011.s012.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"impact%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":783,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001686,"input_uncached_usd":0.001566,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7315397090278566}
{"item_id":"semeval2013.d011.s012.t011","gold_sense_keys":["migration%1:04:00::"],"candidates":[{"index":1,"sense_key":"migration%1:04:00::","synset_id":"migration.n.01"},{"index":2,"sense_key":"migration%1:14:00::","synset_id":"migration.n.02"},{"index":3,"sense_key":"migration%1:11:01::","synset_id":"migration.n.03"},{"index":4,"sense_key":"migration%1:11:00::","synset_id":"migration.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"migration%1:04:00::","call_ids":["semeval2013.d011.s012.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"migration%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":746,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001612,"input_uncached_usd":0.0014919999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5042864170391113}
{"item_id":"semeval2013.d011.s012.t012","gold_sense_keys":["flow%1:26:00::"],"candidates":[{"index":1,"sense_key":"flow%1:11:00::","synset_id":"flow.n.01"},{"index":2,"sense_key":"flow%1:28:00::","synset_id":"flow.n.02"},{"index":3,"sense_key":"flow%1:04:00::","synset_id":"flow.n.03"},{"index":4,"sense_key":"flow%1:22:00::","synset_id":"flow.n.04"},{"index":5,"sense_key":"flow%1:26:00::","synset_id":"stream.n.04"},{"index":6,"sense_key":"flow%1:14:00::","synset_id":"stream.n.02"},{"index":7,"sense_key":"flow%1:22:01::","synset_id":"menstruation.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"flow%1:26:00::","call_ids":["semeval2013.d011.s012.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"flow%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1035,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00264,"input_uncached_usd":0.00207,"input_cached_usd":0.0,"output_usd":0.0005700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3870542499935254}
{"item_id":"semeval2013.d011.s012.t015","gold_sense_keys":["employment%1:04:00::","employment%1:04:02::","employment%1:26:00::"],"candidates":[{"index":1,"sense_key":"employment%1:26:00::","synset_id":"employment.n.01"},{"index":2,"sense_key":"employment%1:04:00::","synset_id":"employment.n.02"},{"index":3,"sense_key":"employment%1:04:02::","synset_id":"employment.n.03"},{"index":4,"sense_key":"employment%1:04:01::","synset_id":"use.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"employment%1:26:00::","call_ids":["semeval2013.d011.s012.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"employment%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":801,"cached_input_tokens":0,"output_tokens":70,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0023020000000000002,"input_uncached_usd":0.001602,"input_cached_usd":0.0,"output_usd":0.0007000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3440297080669552}
{"item_id":"semeval2013.d011.s012.t016","gold_sense_keys":["country%1:14:00::","country%1:15:00::"],"candidates":[{"index":1,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":2,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":3,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":4,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":5,"sense_key":"country%1:15:02::","synset_id":"area.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"country%1:14:00::","call_ids":["semeval2013.d011.s012.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"country%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":912,"cached_input_tokens":0,"output_tokens":139,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003214,"input_uncached_usd":0.0018239999999999999,"input_cached_usd":0.0,"output_usd":0.0013900000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.3875314999604598}
{"item_id":"semeval2013.d011.s013.t000","gold_sense_keys":["immigration%1:04:00::"],"candidates":[{"index":1,"sense_key":"immigration%1:04:00::","synset_id":"immigration.n.01"},{"index":2,"sense_key":"immigration%1:14:00::","synset_id":"immigration.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"immigration%1:04:00::","call_ids":["semeval2013.d011.s013.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"immigration%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":688,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001496,"input_uncached_usd":0.0013759999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1427599169546738}
{"item_id":"semeval2013.d011.s013.t001","gold_sense_keys":["growth%1:22:02::"],"candidates":[{"index":1,"sense_key":"growth%1:22:00::","synset_id":"growth.n.01"},{"index":2,"sense_key":"growth%1:22:01::","synset_id":"growth.n.02"},{"index":3,"sense_key":"growth%1:22:02::","synset_id":"increase.n.03"},{"index":4,"sense_key":"growth%1:14:00::","synset_id":"growth.n.04"},{"index":5,"sense_key":"growth%1:11:01::","synset_id":"emergence.n.01"},{"index":6,"sense_key":"growth%1:26:00::","synset_id":"growth.n.06"},{"index":7,"sense_key":"growth%1:17:00::","synset_id":"growth.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"growth%1:22:02::","call_ids":["semeval2013.d011.s013.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"growth%1:22:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1010,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00214,"input_uncached_usd":0.00202,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8508200829382986}
{"item_id":"semeval2013.d011.s013.t002","gold_sense_keys":["employment%1:04:00::","employment%1:04:02::","employment%1:26:00::"],"candidates":[{"index":1,"sense_key":"employment%1:26:00::","synset_id":"employment.n.01"},{"index":2,"sense_key":"employment%1:04:00::","synset_id":"employment.n.02"},{"index":3,"sense_key":"employment%1:04:02::","synset_id":"employment.n.03"},{"index":4,"sense_key":"employment%1:04:01::","synset_id":"use.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"employment%1:26:00::","call_ids":["semeval2013.d011.s013.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"employment%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":810,"cached_input_tokens":0,"output_tokens":74,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00236,"input_uncached_usd":0.00162,"input_cached_usd":0.0,"output_usd":0.0007400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4977427090052515}
{"item_id":"semeval2013.d011.s014.t000","gold_sense_keys":["fear%1:12:01::"],"candidates":[{"index":1,"sense_key":"fear%1:12:00::","synset_id":"fear.n.01"},{"index":2,"sense_key":"fear%1:12:01::","synset_id":"concern.n.02"},{"index":3,"sense_key":"fear%1:12:02::","synset_id":"fear.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"fear%1:12:01::","call_ids":["semeval2013.d011.s014.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"fear%1:12:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":812,"cached_input_tokens":0,"output_tokens":97,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002594,"input_uncached_usd":0.001624,"input_cached_usd":0.0,"output_usd":0.00097,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.633116125012748}
{"item_id":"semeval2013.d011.s014.t001","gold_sense_keys":["impact%1:19:00::"],"candidates":[{"index":1,"sense_key":"impact%1:11:00::","synset_id":"impact.n.01"},{"index":2,"sense_key":"impact%1:19:00::","synset_id":"impact.n.02"},{"index":3,"sense_key":"impact%1:04:00::","synset_id":"impingement.n.01"},{"index":4,"sense_key":"impact%1:04:01::","synset_id":"shock.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"impact%1:19:00::","call_ids":["semeval2013.d011.s014.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"impact%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":820,"cached_input_tokens":0,"output_tokens":130,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00294,"input_uncached_usd":0.00164,"input_cached_usd":0.0,"output_usd":0.0013000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8323001250391826}
{"item_id":"semeval2013.d011.s014.t003","gold_sense_keys":["notion%1:09:02::"],"candidates":[{"index":1,"sense_key":"notion%1:09:02::","synset_id":"impression.n.01"},{"index":2,"sense_key":"notion%1:09:00::","synset_id":"notion.n.02"},{"index":3,"sense_key":"notion%1:09:01::","synset_id":"notion.n.03"},{"index":4,"sense_key":"notion%1:06:00::","synset_id":"notion.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"notion%1:09:02::","call_ids":["semeval2013.d011.s014.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"notion%1:09:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":859,"cached_input_tokens":0,"output_tokens":99,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002708,"input_uncached_usd":0.001718,"input_cached_usd":0.0,"output_usd":0.00099,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8324542499613017}
{"item_id":"semeval2013.d011.s014.t004","gold_sense_keys":["worker%1:18:00::"],"candidates":[{"index":1,"sense_key":"worker%1:18:00::","synset_id":"worker.n.01"},{"index":2,"sense_key":"worker%1:18:01::","synset_id":"proletarian.n.01"},{"index":3,"sense_key":"worker%1:05:01::","synset_id":"worker.n.03"},{"index":4,"sense_key":"worker%1:18:02::","synset_id":"actor.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"worker%1:18:00::","call_ids":["semeval2013.d011.s014.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"worker%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":819,"cached_input_tokens":0,"output_tokens":98,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002618,"input_uncached_usd":0.001638,"input_cached_usd":0.0,"output_usd":0.0009800000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.733225042000413}
{"item_id":"semeval2013.d011.s014.t005","gold_sense_keys":["one%1:09:00::"],"candidates":[{"index":1,"sense_key":"one%1:23:00::","synset_id":"one.n.01"},{"index":2,"sense_key":"one%1:09:00::","synset_id":"one.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"one%1:09:00::","call_ids":["semeval2013.d011.s014.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"one%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":723,"cached_input_tokens":0,"output_tokens":79,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002236,"input_uncached_usd":0.001446,"input_cached_usd":0.0,"output_usd":0.00079,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":11.311079916078597}
{"item_id":"semeval2013.d011.s014.t006","gold_sense_keys":["pressure%1:07:00::"],"candidates":[{"index":1,"sense_key":"pressure%1:19:00::","synset_id":"pressure.n.01"},{"index":2,"sense_key":"pressure%1:07:00::","synset_id":"pressure.n.02"},{"index":3,"sense_key":"pressure%1:04:00::","synset_id":"press.n.09"},{"index":4,"sense_key":"pressure%1:26:00::","synset_id":"imperativeness.n.01"},{"index":5,"sense_key":"pressure%1:09:00::","synset_id":"pressure.n.05"},{"index":6,"sense_key":"pressure%1:26:01::","synset_id":"pressure.n.06"},{"index":7,"sense_key":"pressure%1:19:01::","synset_id":"atmospheric_pressure.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"pressure%1:07:00::","call_ids":["semeval2013.d011.s014.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"pressure%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1067,"cached_input_tokens":0,"output_tokens":112,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003254,"input_uncached_usd":0.002134,"input_cached_usd":0.0,"output_usd":0.0011200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2637285840464756}
{"item_id":"semeval2013.d011.s015.t000","gold_sense_keys":["experience%1:09:01::"],"candidates":[{"index":1,"sense_key":"experience%1:09:01::","synset_id":"experience.n.01"},{"index":2,"sense_key":"experience%1:09:00::","synset_id":"experience.n.02"},{"index":3,"sense_key":"experience%1:11:00::","synset_id":"experience.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"experience%1:09:01::","call_ids":["semeval2013.d011.s015.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"experience%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":798,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002176,"input_uncached_usd":0.001596,"input_cached_usd":0.0,"output_usd":0.00058,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2031912499805912}
{"item_id":"semeval2013.d011.s015.t003","gold_sense_keys":["replacement%1:09:00::","replacement%1:18:02::"],"candidates":[{"index":1,"sense_key":"replacement%1:04:00::","synset_id":"replacement.n.01"},{"index":2,"sense_key":"replacement%1:18:02::","synset_id":"surrogate.n.01"},{"index":3,"sense_key":"replacement%1:11:00::","synset_id":"substitution.n.01"},{"index":4,"sense_key":"replacement%1:09:00::","synset_id":"substitute.n.01"},{"index":5,"sense_key":"replacement%1:22:00::","synset_id":"refilling.n.01"},{"index":6,"sense_key":"replacement%1:18:01::","synset_id":"successor.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"replacement%1:18:02::","call_ids":["semeval2013.d011.s015.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"replacement%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":974,"cached_input_tokens":0,"output_tokens":262,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004568,"input_uncached_usd":0.0019479999999999999,"input_cached_usd":0.0,"output_usd":0.0026200000000000004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.7353830420179293}
{"item_id":"semeval2013.d011.s015.t004","gold_sense_keys":["labor%1:14:00::"],"candidates":[{"index":1,"sense_key":"labor%1:14:00::","synset_id":"labor.n.01"},{"index":2,"sense_key":"labor%1:04:00::","synset_id":"labor.n.02"},{"index":3,"sense_key":"labor%1:26:00::","synset_id":"parturiency.n.01"},{"index":4,"sense_key":"labor%1:14:04::","synset_id":"labor_movement.n.01"},{"index":5,"sense_key":"labor%1:14:02::","synset_id":"british_labour_party.n.01"},{"index":6,"sense_key":"labor%1:14:01::","synset_id":"department_of_labor.n.01"},{"index":7,"sense_key":"labor%1:04:01::","synset_id":"undertaking.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"labor%1:14:00::","call_ids":["semeval2013.d011.s015.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"labor%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1154,"cached_input_tokens":0,"output_tokens":88,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003188,"input_uncached_usd":0.0023079999999999997,"input_cached_usd":0.0,"output_usd":0.00088,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":4.099274749984033}
{"item_id":"semeval2013.d011.s015.t005","gold_sense_keys":["trade%1:04:00::"],"candidates":[{"index":1,"sense_key":"trade%1:04:05::","synset_id":"trade.n.01"},{"index":2,"sense_key":"trade%1:04:00::","synset_id":"trade.n.02"},{"index":3,"sense_key":"trade%1:04:01::","synset_id":"trade.n.03"},{"index":4,"sense_key":"trade%1:04:02::","synset_id":"deal.n.01"},{"index":5,"sense_key":"trade%1:14:00::","synset_id":"craft.n.03"},{"index":6,"sense_key":"trade%1:19:00::","synset_id":"trade_wind.n.01"},{"index":7,"sense_key":"trade%1:04:03::","synset_id":"barter.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"trade%1:04:00::","call_ids":["semeval2013.d011.s015.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"trade%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1079,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002878,"input_uncached_usd":0.0021579999999999998,"input_cached_usd":0.0,"output_usd":0.00072,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2421487499959767}
{"item_id":"semeval2013.d011.s015.t006","gold_sense_keys":["shortage%1:26:00::"],"candidates":[{"index":1,"sense_key":"shortage%1:07:00::","synset_id":"deficit.n.01"},{"index":2,"sense_key":"shortage%1:26:00::","synset_id":"dearth.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"shortage%1:26:00::","call_ids":["semeval2013.d011.s015.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"shortage%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":758,"cached_input_tokens":0,"output_tokens":97,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002486,"input_uncached_usd":0.001516,"input_cached_usd":0.0,"output_usd":0.00097,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1322004579706118}
{"item_id":"semeval2013.d011.s015.t007","gold_sense_keys":["labor%1:14:00::"],"candidates":[{"index":1,"sense_key":"labor%1:14:00::","synset_id":"labor.n.01"},{"index":2,"sense_key":"labor%1:04:00::","synset_id":"labor.n.02"},{"index":3,"sense_key":"labor%1:26:00::","synset_id":"parturiency.n.01"},{"index":4,"sense_key":"labor%1:14:04::","synset_id":"labor_movement.n.01"},{"index":5,"sense_key":"labor%1:14:02::","synset_id":"british_labour_party.n.01"},{"index":6,"sense_key":"labor%1:14:01::","synset_id":"department_of_labor.n.01"},{"index":7,"sense_key":"labor%1:04:01::","synset_id":"undertaking.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"labor%1:14:00::","call_ids":["semeval2013.d011.s015.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"labor%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1154,"cached_input_tokens":0,"output_tokens":70,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003008,"input_uncached_usd":0.0023079999999999997,"input_cached_usd":0.0,"output_usd":0.0007000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4116863328963518}
{"item_id":"semeval2013.d011.s016.t001","gold_sense_keys":["sector%1:14:00::"],"candidates":[{"index":1,"sense_key":"sector%1:25:00::","synset_id":"sector.n.01"},{"index":2,"sense_key":"sector%1:14:00::","synset_id":"sector.n.02"},{"index":3,"sense_key":"sector%1:09:00::","synset_id":"sector.n.03"},{"index":4,"sense_key":"sector%1:23:00::","synset_id":"sector.n.04"},{"index":5,"sense_key":"sector%1:15:00::","synset_id":"sector.n.05"},{"index":6,"sense_key":"sector%1:06:00::","synset_id":"sector.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"sector%1:14:00::","call_ids":["semeval2013.d011.s016.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"sector%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":874,"cached_input_tokens":0,"output_tokens":49,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002238,"input_uncached_usd":0.001748,"input_cached_usd":0.0,"output_usd":0.0004900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4247379170265049}
{"item_id":"semeval2013.d011.s016.t002","gold_sense_keys":["labor%1:14:00::"],"candidates":[{"index":1,"sense_key":"labor%1:14:00::","synset_id":"labor.n.01"},{"index":2,"sense_key":"labor%1:04:00::","synset_id":"labor.n.02"},{"index":3,"sense_key":"labor%1:26:00::","synset_id":"parturiency.n.01"},{"index":4,"sense_key":"labor%1:14:04::","synset_id":"labor_movement.n.01"},{"index":5,"sense_key":"labor%1:14:02::","synset_id":"british_labour_party.n.01"},{"index":6,"sense_key":"labor%1:14:01::","synset_id":"department_of_labor.n.01"},{"index":7,"sense_key":"labor%1:04:01::","synset_id":"undertaking.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"labor%1:14:00::","call_ids":["semeval2013.d011.s016.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"labor%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1124,"cached_input_tokens":0,"output_tokens":79,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003038,"input_uncached_usd":0.002248,"input_cached_usd":0.0,"output_usd":0.00079,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5754922919441015}
{"item_id":"semeval2013.d011.s016.t003","gold_sense_keys":["one%1:09:00::"],"candidates":[{"index":1,"sense_key":"one%1:23:00::","synset_id":"one.n.01"},{"index":2,"sense_key":"one%1:09:00::","synset_id":"one.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"one%1:09:00::","call_ids":["semeval2013.d011.s016.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"one%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":719,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0015580000000000001,"input_uncached_usd":0.001438,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":5.664854958071373}
{"item_id":"semeval2013.d011.s016.t004","gold_sense_keys":["sector%1:14:00::"],"candidates":[{"index":1,"sense_key":"sector%1:25:00::","synset_id":"sector.n.01"},{"index":2,"sense_key":"sector%1:14:00::","synset_id":"sector.n.02"},{"index":3,"sense_key":"sector%1:09:00::","synset_id":"sector.n.03"},{"index":4,"sense_key":"sector%1:23:00::","synset_id":"sector.n.04"},{"index":5,"sense_key":"sector%1:15:00::","synset_id":"sector.n.05"},{"index":6,"sense_key":"sector%1:06:00::","synset_id":"sector.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"sector%1:14:00::","call_ids":["semeval2013.d011.s016.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"sector%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":874,"cached_input_tokens":0,"output_tokens":82,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002568,"input_uncached_usd":0.001748,"input_cached_usd":0.0,"output_usd":0.0008200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.473618916934356}
{"item_id":"semeval2013.d011.s016.t005","gold_sense_keys":["training%1:04:00::"],"candidates":[{"index":1,"sense_key":"training%1:04:00::","synset_id":"training.n.01"},{"index":2,"sense_key":"training%1:07:00::","synset_id":"education.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"training%1:04:00::","call_ids":["semeval2013.d011.s016.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"training%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":707,"cached_input_tokens":0,"output_tokens":81,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002224,"input_uncached_usd":0.001414,"input_cached_usd":0.0,"output_usd":0.0008100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0558057919843122}
{"item_id":"semeval2013.d011.s016.t006","gold_sense_keys":["offer%1:10:00::"],"candidates":[{"index":1,"sense_key":"offer%1:10:01::","synset_id":"offer.n.01"},{"index":2,"sense_key":"offer%1:10:00::","synset_id":"offer.n.02"},{"index":3,"sense_key":"offer%1:04:01::","synset_id":"crack.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"offer%1:10:00::","call_ids":["semeval2013.d011.s016.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"offer%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":779,"cached_input_tokens":0,"output_tokens":80,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002358,"input_uncached_usd":0.001558,"input_cached_usd":0.0,"output_usd":0.0008,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1839513749582693}
{"item_id":"semeval2013.d011.s016.t007","gold_sense_keys":["employment%1:04:00::","employment%1:26:00::"],"candidates":[{"index":1,"sense_key":"employment%1:26:00::","synset_id":"employment.n.01"},{"index":2,"sense_key":"employment%1:04:00::","synset_id":"employment.n.02"},{"index":3,"sense_key":"employment%1:04:02::","synset_id":"employment.n.03"},{"index":4,"sense_key":"employment%1:04:01::","synset_id":"use.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"employment%1:04:02::","call_ids":["semeval2013.d011.s016.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"employment%1:04:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":834,"cached_input_tokens":0,"output_tokens":101,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002678,"input_uncached_usd":0.001668,"input_cached_usd":0.0,"output_usd":0.00101,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.186653125099838}
{"item_id":"semeval2013.d011.s017.t000","gold_sense_keys":["effect%1:19:00::"],"candidates":[{"index":1,"sense_key":"effect%1:19:00::","synset_id":"consequence.n.01"},{"index":2,"sense_key":"effect%1:07:00::","synset_id":"impression.n.02"},{"index":3,"sense_key":"effect%1:09:00::","synset_id":"effect.n.03"},{"index":4,"sense_key":"effect%1:10:00::","synset_id":"effect.n.04"},{"index":5,"sense_key":"effect%1:07:01::","synset_id":"effect.n.05"},{"index":6,"sense_key":"effect%1:26:00::","synset_id":"effect.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"effect%1:19:00::","call_ids":["semeval2013.d011.s017.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"effect%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1014,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0021479999999999997,"input_uncached_usd":0.002028,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8217453749384731}
{"item_id":"semeval2013.d011.s018.t000","gold_sense_keys":["way%1:04:01::","way%1:07:01::"],"candidates":[{"index":1,"sense_key":"way%1:07:01::","synset_id":"manner.n.01"},{"index":2,"sense_key":"way%1:04:01::","synset_id":"means.n.01"},{"index":3,"sense_key":"way%1:15:00::","synset_id":"direction.n.01"},{"index":4,"sense_key":"way%1:26:00::","synset_id":"way.n.04"},{"index":5,"sense_key":"way%1:04:00::","synset_id":"way.n.05"},{"index":6,"sense_key":"way%1:06:00::","synset_id":"way.n.06"},{"index":7,"sense_key":"way%1:04:02::","synset_id":"way.n.07"},{"index":8,"sense_key":"way%1:23:00::","synset_id":"room.n.02"},{"index":9,"sense_key":"way%1:07:02::","synset_id":"way.n.09"},{"index":10,"sense_key":"way%1:09:00::","synset_id":"way.n.10"},{"index":11,"sense_key":"way%1:09:01::","synset_id":"way.n.11"},{"index":12,"sense_key":"way%1:21:00::","synset_id":"way.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"way%1:07:01::","call_ids":["semeval2013.d011.s018.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"way%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1166,"cached_input_tokens":0,"output_tokens":47,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002802,"input_uncached_usd":0.002332,"input_cached_usd":0.0,"output_usd":0.00047000000000000004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5048399160150439}
{"item_id":"semeval2013.d011.s018.t002","gold_sense_keys":["country%1:14:00::","country%1:15:00::"],"candidates":[{"index":1,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":2,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":3,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":4,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":5,"sense_key":"country%1:15:02::","synset_id":"area.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"country%1:15:00::","call_ids":["semeval2013.d011.s018.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"country%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":899,"cached_input_tokens":0,"output_tokens":103,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002828,"input_uncached_usd":0.001798,"input_cached_usd":0.0,"output_usd":0.00103,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7535283749457449}
{"item_id":"semeval2013.d011.s018.t003","gold_sense_keys":["shortage%1:26:00::"],"candidates":[{"index":1,"sense_key":"shortage%1:07:00::","synset_id":"deficit.n.01"},{"index":2,"sense_key":"shortage%1:26:00::","synset_id":"dearth.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"shortage%1:26:00::","call_ids":["semeval2013.d011.s018.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"shortage%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":683,"cached_input_tokens":0,"output_tokens":128,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002646,"input_uncached_usd":0.001366,"input_cached_usd":0.0,"output_usd":0.00128,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.500356207950972}
{"item_id":"semeval2013.d011.s018.t005","gold_sense_keys":["complementarity%1:24:02::"],"candidates":[{"index":1,"sense_key":"complementarity%1:24:01::","synset_id":"complementarity.n.01"},{"index":2,"sense_key":"complementarity%1:24:02::","synset_id":"complementarity.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"complementarity%1:24:02::","call_ids":["semeval2013.d011.s018.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"complementarity%1:24:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":664,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001448,"input_uncached_usd":0.001328,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.524442124995403}
{"item_id":"semeval2013.d011.s018.t007","gold_sense_keys":["arrival%1:04:01::"],"candidates":[{"index":1,"sense_key":"arrival%1:04:00::","synset_id":"arrival.n.01"},{"index":2,"sense_key":"arrival%1:04:01::","synset_id":"arrival.n.02"},{"index":3,"sense_key":"arrival%1:18:00::","synset_id":"arrival.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"arrival%1:04:01::","call_ids":["semeval2013.d011.s018.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"arrival%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":686,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001972,"input_uncached_usd":0.001372,"input_cached_usd":0.0,"output_usd":0.0006000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2300572079839185}
{"item_id":"semeval2013.d011.s018.t009","gold_sense_keys":["effect%1:19:00::"],"candidates":[{"index":1,"sense_key":"effect%1:19:00::","synset_id":"consequence.n.01"},{"index":2,"sense_key":"effect%1:07:00::","synset_id":"impression.n.02"},{"index":3,"sense_key":"effect%1:09:00::","synset_id":"effect.n.03"},{"index":4,"sense_key":"effect%1:10:00::","synset_id":"effect.n.04"},{"index":5,"sense_key":"effect%1:07:01::","synset_id":"effect.n.05"},{"index":6,"sense_key":"effect%1:26:00::","synset_id":"effect.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"effect%1:19:00::","call_ids":["semeval2013.d011.s018.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"effect%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":926,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001972,"input_uncached_usd":0.001852,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4594758750172332}
{"item_id":"semeval2013.d011.s019.t000","gold_sense_keys":["rise%1:11:00::"],"candidates":[{"index":1,"sense_key":"rise%1:11:00::","synset_id":"rise.n.01"},{"index":2,"sense_key":"rise%1:04:00::","synset_id":"rise.n.02"},{"index":3,"sense_key":"rise%1:17:00::","synset_id":"ascent.n.01"},{"index":4,"sense_key":"rise%1:11:02::","synset_id":"rise.n.04"},{"index":5,"sense_key":"rise%1:07:02::","synset_id":"raise.n.01"},{"index":6,"sense_key":"rise%1:07:01::","synset_id":"upgrade.n.04"},{"index":7,"sense_key":"rise%1:11:01::","synset_id":"lift.n.04"},{"index":8,"sense_key":"rise%1:11:03::","synset_id":"emanation.n.03"},{"index":9,"sense_key":"rise%1:07:03::","synset_id":"rise.n.09"},{"index":10,"sense_key":"rise%1:04:01::","synset_id":"advance.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"rise%1:11:00::","call_ids":["semeval2013.d011.s019.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"rise%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1228,"cached_input_tokens":0,"output_tokens":138,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003836,"input_uncached_usd":0.002456,"input_cached_usd":0.0,"output_usd":0.0013800000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6653174590319395}
{"item_id":"semeval2013.d011.s019.t001","gold_sense_keys":["flow%1:26:00::"],"candidates":[{"index":1,"sense_key":"flow%1:11:00::","synset_id":"flow.n.01"},{"index":2,"sense_key":"flow%1:28:00::","synset_id":"flow.n.02"},{"index":3,"sense_key":"flow%1:04:00::","synset_id":"flow.n.03"},{"index":4,"sense_key":"flow%1:22:00::","synset_id":"flow.n.04"},{"index":5,"sense_key":"flow%1:26:00::","synset_id":"stream.n.04"},{"index":6,"sense_key":"flow%1:14:00::","synset_id":"stream.n.02"},{"index":7,"sense_key":"flow%1:22:01::","synset_id":"menstruation.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"flow%1:26:00::","call_ids":["semeval2013.d011.s019.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"flow%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1103,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002816,"input_uncached_usd":0.002206,"input_cached_usd":0.0,"output_usd":0.0006100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.686272958992049}
{"item_id":"semeval2013.d011.s019.t002","gold_sense_keys":["increase%1:07:00::","increase%1:11:00::"],"candidates":[{"index":1,"sense_key":"increase%1:23:00::","synset_id":"addition.n.03"},{"index":2,"sense_key":"increase%1:11:00::","synset_id":"increase.n.02"},{"index":3,"sense_key":"increase%1:22:00::","synset_id":"increase.n.03"},{"index":4,"sense_key":"increase%1:07:00::","synset_id":"increase.n.04"},{"index":5,"sense_key":"increase%1:04:00::","synset_id":"increase.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"increase%1:07:00::","call_ids":["semeval2013.d011.s019.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"increase%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":934,"cached_input_tokens":0,"output_tokens":92,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0027879999999999997,"input_uncached_usd":0.0018679999999999999,"input_cached_usd":0.0,"output_usd":0.00092,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.750756291905418}
{"item_id":"semeval2013.d011.s020.t000","gold_sense_keys":["fact%1:09:01::"],"candidates":[{"index":1,"sense_key":"fact%1:09:01::","synset_id":"fact.n.01"},{"index":2,"sense_key":"fact%1:10:01::","synset_id":"fact.n.02"},{"index":3,"sense_key":"fact%1:26:01::","synset_id":"fact.n.03"},{"index":4,"sense_key":"fact%1:09:02::","synset_id":"fact.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"fact%1:09:01::","call_ids":["semeval2013.d011.s020.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"fact%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":849,"cached_input_tokens":0,"output_tokens":140,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003098,"input_uncached_usd":0.0016979999999999999,"input_cached_usd":0.0,"output_usd":0.0014000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.614319624961354}
{"item_id":"semeval2013.d011.s020.t001","gold_sense_keys":["diversity%1:07:02::"],"candidates":[{"index":1,"sense_key":"diversity%1:07:02::","synset_id":"diverseness.n.01"},{"index":2,"sense_key":"diversity%1:07:01::","synset_id":"diversity.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"diversity%1:07:02::","call_ids":["semeval2013.d011.s020.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"diversity%1:07:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":722,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001564,"input_uncached_usd":0.001444,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2399324160069227}
{"item_id":"semeval2013.d011.s020.t003","gold_sense_keys":["creation%1:04:00::"],"candidates":[{"index":1,"sense_key":"creation%1:04:00::","synset_id":"creation.n.01"},{"index":2,"sense_key":"creation%1:06:00::","synset_id":"creation.n.02"},{"index":3,"sense_key":"creation%1:11:00::","synset_id":"creation.n.03"},{"index":4,"sense_key":"creation%1:04:01::","synset_id":"initiation.n.02"},{"index":5,"sense_key":"creation%1:04:02::","synset_id":"creation.n.05"},{"index":6,"sense_key":"creation%1:17:00::","synset_id":"universe.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"creation%1:04:00::","call_ids":["semeval2013.d011.s020.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"creation%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":957,"cached_input_tokens":0,"output_tokens":89,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002804,"input_uncached_usd":0.001914,"input_cached_usd":0.0,"output_usd":0.0008900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.684350374969654}
{"item_id":"semeval2013.d011.s020.t004","gold_sense_keys":["idea%1:09:00::"],"candidates":[{"index":1,"sense_key":"idea%1:09:00::","synset_id":"idea.n.01"},{"index":2,"sense_key":"idea%1:09:01::","synset_id":"mind.n.06"},{"index":3,"sense_key":"idea%1:09:03::","synset_id":"idea.n.03"},{"index":4,"sense_key":"idea%1:09:02::","synset_id":"estimate.n.01"},{"index":5,"sense_key":"idea%1:10:00::","synset_id":"theme.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"idea%1:09:00::","call_ids":["semeval2013.d011.s020.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"idea%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":960,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0020399999999999997,"input_uncached_usd":0.0019199999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1709917079424486}
{"item_id":"semeval2013.d011.s020.t006","gold_sense_keys":["proportion%1:24:01::"],"candidates":[{"index":1,"sense_key":"proportion%1:24:00::","synset_id":"proportion.n.01"},{"index":2,"sense_key":"proportion%1:07:00::","synset_id":"proportion.n.02"},{"index":3,"sense_key":"proportion%1:25:00::","synset_id":"symmetry.n.02"},{"index":4,"sense_key":"proportion%1:24:01::","synset_id":"proportion.n.04"},{"index":5,"sense_key":"proportion%1:07:01::","synset_id":"proportion.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"proportion%1:24:01::","call_ids":["semeval2013.d011.s020.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"proportion%1:24:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":963,"cached_input_tokens":0,"output_tokens":116,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003086,"input_uncached_usd":0.001926,"input_cached_usd":0.0,"output_usd":0.00116,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9337810840224847}
{"item_id":"semeval2013.d011.s020.t008","gold_sense_keys":["american%3:01:01::"],"candidates":[{"index":1,"sense_key":"american%3:01:01::","synset_id":"american.a.01"},{"index":2,"sense_key":"american%3:01:00::","synset_id":"american.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"american%3:01:01::","call_ids":["semeval2013.d011.s020.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"american%3:01:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":719,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0015580000000000001,"input_uncached_usd":0.001438,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.073200416052714}
{"item_id":"semeval2013.d011.s020.t010","gold_sense_keys":["winner%1:18:00::"],"candidates":[{"index":1,"sense_key":"winner%1:18:00::","synset_id":"winner.n.01"},{"index":2,"sense_key":"winner%1:18:01::","synset_id":"winner.n.02"},{"index":3,"sense_key":"winner%1:18:02::","synset_id":"achiever.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"winner%1:18:00::","call_ids":["semeval2013.d011.s020.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"winner%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":751,"cached_input_tokens":0,"output_tokens":116,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002662,"input_uncached_usd":0.0015019999999999999,"input_cached_usd":0.0,"output_usd":0.00116,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.058352958993055}
{"item_id":"semeval2013.d011.s020.t012","gold_sense_keys":["yahoo%1:10:00::"],"candidates":[{"index":1,"sense_key":"yahoo%1:18:01::","synset_id":"yokel.n.01"},{"index":2,"sense_key":"yahoo%1:18:00::","synset_id":"yahoo.n.02"},{"index":3,"sense_key":"yahoo%1:10:00::","synset_id":"yahoo.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"yahoo%1:10:00::","call_ids":["semeval2013.d011.s020.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"yahoo%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":795,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00171,"input_uncached_usd":0.0015899999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":14.153625875012949}
{"item_id":"semeval2013.d011.s021.t001","gold_sense_keys":["contributor%1:18:00::"],"candidates":[{"index":1,"sense_key":"contributor%1:18:00::","synset_id":"subscriber.n.03"},{"index":2,"sense_key":"contributor%1:18:01::","synset_id":"contributor.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"contributor%1:18:00::","call_ids":["semeval2013.d011.s021.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"contributor%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":736,"cached_input_tokens":0,"output_tokens":89,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002362,"input_uncached_usd":0.001472,"input_cached_usd":0.0,"output_usd":0.0008900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5991601669229567}
{"item_id":"semeval2013.d011.s021.t003","gold_sense_keys":["budget%1:21:02::"],"candidates":[{"index":1,"sense_key":"budget%1:21:02::","synset_id":"budget.n.01"},{"index":2,"sense_key":"budget%1:21:03::","synset_id":"budget.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"budget%1:21:02::","call_ids":["semeval2013.d011.s021.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"budget%1:21:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":753,"cached_input_tokens":0,"output_tokens":118,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002686,"input_uncached_usd":0.001506,"input_cached_usd":0.0,"output_usd":0.00118,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7303266669623554}
{"item_id":"semeval2013.d011.s022.t001","gold_sense_keys":["labor%1:14:00::"],"candidates":[{"index":1,"sense_key":"labor%1:14:00::","synset_id":"labor.n.01"},{"index":2,"sense_key":"labor%1:04:00::","synset_id":"labor.n.02"},{"index":3,"sense_key":"labor%1:26:00::","synset_id":"parturiency.n.01"},{"index":4,"sense_key":"labor%1:14:04::","synset_id":"labor_movement.n.01"},{"index":5,"sense_key":"labor%1:14:02::","synset_id":"british_labour_party.n.01"},{"index":6,"sense_key":"labor%1:14:01::","synset_id":"department_of_labor.n.01"},{"index":7,"sense_key":"labor%1:04:01::","synset_id":"undertaking.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"labor%1:14:00::","call_ids":["semeval2013.d011.s022.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"labor%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1147,"cached_input_tokens":0,"output_tokens":129,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0035840000000000004,"input_uncached_usd":0.002294,"input_cached_usd":0.0,"output_usd":0.0012900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9470100419130176}
{"item_id":"semeval2013.d011.s022.t002","gold_sense_keys":["organisation%1:14:00::","organisation%1:14:01::"],"candidates":[{"index":1,"sense_key":"organisation%1:14:01::","synset_id":"administration.n.02"},{"index":2,"sense_key":"organisation%1:14:00::","synset_id":"organization.n.01"},{"index":3,"sense_key":"organisation%1:09:00::","synset_id":"arrangement.n.03"},{"index":4,"sense_key":"organisation%1:07:00::","synset_id":"organization.n.05"},{"index":5,"sense_key":"organisation%1:04:00::","synset_id":"organization.n.04"},{"index":6,"sense_key":"organisation%1:04:01::","synset_id":"organization.n.06"},{"index":7,"sense_key":"organisation%1:04:02::","synset_id":"constitution.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"organisation%1:14:00::","call_ids":["semeval2013.d011.s022.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"organisation%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1160,"cached_input_tokens":0,"output_tokens":95,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0032700000000000003,"input_uncached_usd":0.00232,"input_cached_usd":0.0,"output_usd":0.0009500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8038597499253228}
{"item_id":"semeval2013.d011.s022.t006","gold_sense_keys":["age%1:28:00::"],"candidates":[{"index":1,"sense_key":"age%1:07:00::","synset_id":"age.n.01"},{"index":2,"sense_key":"age%1:28:02::","synset_id":"historic_period.n.01"},{"index":3,"sense_key":"age%1:28:00::","synset_id":"age.n.03"},{"index":4,"sense_key":"age%1:28:03::","synset_id":"long_time.n.01"},{"index":5,"sense_key":"age%1:28:01::","synset_id":"old_age.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"age%1:07:00::","call_ids":["semeval2013.d011.s022.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"age%1:07:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":904,"cached_input_tokens":0,"output_tokens":156,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0033680000000000003,"input_uncached_usd":0.001808,"input_cached_usd":0.0,"output_usd":0.0015600000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.853440708015114}
{"item_id":"semeval2013.d011.s022.t007","gold_sense_keys":["contribution%1:21:01::"],"candidates":[{"index":1,"sense_key":"contribution%1:04:02::","synset_id":"contribution.n.01"},{"index":2,"sense_key":"contribution%1:21:00::","synset_id":"contribution.n.02"},{"index":3,"sense_key":"contribution%1:04:00::","synset_id":"contribution.n.03"},{"index":4,"sense_key":"contribution%1:21:01::","synset_id":"contribution.n.04"},{"index":5,"sense_key":"contribution%1:04:01::","synset_id":"contribution.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"contribution%1:21:01::","call_ids":["semeval2013.d011.s022.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"contribution%1:21:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":895,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00247,"input_uncached_usd":0.00179,"input_cached_usd":0.0,"output_usd":0.00068,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3511171659920365}
{"item_id":"semeval2013.d011.s022.t009","gold_sense_keys":["expenditure%1:21:00::"],"candidates":[{"index":1,"sense_key":"expenditure%1:21:00::","synset_id":"outgo.n.01"},{"index":2,"sense_key":"expenditure%1:04:00::","synset_id":"expending.n.01"},{"index":3,"sense_key":"expenditure%1:04:01::","synset_id":"consumption.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"expenditure%1:21:00::","call_ids":["semeval2013.d011.s022.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"expenditure%1:21:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":756,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001632,"input_uncached_usd":0.0015119999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9478545839665458}
{"item_id":"semeval2013.d011.s022.t010","gold_sense_keys":["average%1:09:00::"],"candidates":[{"index":1,"sense_key":"average%1:09:00::","synset_id":"average.n.01"},{"index":2,"sense_key":"average%1:24:00::","synset_id":"average.n.02"},{"index":3,"sense_key":"average%1:09:01::","synset_id":"average.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"average%1:09:00::","call_ids":["semeval2013.d011.s022.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"average%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":767,"cached_input_tokens":0,"output_tokens":122,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0027540000000000004,"input_uncached_usd":0.001534,"input_cached_usd":0.0,"output_usd":0.0012200000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8800127080176026}
{"item_id":"semeval2013.d011.s022.t012","gold_sense_keys":["budget%1:21:02::"],"candidates":[{"index":1,"sense_key":"budget%1:21:02::","synset_id":"budget.n.01"},{"index":2,"sense_key":"budget%1:21:03::","synset_id":"budget.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"budget%1:21:02::","call_ids":["semeval2013.d011.s022.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"budget%1:21:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":735,"cached_input_tokens":0,"output_tokens":133,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0028,"input_uncached_usd":0.00147,"input_cached_usd":0.0,"output_usd":0.00133,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":12.239332624943927}
{"item_id":"semeval2013.d011.s023.t001","gold_sense_keys":["mandate%1:04:00::"],"candidates":[{"index":1,"sense_key":"mandate%1:10:00::","synset_id":"mandate.n.01"},{"index":2,"sense_key":"mandate%1:15:00::","synset_id":"mandate.n.02"},{"index":3,"sense_key":"mandate%1:04:00::","synset_id":"mandate.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"mandate%1:04:00::","call_ids":["semeval2013.d011.s023.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"mandate%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":811,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001742,"input_uncached_usd":0.001622,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.550235208007507}
{"item_id":"semeval2013.d011.s023.t003","gold_sense_keys":["growth%1:22:02::"],"candidates":[{"index":1,"sense_key":"growth%1:22:00::","synset_id":"growth.n.01"},{"index":2,"sense_key":"growth%1:22:01::","synset_id":"growth.n.02"},{"index":3,"sense_key":"growth%1:22:02::","synset_id":"increase.n.03"},{"index":4,"sense_key":"growth%1:14:00::","synset_id":"growth.n.04"},{"index":5,"sense_key":"growth%1:11:01::","synset_id":"emergence.n.01"},{"index":6,"sense_key":"growth%1:26:00::","synset_id":"growth.n.06"},{"index":7,"sense_key":"growth%1:17:00::","synset_id":"growth.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"growth%1:22:02::","call_ids":["semeval2013.d011.s023.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"growth%1:22:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1081,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002812,"input_uncached_usd":0.002162,"input_cached_usd":0.0,"output_usd":0.0006500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7558174589648843}
{"item_id":"semeval2013.d011.s024.t000","gold_sense_keys":["question%1:10:01::"],"candidates":[{"index":1,"sense_key":"question%1:10:00::","synset_id":"question.n.01"},{"index":2,"sense_key":"question%1:10:01::","synset_id":"question.n.02"},{"index":3,"sense_key":"question%1:10:03::","synset_id":"question.n.03"},{"index":4,"sense_key":"question%1:07:00::","synset_id":"doubt.n.02"},{"index":5,"sense_key":"question%1:10:02::","synset_id":"motion.n.05"},{"index":6,"sense_key":"question%1:10:04::","synset_id":"question.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"question%1:10:01::","call_ids":["semeval2013.d011.s024.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"question%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":997,"cached_input_tokens":0,"output_tokens":104,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0030340000000000002,"input_uncached_usd":0.001994,"input_cached_usd":0.0,"output_usd":0.0010400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.975373167078942}
{"item_id":"semeval2013.d011.s024.t002","gold_sense_keys":["effect%1:19:00::"],"candidates":[{"index":1,"sense_key":"effect%1:19:00::","synset_id":"consequence.n.01"},{"index":2,"sense_key":"effect%1:07:00::","synset_id":"impression.n.02"},{"index":3,"sense_key":"effect%1:09:00::","synset_id":"effect.n.03"},{"index":4,"sense_key":"effect%1:10:00::","synset_id":"effect.n.04"},{"index":5,"sense_key":"effect%1:07:01::","synset_id":"effect.n.05"},{"index":6,"sense_key":"effect%1:26:00::","synset_id":"effect.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"effect%1:19:00::","call_ids":["semeval2013.d011.s024.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"effect%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":941,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002002,"input_uncached_usd":0.001882,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":14.796273875050247}
{"item_id":"semeval2013.d012.s000.t000","gold_sense_keys":["pound%1:23:01::"],"candidates":[{"index":1,"sense_key":"pound%1:23:09::","synset_id":"pound.n.01"},{"index":2,"sense_key":"pound%1:23:01::","synset_id":"british_pound.n.01"},{"index":3,"sense_key":"pound%1:23:10::","synset_id":"pound.n.03"},{"index":4,"sense_key":"pound%1:23:08::","synset_id":"syrian_pound.n.01"},{"index":5,"sense_key":"pound%1:23:07::","synset_id":"sudanese_pound.n.01"},{"index":6,"sense_key":"pound%1:23:05::","synset_id":"lebanese_pound.n.01"},{"index":7,"sense_key":"pound%1:23:04::","synset_id":"irish_pound.n.01"},{"index":8,"sense_key":"pound%1:23:03::","synset_id":"egyptian_pound.n.01"},{"index":9,"sense_key":"pound%1:23:02::","synset_id":"cypriot_pound.n.01"},{"index":10,"sense_key":"pound%1:23:00::","synset_id":"pound.n.10"},{"index":11,"sense_key":"pound%1:18:00::","synset_id":"pound.n.11"},{"index":12,"sense_key":"pound%1:10:00::","synset_id":"pound.n.12"},{"index":13,"sense_key":"pound%1:06:00::","synset_id":"pound.n.13"},{"index":14,"sense_key":"pound%1:04:00::","synset_id":"hammer.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"pound%1:23:01::","call_ids":["semeval2013.d012.s000.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"pound%1:23:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1233,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0025859999999999998,"input_uncached_usd":0.002466,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":14.457550082937814}
{"item_id":"semeval2013.d012.s000.t002","gold_sense_keys":["usa%1:15:00::"],"candidates":[{"index":1,"sense_key":"usa%1:15:00::","synset_id":"united_states.n.01"},{"index":2,"sense_key":"usa%1:14:00::","synset_id":"united_states_army.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"usa%1:15:00::","call_ids":["semeval2013.d012.s000.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"usa%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":495,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00111,"input_uncached_usd":0.00099,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2785244160331786}
{"item_id":"semeval2013.d012.s001.t000","gold_sense_keys":["trouble%1:09:00::","trouble%1:11:00::","trouble%1:12:03::"],"candidates":[{"index":1,"sense_key":"trouble%1:09:00::","synset_id":"trouble.n.01"},{"index":2,"sense_key":"trouble%1:11:01::","synset_id":"fuss.n.02"},{"index":3,"sense_key":"trouble%1:11:00::","synset_id":"trouble.n.03"},{"index":4,"sense_key":"trouble%1:04:02::","synset_id":"trouble.n.04"},{"index":5,"sense_key":"trouble%1:12:03::","synset_id":"worry.n.02"},{"index":6,"sense_key":"trouble%1:26:02::","synset_id":"trouble.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"trouble%1:09:00::","call_ids":["semeval2013.d012.s001.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"trouble%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":787,"cached_input_tokens":0,"output_tokens":114,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0027140000000000003,"input_uncached_usd":0.0015739999999999999,"input_cached_usd":0.0,"output_usd":0.0011400000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5497521249344572}
{"item_id":"semeval2013.d012.s001.t001","gold_sense_keys":["uproar%1:26:00::"],"candidates":[{"index":1,"sense_key":"uproar%1:26:00::","synset_id":"tumult.n.01"},{"index":2,"sense_key":"uproar%1:11:00::","synset_id":"hubbub.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"uproar%1:26:00::","call_ids":["semeval2013.d012.s001.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"uproar%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":541,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0017519999999999999,"input_uncached_usd":0.0010819999999999998,"input_cached_usd":0.0,"output_usd":0.00067,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1603732500225306}
{"item_id":"semeval2013.d012.s001.t003","gold_sense_keys":["pound%1:23:01::"],"candidates":[{"index":1,"sense_key":"pound%1:23:09::","synset_id":"pound.n.01"},{"index":2,"sense_key":"pound%1:23:01::","synset_id":"british_pound.n.01"},{"index":3,"sense_key":"pound%1:23:10::","synset_id":"pound.n.03"},{"index":4,"sense_key":"pound%1:23:08::","synset_id":"syrian_pound.n.01"},{"index":5,"sense_key":"pound%1:23:07::","synset_id":"sudanese_pound.n.01"},{"index":6,"sense_key":"pound%1:23:05::","synset_id":"lebanese_pound.n.01"},{"index":7,"sense_key":"pound%1:23:04::","synset_id":"irish_pound.n.01"},{"index":8,"sense_key":"pound%1:23:03::","synset_id":"egyptian_pound.n.01"},{"index":9,"sense_key":"pound%1:23:02::","synset_id":"cypriot_pound.n.01"},{"index":10,"sense_key":"pound%1:23:00::","synset_id":"pound.n.10"},{"index":11,"sense_key":"pound%1:18:00::","synset_id":"pound.n.11"},{"index":12,"sense_key":"pound%1:10:00::","synset_id":"pound.n.12"},{"index":13,"sense_key":"pound%1:06:00::","synset_id":"pound.n.13"},{"index":14,"sense_key":"pound%1:04:00::","synset_id":"hammer.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"pound%1:23:01::","call_ids":["semeval2013.d012.s001.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"pound%1:23:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1340,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0028,"input_uncached_usd":0.00268,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0003037080168724}
{"item_id":"semeval2013.d012.s001.t005","gold_sense_keys":["usa%1:15:00::"],"candidates":[{"index":1,"sense_key":"usa%1:15:00::","synset_id":"united_states.n.01"},{"index":2,"sense_key":"usa%1:14:00::","synset_id":"united_states_army.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"usa%1:15:00::","call_ids":["semeval2013.d012.s001.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"usa%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":602,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001324,"input_uncached_usd":0.001204,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8182972919894382}
{"item_id":"semeval2013.d012.s002.t000","gold_sense_keys":["problem%1:09:00::","problem%1:26:00::"],"candidates":[{"index":1,"sense_key":"problem%1:26:00::","synset_id":"problem.n.01"},{"index":2,"sense_key":"problem%1:10:00::","synset_id":"problem.n.02"},{"index":3,"sense_key":"problem%1:09:00::","synset_id":"trouble.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"problem%1:09:00::","call_ids":["semeval2013.d012.s002.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"problem%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":622,"cached_input_tokens":0,"output_tokens":232,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003564,"input_uncached_usd":0.0012439999999999999,"input_cached_usd":0.0,"output_usd":0.00232,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":14.848410250036977}
{"item_id":"semeval2013.d012.s002.t001","gold_sense_keys":["home%1:15:02::"],"candidates":[{"index":1,"sense_key":"home%1:15:04::","synset_id":"home.n.01"},{"index":2,"sense_key":"home%1:06:00::","synset_id":"dwelling.n.01"},{"index":3,"sense_key":"home%1:15:02::","synset_id":"home.n.03"},{"index":4,"sense_key":"home%1:06:02::","synset_id":"home_plate.n.01"},{"index":5,"sense_key":"home%1:15:05::","synset_id":"base.n.14"},{"index":6,"sense_key":"home%1:15:03::","synset_id":"home.n.06"},{"index":7,"sense_key":"home%1:26:00::","synset_id":"home.n.07"},{"index":8,"sense_key":"home%1:14:00::","synset_id":"family.n.01"},{"index":9,"sense_key":"home%1:06:01::","synset_id":"home.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"home%1:15:02::","call_ids":["semeval2013.d012.s002.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"home%1:15:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1008,"cached_input_tokens":0,"output_tokens":78,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002796,"input_uncached_usd":0.002016,"input_cached_usd":0.0,"output_usd":0.0007800000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":14.428284916095436}
{"item_id":"semeval2013.d012.s002.t002","gold_sense_keys":["industry%1:14:00::"],"candidates":[{"index":1,"sense_key":"industry%1:14:00::","synset_id":"industry.n.01"},{"index":2,"sense_key":"industry%1:04:00::","synset_id":"industry.n.02"},{"index":3,"sense_key":"industry%1:07:00::","synset_id":"diligence.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"industry%1:14:00::","call_ids":["semeval2013.d012.s002.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"industry%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":643,"cached_input_tokens":0,"output_tokens":69,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0019760000000000003,"input_uncached_usd":0.001286,"input_cached_usd":0.0,"output_usd":0.0006900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":14.412455332931131}
{"item_id":"semeval2013.d012.s002.t003","gold_sense_keys":["giant%1:14:00::"],"candidates":[{"index":1,"sense_key":"giant%1:05:00::","synset_id":"giant.n.01"},{"index":2,"sense_key":"giant%1:18:01::","synset_id":"colossus.n.02"},{"index":3,"sense_key":"giant%1:14:00::","synset_id":"giant.n.03"},{"index":4,"sense_key":"giant%1:18:03::","synset_id":"giant.n.04"},{"index":5,"sense_key":"giant%1:18:00::","synset_id":"giant.n.05"},{"index":6,"sense_key":"giant%1:18:02::","synset_id":"giant.n.06"},{"index":7,"sense_key":"giant%1:17:00::","synset_id":"giant_star.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"giant%1:14:00::","call_ids":["semeval2013.d012.s002.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"giant%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":794,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001708,"input_uncached_usd":0.001588,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.7793808750575408}
{"item_id":"semeval2013.d012.s002.t005","gold_sense_keys":["usa%1:15:00::"],"candidates":[{"index":1,"sense_key":"usa%1:15:00::","synset_id":"united_states.n.01"},{"index":2,"sense_key":"usa%1:14:00::","synset_id":"united_states_army.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"usa%1:15:00::","call_ids":["semeval2013.d012.s002.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"usa%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":619,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001358,"input_uncached_usd":0.001238,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":4.783822249970399}
{"item_id":"semeval2013.d012.s002.t006","gold_sense_keys":["dollar%1:23:00::"],"candidates":[{"index":1,"sense_key":"dollar%1:23:00::","synset_id":"dollar.n.01"},{"index":2,"sense_key":"dollar%1:21:00::","synset_id":"dollar.n.02"},{"index":3,"sense_key":"dollar%1:21:01::","synset_id":"dollar.n.03"},{"index":4,"sense_key":"dollar%1:10:00::","synset_id":"dollar.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"dollar%1:23:00::","call_ids":["semeval2013.d012.s002.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"dollar%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":680,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00148,"input_uncached_usd":0.0013599999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.047367458930239}
{"item_id":"semeval2013.d012.s002.t008","gold_sense_keys":["dispute%1:10:00::"],"candidates":[{"index":1,"sense_key":"dispute%1:10:00::","synset_id":"dispute.n.01"},{"index":2,"sense_key":"dispute%1:04:00::","synset_id":"dispute.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"dispute%1:10:00::","call_ids":["semeval2013.d012.s002.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"dispute%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":563,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0012460000000000001,"input_uncached_usd":0.001126,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.129820250091143}
{"item_id":"semeval2013.d012.s002.t009","gold_sense_keys":["bankruptcy%1:26:00::"],"candidates":[{"index":1,"sense_key":"bankruptcy%1:26:01::","synset_id":"bankruptcy.n.01"},{"index":2,"sense_key":"bankruptcy%1:26:00::","synset_id":"bankruptcy.n.02"},{"index":3,"sense_key":"bankruptcy%1:04:00::","synset_id":"bankruptcy.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bankruptcy%1:26:00::","call_ids":["semeval2013.d012.s002.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bankruptcy%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":650,"cached_input_tokens":0,"output_tokens":129,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0025900000000000003,"input_uncached_usd":0.0013,"input_cached_usd":0.0,"output_usd":0.0012900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8019349579699337}
{"item_id":"semeval2013.d012.s003.t000","gold_sense_keys":["issue%1:09:01::"],"candidates":[{"index":1,"sense_key":"issue%1:09:01::","synset_id":"issue.n.01"},{"index":2,"sense_key":"issue%1:10:00::","synset_id":"issue.n.02"},{"index":3,"sense_key":"issue%1:09:00::","synset_id":"topic.n.02"},{"index":4,"sense_key":"issue%1:04:01::","synset_id":"issue.n.04"},{"index":5,"sense_key":"issue%1:21:01::","synset_id":"issue.n.05"},{"index":6,"sense_key":"issue%1:21:00::","synset_id":"return.n.06"},{"index":7,"sense_key":"issue%1:19:00::","synset_id":"consequence.n.01"},{"index":8,"sense_key":"issue%1:18:00::","synset_id":"offspring.n.01"},{"index":9,"sense_key":"issue%1:11:00::","synset_id":"emergence.n.02"},{"index":10,"sense_key":"issue%1:06:00::","synset_id":"exit.n.01"},{"index":11,"sense_key":"issue%1:04:00::","synset_id":"issue.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"issue%1:09:00::","call_ids":["semeval2013.d012.s003.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"issue%1:09:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1276,"cached_input_tokens":0,"output_tokens":149,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0040420000000000005,"input_uncached_usd":0.002552,"input_cached_usd":0.0,"output_usd":0.0014900000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":14.740683500072919}
{"item_id":"semeval2013.d012.s003.t001","gold_sense_keys":["sale%1:04:02::"],"candidates":[{"index":1,"sense_key":"sale%1:04:00::","synset_id":"sale.n.01"},{"index":2,"sense_key":"sale%1:04:02::","synset_id":"sale.n.02"},{"index":3,"sense_key":"sale%1:04:01::","synset_id":"sale.n.03"},{"index":4,"sense_key":"sale%1:26:00::","synset_id":"sale.n.04"},{"index":5,"sense_key":"sale%1:10:00::","synset_id":"sale.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"sale%1:04:02::","call_ids":["semeval2013.d012.s003.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"sale%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":892,"cached_input_tokens":0,"output_tokens":71,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002494,"input_uncached_usd":0.001784,"input_cached_usd":0.0,"output_usd":0.00071,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.4466574999969453}
{"item_id":"semeval2013.d012.s004.t000","gold_sense_keys":["regulator%1:18:00::"],"candidates":[{"index":1,"sense_key":"regulator%1:06:00::","synset_id":"regulator.n.01"},{"index":2,"sense_key":"regulator%1:18:00::","synset_id":"regulator.n.02"},{"index":3,"sense_key":"regulator%1:06:01::","synset_id":"governor.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"regulator%1:18:00::","call_ids":["semeval2013.d012.s004.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"regulator%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":693,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001506,"input_uncached_usd":0.0013859999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0320164590375498}
{"item_id":"semeval2013.d012.s004.t001","gold_sense_keys":["string%1:14:00::"],"candidates":[{"index":1,"sense_key":"string%1:06:00::","synset_id":"string.n.01"},{"index":2,"sense_key":"string%1:06:05::","synset_id":"bowed_stringed_instrument.n.01"},{"index":3,"sense_key":"string%1:06:03::","synset_id":"string.n.03"},{"index":4,"sense_key":"string%1:14:00::","synset_id":"string.n.04"},{"index":5,"sense_key":"string%1:10:00::","synset_id":"string.n.05"},{"index":6,"sense_key":"string%1:06:01::","synset_id":"drawstring.n.01"},{"index":7,"sense_key":"string%1:27:00::","synset_id":"string.n.07"},{"index":8,"sense_key":"string%1:17:00::","synset_id":"string.n.08"},{"index":9,"sense_key":"string%1:06:04::","synset_id":"string.n.09"},{"index":10,"sense_key":"string%1:06:02::","synset_id":"chain.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"string%1:14:00::","call_ids":["semeval2013.d012.s004.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"string%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1110,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0023399999999999996,"input_uncached_usd":0.0022199999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.149841457954608}
{"item_id":"semeval2013.d012.s004.t002","gold_sense_keys":["bank%1:14:00::"],"candidates":[{"index":1,"sense_key":"bank%1:17:01::","synset_id":"bank.n.01"},{"index":2,"sense_key":"bank%1:14:00::","synset_id":"depository_financial_institution.n.01"},{"index":3,"sense_key":"bank%1:17:00::","synset_id":"bank.n.03"},{"index":4,"sense_key":"bank%1:14:01::","synset_id":"bank.n.04"},{"index":5,"sense_key":"bank%1:21:00::","synset_id":"bank.n.05"},{"index":6,"sense_key":"bank%1:21:01::","synset_id":"bank.n.06"},{"index":7,"sense_key":"bank%1:17:02::","synset_id":"bank.n.07"},{"index":8,"sense_key":"bank%1:06:01::","synset_id":"savings_bank.n.02"},{"index":9,"sense_key":"bank%1:06:00::","synset_id":"bank.n.09"},{"index":10,"sense_key":"bank%1:04:00::","synset_id":"bank.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bank%1:14:00::","call_ids":["semeval2013.d012.s004.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bank%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1178,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002476,"input_uncached_usd":0.002356,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1593954591080546}
{"item_id":"semeval2013.d012.s004.t003","gold_sense_keys":["promise%1:10:00::"],"candidates":[{"index":1,"sense_key":"promise%1:10:00::","synset_id":"promise.n.01"},{"index":2,"sense_key":"promise%1:09:00::","synset_id":"promise.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"promise%1:10:00::","call_ids":["semeval2013.d012.s004.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"promise%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":639,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001938,"input_uncached_usd":0.0012779999999999998,"input_cached_usd":0.0,"output_usd":0.0006600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.9046983750304207}
{"item_id":"semeval2013.d012.s005.t000","gold_sense_keys":["value%1:21:00::"],"candidates":[{"index":1,"sense_key":"value%1:09:00::","synset_id":"value.n.01"},{"index":2,"sense_key":"value%1:07:00::","synset_id":"value.n.02"},{"index":3,"sense_key":"value%1:21:00::","synset_id":"value.n.03"},{"index":4,"sense_key":"value%1:07:01::","synset_id":"value.n.04"},{"index":5,"sense_key":"value%1:28:00::","synset_id":"value.n.05"},{"index":6,"sense_key":"value%1:09:01::","synset_id":"value.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"value%1:21:00::","call_ids":["semeval2013.d012.s005.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"value%1:21:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":939,"cached_input_tokens":0,"output_tokens":52,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002398,"input_uncached_usd":0.001878,"input_cached_usd":0.0,"output_usd":0.0005200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4341302920365706}
{"item_id":"semeval2013.d012.s005.t001","gold_sense_keys":["security%1:21:04::"],"candidates":[{"index":1,"sense_key":"security%1:26:00::","synset_id":"security.n.01"},{"index":2,"sense_key":"security%1:21:01::","synset_id":"security.n.02"},{"index":3,"sense_key":"security%1:12:00::","synset_id":"security.n.03"},{"index":4,"sense_key":"security%1:21:04::","synset_id":"security.n.04"},{"index":5,"sense_key":"security%1:21:00::","synset_id":"security.n.05"},{"index":6,"sense_key":"security%1:14:00::","synset_id":"security.n.06"},{"index":7,"sense_key":"security%1:10:00::","synset_id":"security.n.07"},{"index":8,"sense_key":"security%1:06:00::","synset_id":"security_system.n.02"},{"index":9,"sense_key":"security%1:04:00::","synset_id":"security.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"security%1:21:04::","call_ids":["semeval2013.d012.s005.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"security%1:21:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1113,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002346,"input_uncached_usd":0.002226,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8974441669415683}
{"item_id":"semeval2013.d012.s005.t002","gold_sense_keys":["crisis%1:26:00::"],"candidates":[{"index":1,"sense_key":"crisis%1:26:00::","synset_id":"crisis.n.01"},{"index":2,"sense_key":"crisis%1:11:00::","synset_id":"crisis.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"crisis%1:26:00::","call_ids":["semeval2013.d012.s005.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"crisis%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":685,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00149,"input_uncached_usd":0.00137,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8255717919673771}
{"item_id":"semeval2013.d012.s006.t000","gold_sense_keys":["issue%1:09:01::"],"candidates":[{"index":1,"sense_key":"issue%1:09:01::","synset_id":"issue.n.01"},{"index":2,"sense_key":"issue%1:10:00::","synset_id":"issue.n.02"},{"index":3,"sense_key":"issue%1:09:00::","synset_id":"topic.n.02"},{"index":4,"sense_key":"issue%1:04:01::","synset_id":"issue.n.04"},{"index":5,"sense_key":"issue%1:21:01::","synset_id":"issue.n.05"},{"index":6,"sense_key":"issue%1:21:00::","synset_id":"return.n.06"},{"index":7,"sense_key":"issue%1:19:00::","synset_id":"consequence.n.01"},{"index":8,"sense_key":"issue%1:18:00::","synset_id":"offspring.n.01"},{"index":9,"sense_key":"issue%1:11:00::","synset_id":"emergence.n.02"},{"index":10,"sense_key":"issue%1:06:00::","synset_id":"exit.n.01"},{"index":11,"sense_key":"issue%1:04:00::","synset_id":"issue.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"issue%1:09:01::","call_ids":["semeval2013.d012.s006.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"issue%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1339,"cached_input_tokens":0,"output_tokens":141,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004088,"input_uncached_usd":0.002678,"input_cached_usd":0.0,"output_usd":0.00141,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":11.512794958078302}
{"item_id":"semeval2013.d012.s006.t004","gold_sense_keys":["new_york%1:15:00::","new_york%1:15:01::"],"candidates":[{"index":1,"sense_key":"new_york%1:15:01::","synset_id":"new_york.n.01"},{"index":2,"sense_key":"new_york%1:15:00::","synset_id":"new_york.n.02"},{"index":3,"sense_key":"new_york%1:15:02::","synset_id":"new_york.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"new_york%1:15:01::","call_ids":["semeval2013.d012.s006.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"new_york%1:15:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":743,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001606,"input_uncached_usd":0.001486,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1479269159026444}
{"item_id":"semeval2013.d012.s007.t000","gold_sense_keys":["bank%1:14:00::"],"candidates":[{"index":1,"sense_key":"bank%1:17:01::","synset_id":"bank.n.01"},{"index":2,"sense_key":"bank%1:14:00::","synset_id":"depository_financial_institution.n.01"},{"index":3,"sense_key":"bank%1:17:00::","synset_id":"bank.n.03"},{"index":4,"sense_key":"bank%1:14:01::","synset_id":"bank.n.04"},{"index":5,"sense_key":"bank%1:21:00::","synset_id":"bank.n.05"},{"index":6,"sense_key":"bank%1:21:01::","synset_id":"bank.n.06"},{"index":7,"sense_key":"bank%1:17:02::","synset_id":"bank.n.07"},{"index":8,"sense_key":"bank%1:06:01::","synset_id":"savings_bank.n.02"},{"index":9,"sense_key":"bank%1:06:00::","synset_id":"bank.n.09"},{"index":10,"sense_key":"bank%1:04:00::","synset_id":"bank.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bank%1:14:00::","call_ids":["semeval2013.d012.s007.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bank%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1172,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0024639999999999996,"input_uncached_usd":0.0023439999999999997,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9607287920080125}
{"item_id":"semeval2013.d012.s007.t002","gold_sense_keys":["settlement%1:10:00::"],"candidates":[{"index":1,"sense_key":"settlement%1:14:00::","synset_id":"colony.n.01"},{"index":2,"sense_key":"settlement%1:14:01::","synset_id":"village.n.01"},{"index":3,"sense_key":"settlement%1:10:00::","synset_id":"settlement.n.03"},{"index":4,"sense_key":"settlement%1:04:01::","synset_id":"colonization.n.01"},{"index":5,"sense_key":"settlement%1:09:00::","synset_id":"settlement.n.05"},{"index":6,"sense_key":"settlement%1:15:00::","synset_id":"settlement.n.06"},{"index":7,"sense_key":"settlement%1:04:00::","synset_id":"liquidation.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"settlement%1:09:00::","call_ids":["semeval2013.d012.s007.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"settlement%1:09:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":934,"cached_input_tokens":0,"output_tokens":279,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004658000000000001,"input_uncached_usd":0.0018679999999999999,"input_cached_usd":0.0,"output_usd":0.0027900000000000004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.729869124945253}
{"item_id":"semeval2013.d012.s008.t000","gold_sense_keys":["dollar%1:23:00::"],"candidates":[{"index":1,"sense_key":"dollar%1:23:00::","synset_id":"dollar.n.01"},{"index":2,"sense_key":"dollar%1:21:00::","synset_id":"dollar.n.02"},{"index":3,"sense_key":"dollar%1:21:01::","synset_id":"dollar.n.03"},{"index":4,"sense_key":"dollar%1:10:00::","synset_id":"dollar.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"dollar%1:23:00::","call_ids":["semeval2013.d012.s008.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"dollar%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":685,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00149,"input_uncached_usd":0.00137,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":13.663525457959622}
{"item_id":"semeval2013.d012.s009.t001","gold_sense_keys":["responsiveness%1:07:00::"],"candidates":[{"index":1,"sense_key":"responsiveness%1:09:00::","synset_id":"responsiveness.n.01"},{"index":2,"sense_key":"responsiveness%1:07:00::","synset_id":"responsiveness.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"responsiveness%1:07:00::","call_ids":["semeval2013.d012.s009.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"responsiveness%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":590,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0013,"input_uncached_usd":0.0011799999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.573947916040197}
{"item_id":"semeval2013.d012.s009.t002","gold_sense_keys":["bank%1:14:00::"],"candidates":[{"index":1,"sense_key":"bank%1:17:01::","synset_id":"bank.n.01"},{"index":2,"sense_key":"bank%1:14:00::","synset_id":"depository_financial_institution.n.01"},{"index":3,"sense_key":"bank%1:17:00::","synset_id":"bank.n.03"},{"index":4,"sense_key":"bank%1:14:01::","synset_id":"bank.n.04"},{"index":5,"sense_key":"bank%1:21:00::","synset_id":"bank.n.05"},{"index":6,"sense_key":"bank%1:21:01::","synset_id":"bank.n.06"},{"index":7,"sense_key":"bank%1:17:02::","synset_id":"bank.n.07"},{"index":8,"sense_key":"bank%1:06:01::","synset_id":"savings_bank.n.02"},{"index":9,"sense_key":"bank%1:06:00::","synset_id":"bank.n.09"},{"index":10,"sense_key":"bank%1:04:00::","synset_id":"bank.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bank%1:14:00::","call_ids":["semeval2013.d012.s009.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bank%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1129,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002378,"input_uncached_usd":0.002258,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9111299579963088}
{"item_id":"semeval2013.d012.s010.t002","gold_sense_keys":["bankruptcy%1:04:00::"],"candidates":[{"index":1,"sense_key":"bankruptcy%1:26:01::","synset_id":"bankruptcy.n.01"},{"index":2,"sense_key":"bankruptcy%1:26:00::","synset_id":"bankruptcy.n.02"},{"index":3,"sense_key":"bankruptcy%1:04:00::","synset_id":"bankruptcy.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bankruptcy%1:26:00::","call_ids":["semeval2013.d012.s010.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bankruptcy%1:26:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":654,"cached_input_tokens":0,"output_tokens":129,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002598,"input_uncached_usd":0.001308,"input_cached_usd":0.0,"output_usd":0.0012900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.440881500020623}
{"item_id":"semeval2013.d012.s010.t004","gold_sense_keys":["deposit%1:21:01::"],"candidates":[{"index":1,"sense_key":"deposit%1:19:00::","synset_id":"deposit.n.01"},{"index":2,"sense_key":"deposit%1:17:00::","synset_id":"sediment.n.01"},{"index":3,"sense_key":"deposit%1:22:00::","synset_id":"deposition.n.01"},{"index":4,"sense_key":"deposit%1:21:01::","synset_id":"deposit.n.04"},{"index":5,"sense_key":"deposit%1:21:00::","synset_id":"down_payment.n.01"},{"index":6,"sense_key":"deposit%1:21:02::","synset_id":"deposit.n.06"},{"index":7,"sense_key":"deposit%1:10:01::","synset_id":"deposit.n.07"},{"index":8,"sense_key":"deposit%1:06:00::","synset_id":"depository.n.01"},{"index":9,"sense_key":"deposit%1:04:00::","synset_id":"deposit.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"deposit%1:21:01::","call_ids":["semeval2013.d012.s010.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"deposit%1:21:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":896,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001912,"input_uncached_usd":0.001792,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.124011458014138}
{"item_id":"semeval2013.d012.s011.t000","gold_sense_keys":["compensation%1:21:00::"],"candidates":[{"index":1,"sense_key":"compensation%1:21:00::","synset_id":"compensation.n.01"},{"index":2,"sense_key":"compensation%1:22:00::","synset_id":"compensation.n.02"},{"index":3,"sense_key":"compensation%1:04:00::","synset_id":"recompense.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"compensation%1:21:00::","call_ids":["semeval2013.d012.s011.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"compensation%1:21:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":594,"cached_input_tokens":0,"output_tokens":75,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001938,"input_uncached_usd":0.001188,"input_cached_usd":0.0,"output_usd":0.00075,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":14.260876082931645}
{"item_id":"semeval2013.d012.s011.t002","gold_sense_keys":["billion%1:23:00::"],"candidates":[{"index":1,"sense_key":"billion%1:23:01::","synset_id":"billion.n.01"},{"index":2,"sense_key":"billion%1:23:02::","synset_id":"million.n.02"},{"index":3,"sense_key":"billion%1:23:00::","synset_id":"billion.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"billion%1:23:00::","call_ids":["semeval2013.d012.s011.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"billion%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":642,"cached_input_tokens":0,"output_tokens":199,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003274,"input_uncached_usd":0.001284,"input_cached_usd":0.0,"output_usd":0.00199,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":15.726224374957383}
{"item_id":"semeval2013.d012.s012.t000","gold_sense_keys":["settlement%1:10:00::"],"candidates":[{"index":1,"sense_key":"settlement%1:14:00::","synset_id":"colony.n.01"},{"index":2,"sense_key":"settlement%1:14:01::","synset_id":"village.n.01"},{"index":3,"sense_key":"settlement%1:10:00::","synset_id":"settlement.n.03"},{"index":4,"sense_key":"settlement%1:04:01::","synset_id":"colonization.n.01"},{"index":5,"sense_key":"settlement%1:09:00::","synset_id":"settlement.n.05"},{"index":6,"sense_key":"settlement%1:15:00::","synset_id":"settlement.n.06"},{"index":7,"sense_key":"settlement%1:04:00::","synset_id":"liquidation.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"settlement%1:09:00::","call_ids":["semeval2013.d012.s012.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"settlement%1:09:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":854,"cached_input_tokens":0,"output_tokens":147,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0031780000000000003,"input_uncached_usd":0.0017079999999999999,"input_cached_usd":0.0,"output_usd":0.0014700000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.5178365419851616}
{"item_id":"semeval2013.d012.s013.t000","gold_sense_keys":["regulator%1:18:00::"],"candidates":[{"index":1,"sense_key":"regulator%1:06:00::","synset_id":"regulator.n.01"},{"index":2,"sense_key":"regulator%1:18:00::","synset_id":"regulator.n.02"},{"index":3,"sense_key":"regulator%1:06:01::","synset_id":"governor.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"regulator%1:18:00::","call_ids":["semeval2013.d012.s013.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"regulator%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":615,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00135,"input_uncached_usd":0.00123,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1313294579740614}
{"item_id":"semeval2013.d012.s013.t001","gold_sense_keys":["bank%1:14:00::"],"candidates":[{"index":1,"sense_key":"bank%1:17:01::","synset_id":"bank.n.01"},{"index":2,"sense_key":"bank%1:14:00::","synset_id":"depository_financial_institution.n.01"},{"index":3,"sense_key":"bank%1:17:00::","synset_id":"bank.n.03"},{"index":4,"sense_key":"bank%1:14:01::","synset_id":"bank.n.04"},{"index":5,"sense_key":"bank%1:21:00::","synset_id":"bank.n.05"},{"index":6,"sense_key":"bank%1:21:01::","synset_id":"bank.n.06"},{"index":7,"sense_key":"bank%1:17:02::","synset_id":"bank.n.07"},{"index":8,"sense_key":"bank%1:06:01::","synset_id":"savings_bank.n.02"},{"index":9,"sense_key":"bank%1:06:00::","synset_id":"bank.n.09"},{"index":10,"sense_key":"bank%1:04:00::","synset_id":"bank.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bank%1:14:00::","call_ids":["semeval2013.d012.s013.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bank%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1100,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0023199999999999996,"input_uncached_usd":0.0021999999999999997,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.7257804170949385}
{"item_id":"semeval2013.d012.s013.t002","gold_sense_keys":["summer%1:28:00::"],"candidates":[{"index":1,"sense_key":"summer%1:28:00::","synset_id":"summer.n.01"},{"index":2,"sense_key":"summer%1:28:01::","synset_id":"summer.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"summer%1:28:00::","call_ids":["semeval2013.d012.s013.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"summer%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":585,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0012900000000000001,"input_uncached_usd":0.00117,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.7757041249424219}
{"item_id":"semeval2013.d012.s016.t000","gold_sense_keys":["fate%1:11:00::"],"candidates":[{"index":1,"sense_key":"fate%1:11:00::","synset_id":"destiny.n.01"},{"index":2,"sense_key":"fate%1:18:00::","synset_id":"destiny.n.02"},{"index":3,"sense_key":"fate%1:26:00::","synset_id":"fortune.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"fate%1:26:00::","call_ids":["semeval2013.d012.s016.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"fate%1:26:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":639,"cached_input_tokens":0,"output_tokens":108,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002358,"input_uncached_usd":0.0012779999999999998,"input_cached_usd":0.0,"output_usd":0.00108,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.208540291991085}
{"item_id":"semeval2013.d012.s016.t001","gold_sense_keys":["number%1:07:00::"],"candidates":[{"index":1,"sense_key":"number%1:07:00::","synset_id":"number.n.01"},{"index":2,"sense_key":"number%1:23:00::","synset_id":"number.n.02"},{"index":3,"sense_key":"number%1:10:04::","synset_id":"act.n.04"},{"index":4,"sense_key":"number%1:10:03::","synset_id":"phone_number.n.01"},{"index":5,"sense_key":"number%1:10:01::","synset_id":"numeral.n.01"},{"index":6,"sense_key":"number%1:10:00::","synset_id":"issue.n.02"},{"index":7,"sense_key":"number%1:14:00::","synset_id":"number.n.07"},{"index":8,"sense_key":"number%1:10:02::","synset_id":"number.n.08"},{"index":9,"sense_key":"number%1:07:01::","synset_id":"number.n.09"},{"index":10,"sense_key":"number%1:10:05::","synset_id":"number.n.10"},{"index":11,"sense_key":"number%1:06:00::","synset_id":"number.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"number%1:07:00::","call_ids":["semeval2013.d012.s016.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"number%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1089,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0022979999999999997,"input_uncached_usd":0.002178,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9456542079569772}
{"item_id":"semeval2013.d012.s016.t002","gold_sense_keys":["company%1:14:01::"],"candidates":[{"index":1,"sense_key":"company%1:14:01::","synset_id":"company.n.01"},{"index":2,"sense_key":"company%1:14:03::","synset_id":"company.n.02"},{"index":3,"sense_key":"company%1:26:00::","synset_id":"company.n.03"},{"index":4,"sense_key":"company%1:14:02::","synset_id":"company.n.04"},{"index":5,"sense_key":"company%1:18:00::","synset_id":"caller.n.01"},{"index":6,"sense_key":"company%1:14:00::","synset_id":"company.n.06"},{"index":7,"sense_key":"company%1:14:06::","synset_id":"party.n.03"},{"index":8,"sense_key":"company%1:14:04::","synset_id":"ship's_company.n.01"},{"index":9,"sense_key":"company%1:14:05::","synset_id":"company.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"company%1:14:01::","call_ids":["semeval2013.d012.s016.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"company%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":917,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001954,"input_uncached_usd":0.001834,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0083607499254867}
{"item_id":"semeval2013.d012.s017.t002","gold_sense_keys":["height%1:26:00::"],"candidates":[{"index":1,"sense_key":"height%1:07:00::","synset_id":"height.n.01"},{"index":2,"sense_key":"height%1:26:00::","synset_id":"acme.n.01"},{"index":3,"sense_key":"height%1:07:01::","synset_id":"stature.n.02"},{"index":4,"sense_key":"height%1:07:02::","synset_id":"altitude.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"height%1:26:00::","call_ids":["semeval2013.d012.s017.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"height%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":671,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001462,"input_uncached_usd":0.0013419999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8211782090365887}
{"item_id":"semeval2013.d012.s017.t003","gold_sense_keys":["crisis%1:26:00::"],"candidates":[{"index":1,"sense_key":"crisis%1:26:00::","synset_id":"crisis.n.01"},{"index":2,"sense_key":"crisis%1:11:00::","synset_id":"crisis.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"crisis%1:26:00::","call_ids":["semeval2013.d012.s017.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"crisis%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":534,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001188,"input_uncached_usd":0.001068,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":13.685391624923795}
{"item_id":"semeval2013.d012.s018.t000","gold_sense_keys":["bank%1:14:00::"],"candidates":[{"index":1,"sense_key":"bank%1:17:01::","synset_id":"bank.n.01"},{"index":2,"sense_key":"bank%1:14:00::","synset_id":"depository_financial_institution.n.01"},{"index":3,"sense_key":"bank%1:17:00::","synset_id":"bank.n.03"},{"index":4,"sense_key":"bank%1:14:01::","synset_id":"bank.n.04"},{"index":5,"sense_key":"bank%1:21:00::","synset_id":"bank.n.05"},{"index":6,"sense_key":"bank%1:21:01::","synset_id":"bank.n.06"},{"index":7,"sense_key":"bank%1:17:02::","synset_id":"bank.n.07"},{"index":8,"sense_key":"bank%1:06:01::","synset_id":"savings_bank.n.02"},{"index":9,"sense_key":"bank%1:06:00::","synset_id":"bank.n.09"},{"index":10,"sense_key":"bank%1:04:00::","synset_id":"bank.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bank%1:14:00::","call_ids":["semeval2013.d012.s018.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bank%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1083,"cached_input_tokens":0,"output_tokens":120,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003366,"input_uncached_usd":0.002166,"input_cached_usd":0.0,"output_usd":0.0012000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8136170000070706}
{"item_id":"semeval2013.d012.s018.t001","gold_sense_keys":["wind%1:19:01::"],"candidates":[{"index":1,"sense_key":"wind%1:19:00::","synset_id":"wind.n.01"},{"index":2,"sense_key":"wind%1:19:01::","synset_id":"wind.n.02"},{"index":3,"sense_key":"wind%1:04:01::","synset_id":"wind.n.03"},{"index":4,"sense_key":"wind%1:10:01::","synset_id":"wind.n.04"},{"index":5,"sense_key":"wind%1:10:00::","synset_id":"tip.n.03"},{"index":6,"sense_key":"wind%1:06:00::","synset_id":"wind_instrument.n.01"},{"index":7,"sense_key":"wind%1:04:02::","synset_id":"fart.n.01"},{"index":8,"sense_key":"wind%1:04:00::","synset_id":"wind.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"wind%1:19:01::","call_ids":["semeval2013.d012.s018.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"wind%1:19:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":957,"cached_input_tokens":0,"output_tokens":82,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002734,"input_uncached_usd":0.001914,"input_cached_usd":0.0,"output_usd":0.0008200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6981152499793097}
{"item_id":"semeval2013.d012.s018.t002","gold_sense_keys":["usa%1:15:00::"],"candidates":[{"index":1,"sense_key":"usa%1:15:00::","synset_id":"united_states.n.01"},{"index":2,"sense_key":"usa%1:14:00::","synset_id":"united_states_army.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"usa%1:15:00::","call_ids":["semeval2013.d012.s018.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"usa%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":609,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001338,"input_uncached_usd":0.001218,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3985008329618722}
{"item_id":"semeval2013.d012.s018.t003","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["semeval2013.d012.s018.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":692,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0015040000000000001,"input_uncached_usd":0.001384,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.807638458092697}
{"item_id":"semeval2013.d012.s019.t001","gold_sense_keys":["regulator%1:18:00::"],"candidates":[{"index":1,"sense_key":"regulator%1:06:00::","synset_id":"regulator.n.01"},{"index":2,"sense_key":"regulator%1:18:00::","synset_id":"regulator.n.02"},{"index":3,"sense_key":"regulator%1:06:01::","synset_id":"governor.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"regulator%1:18:00::","call_ids":["semeval2013.d012.s019.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"regulator%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":579,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001278,"input_uncached_usd":0.001158,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8260234160115942}
{"item_id":"semeval2013.d012.s019.t002","gold_sense_keys":["number%1:07:00::"],"candidates":[{"index":1,"sense_key":"number%1:07:00::","synset_id":"number.n.01"},{"index":2,"sense_key":"number%1:23:00::","synset_id":"number.n.02"},{"index":3,"sense_key":"number%1:10:04::","synset_id":"act.n.04"},{"index":4,"sense_key":"number%1:10:03::","synset_id":"phone_number.n.01"},{"index":5,"sense_key":"number%1:10:01::","synset_id":"numeral.n.01"},{"index":6,"sense_key":"number%1:10:00::","synset_id":"issue.n.02"},{"index":7,"sense_key":"number%1:14:00::","synset_id":"number.n.07"},{"index":8,"sense_key":"number%1:10:02::","synset_id":"number.n.08"},{"index":9,"sense_key":"number%1:07:01::","synset_id":"number.n.09"},{"index":10,"sense_key":"number%1:10:05::","synset_id":"number.n.10"},{"index":11,"sense_key":"number%1:06:00::","synset_id":"number.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"number%1:07:00::","call_ids":["semeval2013.d012.s019.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"number%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1083,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002286,"input_uncached_usd":0.002166,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1799093330046162}
{"item_id":"semeval2013.d012.s019.t003","gold_sense_keys":["claim%1:04:00::","claim%1:10:00::"],"candidates":[{"index":1,"sense_key":"claim%1:10:00::","synset_id":"claim.n.01"},{"index":2,"sense_key":"claim%1:10:02::","synset_id":"claim.n.02"},{"index":3,"sense_key":"claim%1:04:00::","synset_id":"claim.n.03"},{"index":4,"sense_key":"claim%1:07:02::","synset_id":"claim.n.04"},{"index":5,"sense_key":"claim%1:07:01::","synset_id":"title.n.07"},{"index":6,"sense_key":"claim%1:10:01::","synset_id":"call.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"claim%1:10:00::","call_ids":["semeval2013.d012.s019.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"claim%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":742,"cached_input_tokens":0,"output_tokens":79,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002274,"input_uncached_usd":0.0014839999999999999,"input_cached_usd":0.0,"output_usd":0.00079,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0702399580040947}
{"item_id":"semeval2013.d012.s019.t004","gold_sense_keys":["compensation%1:21:00::"],"candidates":[{"index":1,"sense_key":"compensation%1:21:00::","synset_id":"compensation.n.01"},{"index":2,"sense_key":"compensation%1:22:00::","synset_id":"compensation.n.02"},{"index":3,"sense_key":"compensation%1:04:00::","synset_id":"recompense.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"compensation%1:21:00::","call_ids":["semeval2013.d012.s019.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"compensation%1:21:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":578,"cached_input_tokens":0,"output_tokens":106,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002216,"input_uncached_usd":0.001156,"input_cached_usd":0.0,"output_usd":0.0010600000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6592540419660509}
{"item_id":"semeval2013.d012.s019.t005","gold_sense_keys":["misconduct%1:04:00::"],"candidates":[{"index":1,"sense_key":"misconduct%1:04:01::","synset_id":"misconduct.n.01"},{"index":2,"sense_key":"misconduct%1:04:00::","synset_id":"wrongdoing.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"misconduct%1:04:00::","call_ids":["semeval2013.d012.s019.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"misconduct%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":541,"cached_input_tokens":0,"output_tokens":142,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002502,"input_uncached_usd":0.0010819999999999998,"input_cached_usd":0.0,"output_usd":0.00142,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.7835683749290183}
{"item_id":"semeval2013.d012.s020.t000","gold_sense_keys":["wave%1:04:02::"],"candidates":[{"index":1,"sense_key":"wave%1:11:01::","synset_id":"wave.n.01"},{"index":2,"sense_key":"wave%1:04:02::","synset_id":"wave.n.02"},{"index":3,"sense_key":"wave%1:11:00::","synset_id":"wave.n.03"},{"index":4,"sense_key":"wave%1:11:02::","synset_id":"wave.n.04"},{"index":5,"sense_key":"wave%1:10:01::","synset_id":"wave.n.05"},{"index":6,"sense_key":"wave%1:08:00::","synset_id":"wave.n.06"},{"index":7,"sense_key":"wave%1:25:00::","synset_id":"wave.n.07"},{"index":8,"sense_key":"wave%1:19:00::","synset_id":"wave.n.08"},{"index":9,"sense_key":"wave%1:18:00::","synset_id":"wave.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"wave%1:04:02::","call_ids":["semeval2013.d012.s020.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"wave%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":923,"cached_input_tokens":0,"output_tokens":178,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003626,"input_uncached_usd":0.001846,"input_cached_usd":0.0,"output_usd":0.0017800000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9933136670151725}
{"item_id":"semeval2013.d012.s020.t001","gold_sense_keys":["claim%1:04:00::","claim%1:10:00::"],"candidates":[{"index":1,"sense_key":"claim%1:10:00::","synset_id":"claim.n.01"},{"index":2,"sense_key":"claim%1:10:02::","synset_id":"claim.n.02"},{"index":3,"sense_key":"claim%1:04:00::","synset_id":"claim.n.03"},{"index":4,"sense_key":"claim%1:07:02::","synset_id":"claim.n.04"},{"index":5,"sense_key":"claim%1:07:01::","synset_id":"title.n.07"},{"index":6,"sense_key":"claim%1:10:01::","synset_id":"call.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"claim%1:10:00::","call_ids":["semeval2013.d012.s020.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"claim%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":774,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002208,"input_uncached_usd":0.001548,"input_cached_usd":0.0,"output_usd":0.0006600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":15.18963508401066}
{"item_id":"semeval2013.d012.s021.t000","gold_sense_keys":["bank%1:14:00::"],"candidates":[{"index":1,"sense_key":"bank%1:17:01::","synset_id":"bank.n.01"},{"index":2,"sense_key":"bank%1:14:00::","synset_id":"depository_financial_institution.n.01"},{"index":3,"sense_key":"bank%1:17:00::","synset_id":"bank.n.03"},{"index":4,"sense_key":"bank%1:14:01::","synset_id":"bank.n.04"},{"index":5,"sense_key":"bank%1:21:00::","synset_id":"bank.n.05"},{"index":6,"sense_key":"bank%1:21:01::","synset_id":"bank.n.06"},{"index":7,"sense_key":"bank%1:17:02::","synset_id":"bank.n.07"},{"index":8,"sense_key":"bank%1:06:01::","synset_id":"savings_bank.n.02"},{"index":9,"sense_key":"bank%1:06:00::","synset_id":"bank.n.09"},{"index":10,"sense_key":"bank%1:04:00::","synset_id":"bank.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bank%1:14:00::","call_ids":["semeval2013.d012.s021.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bank%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1138,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0023959999999999997,"input_uncached_usd":0.002276,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4588377500185743}
{"item_id":"semeval2013.d012.s021.t007","gold_sense_keys":["dollar%1:23:00::"],"candidates":[{"index":1,"sense_key":"dollar%1:23:00::","synset_id":"dollar.n.01"},{"index":2,"sense_key":"dollar%1:21:00::","synset_id":"dollar.n.02"},{"index":3,"sense_key":"dollar%1:21:01::","synset_id":"dollar.n.03"},{"index":4,"sense_key":"dollar%1:10:00::","synset_id":"dollar.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"dollar%1:23:00::","call_ids":["semeval2013.d012.s021.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"dollar%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":725,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00157,"input_uncached_usd":0.00145,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.7634395000059158}
{"item_id":"semeval2013.d012.s022.t002","gold_sense_keys":["value%1:21:00::"],"candidates":[{"index":1,"sense_key":"value%1:09:00::","synset_id":"value.n.01"},{"index":2,"sense_key":"value%1:07:00::","synset_id":"value.n.02"},{"index":3,"sense_key":"value%1:21:00::","synset_id":"value.n.03"},{"index":4,"sense_key":"value%1:07:01::","synset_id":"value.n.04"},{"index":5,"sense_key":"value%1:28:00::","synset_id":"value.n.05"},{"index":6,"sense_key":"value%1:09:01::","synset_id":"value.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"value%1:21:00::","call_ids":["semeval2013.d012.s022.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"value%1:21:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":876,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002522,"input_uncached_usd":0.0017519999999999999,"input_cached_usd":0.0,"output_usd":0.0007700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.7487990419613197}
{"item_id":"semeval2013.d012.s022.t003","gold_sense_keys":["dollar%1:23:00::"],"candidates":[{"index":1,"sense_key":"dollar%1:23:00::","synset_id":"dollar.n.01"},{"index":2,"sense_key":"dollar%1:21:00::","synset_id":"dollar.n.02"},{"index":3,"sense_key":"dollar%1:21:01::","synset_id":"dollar.n.03"},{"index":4,"sense_key":"dollar%1:10:00::","synset_id":"dollar.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"dollar%1:23:00::","call_ids":["semeval2013.d012.s022.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"dollar%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":741,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0016020000000000001,"input_uncached_usd":0.001482,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.195938041084446}
{"item_id":"semeval2013.d012.s023.t000","gold_sense_keys":["regulator%1:18:00::"],"candidates":[{"index":1,"sense_key":"regulator%1:06:00::","synset_id":"regulator.n.01"},{"index":2,"sense_key":"regulator%1:18:00::","synset_id":"regulator.n.02"},{"index":3,"sense_key":"regulator%1:06:01::","synset_id":"governor.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"regulator%1:18:00::","call_ids":["semeval2013.d012.s023.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"regulator%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":665,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0014500000000000001,"input_uncached_usd":0.00133,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1987017079954967}
{"item_id":"semeval2013.d012.s023.t001","gold_sense_keys":["bank%1:14:00::"],"candidates":[{"index":1,"sense_key":"bank%1:17:01::","synset_id":"bank.n.01"},{"index":2,"sense_key":"bank%1:14:00::","synset_id":"depository_financial_institution.n.01"},{"index":3,"sense_key":"bank%1:17:00::","synset_id":"bank.n.03"},{"index":4,"sense_key":"bank%1:14:01::","synset_id":"bank.n.04"},{"index":5,"sense_key":"bank%1:21:00::","synset_id":"bank.n.05"},{"index":6,"sense_key":"bank%1:21:01::","synset_id":"bank.n.06"},{"index":7,"sense_key":"bank%1:17:02::","synset_id":"bank.n.07"},{"index":8,"sense_key":"bank%1:06:01::","synset_id":"savings_bank.n.02"},{"index":9,"sense_key":"bank%1:06:00::","synset_id":"bank.n.09"},{"index":10,"sense_key":"bank%1:04:00::","synset_id":"bank.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bank%1:14:00::","call_ids":["semeval2013.d012.s023.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bank%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1150,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00242,"input_uncached_usd":0.0023,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":14.159345709020272}
{"item_id":"semeval2013.d012.s023.t002","gold_sense_keys":["responsibility%1:07:00::"],"candidates":[{"index":1,"sense_key":"responsibility%1:04:00::","synset_id":"duty.n.01"},{"index":2,"sense_key":"responsibility%1:26:00::","synset_id":"province.n.02"},{"index":3,"sense_key":"responsibility%1:07:00::","synset_id":"responsibility.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"responsibility%1:07:00::","call_ids":["semeval2013.d012.s023.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"responsibility%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":764,"cached_input_tokens":0,"output_tokens":291,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004438,"input_uncached_usd":0.0015279999999999998,"input_cached_usd":0.0,"output_usd":0.0029100000000000003,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":12.1666306250263}
{"item_id":"semeval2013.d012.s023.t004","gold_sense_keys":["sum%1:21:00::"],"candidates":[{"index":1,"sense_key":"sum%1:21:00::","synset_id":"sum.n.01"},{"index":2,"sense_key":"sum%1:09:01::","synset_id":"sum.n.02"},{"index":3,"sense_key":"sum%1:14:00::","synset_id":"sum.n.03"},{"index":4,"sense_key":"sum%1:09:00::","synset_id":"kernel.n.03"},{"index":5,"sense_key":"sum%1:06:00::","synset_id":"sum.n.05"},{"index":6,"sense_key":"sum%1:14:01::","synset_id":"union.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"sum%1:21:00::","call_ids":["semeval2013.d012.s023.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"sum%1:21:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":888,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0018960000000000001,"input_uncached_usd":0.001776,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6086796659510583}
{"item_id":"semeval2013.d012.s024.t000","gold_sense_keys":["claim%1:04:00::","claim%1:10:00::"],"candidates":[{"index":1,"sense_key":"claim%1:10:00::","synset_id":"claim.n.01"},{"index":2,"sense_key":"claim%1:10:02::","synset_id":"claim.n.02"},{"index":3,"sense_key":"claim%1:04:00::","synset_id":"claim.n.03"},{"index":4,"sense_key":"claim%1:07:02::","synset_id":"claim.n.04"},{"index":5,"sense_key":"claim%1:07:01::","synset_id":"title.n.07"},{"index":6,"sense_key":"claim%1:10:01::","synset_id":"call.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"claim%1:10:00::","call_ids":["semeval2013.d012.s024.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"claim%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":809,"cached_input_tokens":0,"output_tokens":359,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.005208,"input_uncached_usd":0.0016179999999999999,"input_cached_usd":0.0,"output_usd":0.0035900000000000003,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.937763207941316}
{"item_id":"semeval2013.d012.s024.t001","gold_sense_keys":["intention%1:09:00::"],"candidates":[{"index":1,"sense_key":"intention%1:09:00::","synset_id":"purpose.n.01"},{"index":2,"sense_key":"intention%1:09:01::","synset_id":"intention.n.02"},{"index":3,"sense_key":"intention%1:04:00::","synset_id":"intention.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"intention%1:04:00::","call_ids":["semeval2013.d012.s024.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"intention%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":676,"cached_input_tokens":0,"output_tokens":166,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003012,"input_uncached_usd":0.001352,"input_cached_usd":0.0,"output_usd":0.0016600000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":15.44000662502367}
{"item_id":"semeval2013.d012.s024.t002","gold_sense_keys":["claim%1:04:00::","claim%1:10:00::"],"candidates":[{"index":1,"sense_key":"claim%1:10:00::","synset_id":"claim.n.01"},{"index":2,"sense_key":"claim%1:10:02::","synset_id":"claim.n.02"},{"index":3,"sense_key":"claim%1:04:00::","synset_id":"claim.n.03"},{"index":4,"sense_key":"claim%1:07:02::","synset_id":"claim.n.04"},{"index":5,"sense_key":"claim%1:07:01::","synset_id":"title.n.07"},{"index":6,"sense_key":"claim%1:10:01::","synset_id":"call.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"claim%1:10:00::","call_ids":["semeval2013.d012.s024.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"claim%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":809,"cached_input_tokens":0,"output_tokens":139,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0030080000000000003,"input_uncached_usd":0.0016179999999999999,"input_cached_usd":0.0,"output_usd":0.0013900000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7132216659374535}
{"item_id":"semeval2015.d000.s000.t000","gold_sense_keys":["document%1:10:00::"],"candidates":[{"index":1,"sense_key":"document%1:10:00::","synset_id":"document.n.01"},{"index":2,"sense_key":"document%1:06:00::","synset_id":"document.n.02"},{"index":3,"sense_key":"document%1:21:00::","synset_id":"document.n.03"},{"index":4,"sense_key":"document%1:10:01::","synset_id":"text_file.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"document%1:10:00::","call_ids":["semeval2015.d000.s000.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"document%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":496,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0011120000000000001,"input_uncached_usd":0.000992,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4464054580312222}
{"item_id":"semeval2015.d000.s000.t003","gold_sense_keys":["public%3:00:00::"],"candidates":[{"index":1,"sense_key":"public%3:00:00::","synset_id":"public.a.01"},{"index":2,"sense_key":"public%5:00:00:common:02","synset_id":"public.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"public%3:00:00::","call_ids":["semeval2015.d000.s000.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"public%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":401,"cached_input_tokens":0,"output_tokens":87,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001672,"input_uncached_usd":0.000802,"input_cached_usd":0.0,"output_usd":0.0008700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":14.638668083003722}
{"item_id":"semeval2015.d000.s000.t004","gold_sense_keys":["assessment%1:04:00::"],"candidates":[{"index":1,"sense_key":"assessment%1:09:00::","synset_id":"appraisal.n.01"},{"index":2,"sense_key":"assessment%1:21:00::","synset_id":"assessment.n.02"},{"index":3,"sense_key":"assessment%1:07:00::","synset_id":"assessment.n.03"},{"index":4,"sense_key":"assessment%1:04:00::","synset_id":"judgment.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"assessment%1:04:00::","call_ids":["semeval2015.d000.s000.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"assessment%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":508,"cached_input_tokens":0,"output_tokens":140,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002416,"input_uncached_usd":0.001016,"input_cached_usd":0.0,"output_usd":0.0014000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9025299999630079}
{"item_id":"semeval2015.d000.s000.t005","gold_sense_keys":["report%1:10:03::"],"candidates":[{"index":1,"sense_key":"report%1:10:03::","synset_id":"report.n.01"},{"index":2,"sense_key":"report%1:10:01::","synset_id":"report.n.02"},{"index":3,"sense_key":"report%1:10:00::","synset_id":"report.n.03"},{"index":4,"sense_key":"report%1:11:00::","synset_id":"report.n.04"},{"index":5,"sense_key":"report%1:10:04::","synset_id":"report_card.n.01"},{"index":6,"sense_key":"report%1:10:02::","synset_id":"composition.n.08"},{"index":7,"sense_key":"report%1:09:00::","synset_id":"reputation.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"report%1:10:03::","call_ids":["semeval2015.d000.s000.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"report%1:10:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":755,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00163,"input_uncached_usd":0.0015099999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":14.002864042064175}
{"item_id":"semeval2015.d000.s001.t000","gold_sense_keys":["explain%2:32:00::","explain%2:32:01::"],"candidates":[{"index":1,"sense_key":"explain%2:32:00::","synset_id":"explain.v.01"},{"index":2,"sense_key":"explain%2:32:01::","synset_id":"explain.v.02"},{"index":3,"sense_key":"explain%2:32:02::","synset_id":"excuse.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"explain%2:32:00::","call_ids":["semeval2015.d000.s001.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"explain%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":546,"cached_input_tokens":0,"output_tokens":124,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002332,"input_uncached_usd":0.0010919999999999999,"input_cached_usd":0.0,"output_usd":0.00124,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.161770583014004}
{"item_id":"semeval2015.d000.s001.t001","gold_sense_keys":["committee%1:14:00::"],"candidates":[{"index":1,"sense_key":"committee%1:14:00::","synset_id":"committee.n.01"},{"index":2,"sense_key":"committee%1:14:01::","synset_id":"committee.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"committee%1:14:00::","call_ids":["semeval2015.d000.s001.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"committee%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":476,"cached_input_tokens":0,"output_tokens":83,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0017820000000000002,"input_uncached_usd":0.0009519999999999999,"input_cached_usd":0.0,"output_usd":0.0008300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4110943339765072}
{"item_id":"semeval2015.d000.s001.t003","gold_sense_keys":["product%1:06:01::"],"candidates":[{"index":1,"sense_key":"product%1:06:01::","synset_id":"merchandise.n.01"},{"index":2,"sense_key":"product%1:06:00::","synset_id":"product.n.02"},{"index":3,"sense_key":"product%1:09:00::","synset_id":"product.n.03"},{"index":4,"sense_key":"product%1:27:00::","synset_id":"product.n.04"},{"index":5,"sense_key":"product%1:19:00::","synset_id":"product.n.05"},{"index":6,"sense_key":"product%1:14:00::","synset_id":"intersection.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"product%1:06:01::","call_ids":["semeval2015.d000.s001.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"product%1:06:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":754,"cached_input_tokens":0,"output_tokens":105,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0025580000000000004,"input_uncached_usd":0.001508,"input_cached_usd":0.0,"output_usd":0.0010500000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6604298329912126}
{"item_id":"semeval2015.d000.s001.t004","gold_sense_keys":["human%3:01:01::"],"candidates":[{"index":1,"sense_key":"human%3:01:00::","synset_id":"human.a.01"},{"index":2,"sense_key":"human%3:01:01::","synset_id":"human.a.02"},{"index":3,"sense_key":"human%3:00:00::","synset_id":"human.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"human%3:01:01::","call_ids":["semeval2015.d000.s001.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"human%3:01:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":499,"cached_input_tokens":0,"output_tokens":128,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002278,"input_uncached_usd":0.000998,"input_cached_usd":0.0,"output_usd":0.00128,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8613065420649946}
{"item_id":"semeval2015.d000.s001.t005","gold_sense_keys":["use%1:04:00::","use%1:07:02::"],"candidates":[{"index":1,"sense_key":"use%1:04:00::","synset_id":"use.n.01"},{"index":2,"sense_key":"use%1:07:00::","synset_id":"function.n.02"},{"index":3,"sense_key":"use%1:07:02::","synset_id":"use.n.03"},{"index":4,"sense_key":"use%1:22:00::","synset_id":"consumption.n.03"},{"index":5,"sense_key":"use%1:04:01::","synset_id":"habit.n.02"},{"index":6,"sense_key":"use%1:04:02::","synset_id":"manipulation.n.01"},{"index":7,"sense_key":"use%1:07:03::","synset_id":"use.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%1:04:00::","call_ids":["semeval2015.d000.s001.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":884,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002378,"input_uncached_usd":0.0017679999999999998,"input_cached_usd":0.0,"output_usd":0.0006100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3337540829088539}
{"item_id":"semeval2015.d000.s001.t006","gold_sense_keys":["assess%2:31:00::"],"candidates":[{"index":1,"sense_key":"assess%2:31:00::","synset_id":"measure.v.04"},{"index":2,"sense_key":"assess%2:40:01::","synset_id":"assess.v.02"},{"index":3,"sense_key":"assess%2:40:00::","synset_id":"tax.v.02"},{"index":4,"sense_key":"assess%2:31:01::","synset_id":"assess.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"assess%2:31:00::","call_ids":["semeval2015.d000.s001.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"assess%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":604,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001328,"input_uncached_usd":0.0012079999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8169152080081403}
{"item_id":"semeval2015.d000.s001.t007","gold_sense_keys":["study%1:04:00::"],"candidates":[{"index":1,"sense_key":"study%1:04:00::","synset_id":"survey.n.01"},{"index":2,"sense_key":"study%1:09:03::","synset_id":"study.n.02"},{"index":3,"sense_key":"study%1:10:00::","synset_id":"report.n.01"},{"index":4,"sense_key":"study%1:09:04::","synset_id":"study.n.04"},{"index":5,"sense_key":"study%1:06:00::","synset_id":"study.n.05"},{"index":6,"sense_key":"study%1:09:02::","synset_id":"discipline.n.01"},{"index":7,"sense_key":"study%1:06:01::","synset_id":"sketch.n.01"},{"index":8,"sense_key":"study%1:09:01::","synset_id":"cogitation.n.02"},{"index":9,"sense_key":"study%1:18:00::","synset_id":"study.n.09"},{"index":10,"sense_key":"study%1:10:01::","synset_id":"study.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"study%1:04:00::","call_ids":["semeval2015.d000.s001.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"study%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":944,"cached_input_tokens":0,"output_tokens":142,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003308,"input_uncached_usd":0.001888,"input_cached_usd":0.0,"output_usd":0.00142,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.5724982080282643}
{"item_id":"semeval2015.d000.s001.t008","gold_sense_keys":["perform%2:36:00::"],"candidates":[{"index":1,"sense_key":"perform%2:36:00::","synset_id":"perform.v.01"},{"index":2,"sense_key":"perform%2:41:02::","synset_id":"perform.v.02"},{"index":3,"sense_key":"perform%2:36:01::","synset_id":"perform.v.03"},{"index":4,"sense_key":"perform%2:41:00::","synset_id":"do.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"perform%2:36:00::","call_ids":["semeval2015.d000.s001.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"perform%2:36:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":592,"cached_input_tokens":0,"output_tokens":89,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002074,"input_uncached_usd":0.001184,"input_cached_usd":0.0,"output_usd":0.0008900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5342804579995573}
{"item_id":"semeval2015.d000.s001.t009","gold_sense_keys":["reach%2:38:06::","reach%2:41:00::"],"candidates":[{"index":1,"sense_key":"reach%2:38:01::","synset_id":"reach.v.01"},{"index":2,"sense_key":"reach%2:38:00::","synset_id":"reach.v.02"},{"index":3,"sense_key":"reach%2:35:00::","synset_id":"reach.v.03"},{"index":4,"sense_key":"reach%2:32:00::","synset_id":"reach.v.04"},{"index":5,"sense_key":"reach%2:41:00::","synset_id":"achieve.v.01"},{"index":6,"sense_key":"reach%2:42:00::","synset_id":"reach.v.06"},{"index":7,"sense_key":"reach%2:38:06::","synset_id":"reach.v.07"},{"index":8,"sense_key":"reach%2:40:00::","synset_id":"pass.v.05"},{"index":9,"sense_key":"reach%2:33:00::","synset_id":"strive.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"reach%2:38:01::","call_ids":["semeval2015.d000.s001.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"reach%2:38:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":974,"cached_input_tokens":0,"output_tokens":141,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003358,"input_uncached_usd":0.0019479999999999999,"input_cached_usd":0.0,"output_usd":0.00141,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.4891152089694515}
{"item_id":"semeval2015.d000.s001.t010","gold_sense_keys":["recommendation%1:10:00::"],"candidates":[{"index":1,"sense_key":"recommendation%1:10:00::","synset_id":"recommendation.n.01"},{"index":2,"sense_key":"recommendation%1:10:02::","synset_id":"recommendation.n.02"},{"index":3,"sense_key":"recommendation%1:07:00::","synset_id":"recommendation.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"recommendation%1:10:00::","call_ids":["semeval2015.d000.s001.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"recommendation%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":564,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001248,"input_uncached_usd":0.0011279999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.3227201249683276}
{"item_id":"semeval2015.d000.s001.t011","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":2,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":3,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":4,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":5,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":6,"sense_key":"use%2:41:03::","synset_id":"use.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%2:34:01::","call_ids":["semeval2015.d000.s001.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":742,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001604,"input_uncached_usd":0.0014839999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3667898749699816}
{"item_id":"semeval2015.d000.s001.t012","gold_sense_keys":["medicine%1:06:00::"],"candidates":[{"index":1,"sense_key":"medicine%1:09:00::","synset_id":"medicine.n.01"},{"index":2,"sense_key":"medicine%1:06:00::","synset_id":"medicine.n.02"},{"index":3,"sense_key":"medicine%1:04:00::","synset_id":"medicine.n.03"},{"index":4,"sense_key":"medicine%1:04:02::","synset_id":"music.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"medicine%1:06:00::","call_ids":["semeval2015.d000.s001.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"medicine%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":630,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00138,"input_uncached_usd":0.0012599999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6297094590263441}
{"item_id":"semeval2015.d000.s002.t000","gold_sense_keys":["need%2:34:00::","need%2:34:01::"],"candidates":[{"index":1,"sense_key":"need%2:42:00::","synset_id":"necessitate.v.01"},{"index":2,"sense_key":"need%2:34:00::","synset_id":"want.v.02"},{"index":3,"sense_key":"need%2:34:01::","synset_id":"need.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"need%2:34:01::","call_ids":["semeval2015.d000.s002.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"need%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":565,"cached_input_tokens":0,"output_tokens":123,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00236,"input_uncached_usd":0.00113,"input_cached_usd":0.0,"output_usd":0.0012300000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.5283307910431176}
{"item_id":"semeval2015.d000.s002.t001","gold_sense_keys":["more%3:00:01::"],"candidates":[{"index":1,"sense_key":"more%3:00:01::","synset_id":"more.a.01"},{"index":2,"sense_key":"more%3:00:02::","synset_id":"more.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"more%3:00:01::","call_ids":["semeval2015.d000.s002.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"more%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":536,"cached_input_tokens":0,"output_tokens":48,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001552,"input_uncached_usd":0.001072,"input_cached_usd":0.0,"output_usd":0.00048000000000000007,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4906346671050414}
{"item_id":"semeval2015.d000.s002.t002","gold_sense_keys":["information%1:09:00::"],"candidates":[{"index":1,"sense_key":"information%1:10:00::","synset_id":"information.n.01"},{"index":2,"sense_key":"information%1:09:00::","synset_id":"information.n.02"},{"index":3,"sense_key":"information%1:10:02::","synset_id":"information.n.03"},{"index":4,"sense_key":"information%1:14:00::","synset_id":"data.n.01"},{"index":5,"sense_key":"information%1:07:00::","synset_id":"information.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"information%1:10:00::","call_ids":["semeval2015.d000.s002.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"information%1:10:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":624,"cached_input_tokens":0,"output_tokens":306,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004308,"input_uncached_usd":0.001248,"input_cached_usd":0.0,"output_usd":0.0030600000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":16.481288915965706}
{"item_id":"semeval2015.d000.s002.t003","gold_sense_keys":["medical%3:00:00::"],"candidates":[{"index":1,"sense_key":"medical%3:01:00::","synset_id":"medical.a.01"},{"index":2,"sense_key":"medical%3:00:00::","synset_id":"medical.a.02"},{"index":3,"sense_key":"medical%3:01:02::","synset_id":"aesculapian.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"medical%3:00:00::","call_ids":["semeval2015.d000.s002.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"medical%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":563,"cached_input_tokens":0,"output_tokens":137,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002496,"input_uncached_usd":0.001126,"input_cached_usd":0.0,"output_usd":0.0013700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8897358750691637}
{"item_id":"semeval2015.d000.s002.t004","gold_sense_keys":["condition%1:26:05::"],"candidates":[{"index":1,"sense_key":"condition%1:26:00::","synset_id":"condition.n.01"},{"index":2,"sense_key":"condition%1:10:01::","synset_id":"condition.n.02"},{"index":3,"sense_key":"condition%1:26:01::","synset_id":"condition.n.03"},{"index":4,"sense_key":"condition%1:09:00::","synset_id":"circumstance.n.03"},{"index":5,"sense_key":"condition%1:26:02::","synset_id":"condition.n.05"},{"index":6,"sense_key":"condition%1:26:05::","synset_id":"condition.n.06"},{"index":7,"sense_key":"condition%1:10:02::","synset_id":"condition.n.07"},{"index":8,"sense_key":"condition%1:09:01::","synset_id":"condition.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"condition%1:26:05::","call_ids":["semeval2015.d000.s002.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"condition%1:26:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":884,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001888,"input_uncached_usd":0.0017679999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3690090000163764}
{"item_id":"semeval2015.d000.s002.t005","gold_sense_keys":["treatment%1:04:00::"],"candidates":[{"index":1,"sense_key":"treatment%1:04:00::","synset_id":"treatment.n.01"},{"index":2,"sense_key":"treatment%1:04:01::","synset_id":"treatment.n.02"},{"index":3,"sense_key":"treatment%1:07:00::","synset_id":"treatment.n.03"},{"index":4,"sense_key":"treatment%1:10:00::","synset_id":"discussion.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"treatment%1:04:00::","call_ids":["semeval2015.d000.s002.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"treatment%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":670,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0014600000000000001,"input_uncached_usd":0.00134,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.05877916701138}
{"item_id":"semeval2015.d000.s002.t006","gold_sense_keys":["read%2:31:00::"],"candidates":[{"index":1,"sense_key":"read%2:31:00::","synset_id":"read.v.01"},{"index":2,"sense_key":"read%2:42:00::","synset_id":"read.v.02"},{"index":3,"sense_key":"read%2:31:01::","synset_id":"read.v.03"},{"index":4,"sense_key":"read%2:31:06::","synset_id":"read.v.04"},{"index":5,"sense_key":"read%2:31:02::","synset_id":"read.v.05"},{"index":6,"sense_key":"read%2:31:09::","synset_id":"take.v.06"},{"index":7,"sense_key":"read%2:31:05::","synset_id":"learn.v.04"},{"index":8,"sense_key":"read%2:32:02::","synset_id":"read.v.08"},{"index":9,"sense_key":"read%2:36:05::","synset_id":"read.v.09"},{"index":10,"sense_key":"read%2:31:08::","synset_id":"read.v.10"},{"index":11,"sense_key":"read%2:31:04::","synset_id":"understand.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"read%2:31:00::","call_ids":["semeval2015.d000.s002.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"read%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1068,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0022559999999999998,"input_uncached_usd":0.002136,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":13.86714166693855}
{"item_id":"semeval2015.d000.s002.t008","gold_sense_keys":["leaflet%1:10:00::"],"candidates":[{"index":1,"sense_key":"leaflet%1:08:00::","synset_id":"cusp.n.02"},{"index":2,"sense_key":"leaflet%1:20:00::","synset_id":"leaflet.n.02"},{"index":3,"sense_key":"leaflet%1:10:00::","synset_id":"booklet.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"leaflet%1:10:00::","call_ids":["semeval2015.d000.s002.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"leaflet%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":538,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001196,"input_uncached_usd":0.001076,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.7766604999778792}
{"item_id":"semeval2015.d000.s002.t010","gold_sense_keys":["part%1:09:00::","part%1:24:00::"],"candidates":[{"index":1,"sense_key":"part%1:24:00::","synset_id":"part.n.01"},{"index":2,"sense_key":"part%1:06:00::","synset_id":"part.n.02"},{"index":3,"sense_key":"part%1:17:00::","synset_id":"part.n.03"},{"index":4,"sense_key":"part%1:09:02::","synset_id":"part.n.04"},{"index":5,"sense_key":"part%1:15:00::","synset_id":"region.n.01"},{"index":6,"sense_key":"part%1:04:01::","synset_id":"function.n.03"},{"index":7,"sense_key":"part%1:09:01::","synset_id":"character.n.04"},{"index":8,"sense_key":"part%1:21:00::","synset_id":"share.n.01"},{"index":9,"sense_key":"part%1:09:00::","synset_id":"part.n.09"},{"index":10,"sense_key":"part%1:08:01::","synset_id":"part.n.10"},{"index":11,"sense_key":"part%1:10:00::","synset_id":"part.n.11"},{"index":12,"sense_key":"part%1:04:03::","synset_id":"contribution.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"part%1:09:00::","call_ids":["semeval2015.d000.s002.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"part%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1206,"cached_input_tokens":0,"output_tokens":141,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003822,"input_uncached_usd":0.002412,"input_cached_usd":0.0,"output_usd":0.00141,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.812735665938817}
{"item_id":"semeval2015.d000.s002.t011","gold_sense_keys":["contact%2:32:00::"],"candidates":[{"index":1,"sense_key":"contact%2:32:00::","synset_id":"reach.v.04"},{"index":2,"sense_key":"contact%2:35:03::","synset_id":"touch.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"contact%2:32:00::","call_ids":["semeval2015.d000.s002.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"contact%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":544,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001208,"input_uncached_usd":0.001088,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0698803330305964}
{"item_id":"semeval2015.d000.s002.t012","gold_sense_keys":["doctor%1:18:00::"],"candidates":[{"index":1,"sense_key":"doctor%1:18:00::","synset_id":"doctor.n.01"},{"index":2,"sense_key":"doctor%1:18:02::","synset_id":"doctor_of_the_church.n.01"},{"index":3,"sense_key":"doctor%1:04:00::","synset_id":"doctor.n.03"},{"index":4,"sense_key":"doctor%1:18:01::","synset_id":"doctor.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"doctor%1:18:00::","call_ids":["semeval2015.d000.s002.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"doctor%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":738,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001596,"input_uncached_usd":0.0014759999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4937134999781847}
{"item_id":"semeval2015.d000.s003.t000","gold_sense_keys":["want%2:37:00::"],"candidates":[{"index":1,"sense_key":"want%2:37:00::","synset_id":"desire.v.01"},{"index":2,"sense_key":"want%2:34:00::","synset_id":"want.v.02"},{"index":3,"sense_key":"want%2:35:00::","synset_id":"want.v.03"},{"index":4,"sense_key":"want%2:31:01::","synset_id":"want.v.04"},{"index":5,"sense_key":"want%2:42:01::","synset_id":"want.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"want%2:37:00::","call_ids":["semeval2015.d000.s003.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"want%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":670,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0014600000000000001,"input_uncached_usd":0.00134,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9983698750147596}
{"item_id":"semeval2015.d000.s003.t001","gold_sense_keys":["more%3:00:01::"],"candidates":[{"index":1,"sense_key":"more%3:00:01::","synset_id":"more.a.01"},{"index":2,"sense_key":"more%3:00:02::","synset_id":"more.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"more%3:00:01::","call_ids":["semeval2015.d000.s003.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"more%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":543,"cached_input_tokens":0,"output_tokens":45,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001536,"input_uncached_usd":0.001086,"input_cached_usd":0.0,"output_usd":0.00045000000000000004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":14.362911790958606}
{"item_id":"semeval2015.d000.s003.t002","gold_sense_keys":["information%1:09:00::","information%1:14:00::"],"candidates":[{"index":1,"sense_key":"information%1:10:00::","synset_id":"information.n.01"},{"index":2,"sense_key":"information%1:09:00::","synset_id":"information.n.02"},{"index":3,"sense_key":"information%1:10:02::","synset_id":"information.n.03"},{"index":4,"sense_key":"information%1:14:00::","synset_id":"data.n.01"},{"index":5,"sense_key":"information%1:07:00::","synset_id":"information.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"information%1:10:00::","call_ids":["semeval2015.d000.s003.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"information%1:10:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":631,"cached_input_tokens":0,"output_tokens":243,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003692,"input_uncached_usd":0.0012619999999999999,"input_cached_usd":0.0,"output_usd":0.0024300000000000003,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.5962382920552045}
{"item_id":"semeval2015.d000.s003.t003","gold_sense_keys":["basis%1:09:00::"],"candidates":[{"index":1,"sense_key":"basis%1:24:00::","synset_id":"footing.n.02"},{"index":2,"sense_key":"basis%1:09:00::","synset_id":"basis.n.02"},{"index":3,"sense_key":"basis%1:24:01::","synset_id":"basis.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"basis%1:09:00::","call_ids":["semeval2015.d000.s003.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"basis%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":620,"cached_input_tokens":0,"output_tokens":76,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002,"input_uncached_usd":0.00124,"input_cached_usd":0.0,"output_usd":0.00076,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.477392042055726}
{"item_id":"semeval2015.d000.s003.t004","gold_sense_keys":["recommendation%1:10:00::"],"candidates":[{"index":1,"sense_key":"recommendation%1:10:00::","synset_id":"recommendation.n.01"},{"index":2,"sense_key":"recommendation%1:10:02::","synset_id":"recommendation.n.02"},{"index":3,"sense_key":"recommendation%1:07:00::","synset_id":"recommendation.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"recommendation%1:10:00::","call_ids":["semeval2015.d000.s003.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"recommendation%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":611,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001342,"input_uncached_usd":0.001222,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4926660420605913}
{"item_id":"semeval2015.d000.s003.t005","gold_sense_keys":["read%2:31:00::"],"candidates":[{"index":1,"sense_key":"read%2:31:00::","synset_id":"read.v.01"},{"index":2,"sense_key":"read%2:42:00::","synset_id":"read.v.02"},{"index":3,"sense_key":"read%2:31:01::","synset_id":"read.v.03"},{"index":4,"sense_key":"read%2:31:06::","synset_id":"read.v.04"},{"index":5,"sense_key":"read%2:31:02::","synset_id":"read.v.05"},{"index":6,"sense_key":"read%2:31:09::","synset_id":"take.v.06"},{"index":7,"sense_key":"read%2:31:05::","synset_id":"learn.v.04"},{"index":8,"sense_key":"read%2:32:02::","synset_id":"read.v.08"},{"index":9,"sense_key":"read%2:36:05::","synset_id":"read.v.09"},{"index":10,"sense_key":"read%2:31:08::","synset_id":"read.v.10"},{"index":11,"sense_key":"read%2:31:04::","synset_id":"understand.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"read%2:31:00::","call_ids":["semeval2015.d000.s003.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"read%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1075,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00227,"input_uncached_usd":0.00215,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8720383329782635}
{"item_id":"semeval2015.d000.s003.t007","gold_sense_keys":["discussion%1:10:02::"],"candidates":[{"index":1,"sense_key":"discussion%1:10:02::","synset_id":"discussion.n.01"},{"index":2,"sense_key":"discussion%1:10:00::","synset_id":"discussion.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"discussion%1:10:02::","call_ids":["semeval2015.d000.s003.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"discussion%1:10:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":563,"cached_input_tokens":0,"output_tokens":109,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002216,"input_uncached_usd":0.001126,"input_cached_usd":0.0,"output_usd":0.00109,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8717196659417823}
{"item_id":"semeval2015.d000.s003.t009","gold_sense_keys":["part%1:09:00::","part%1:24:00::"],"candidates":[{"index":1,"sense_key":"part%1:24:00::","synset_id":"part.n.01"},{"index":2,"sense_key":"part%1:06:00::","synset_id":"part.n.02"},{"index":3,"sense_key":"part%1:17:00::","synset_id":"part.n.03"},{"index":4,"sense_key":"part%1:09:02::","synset_id":"part.n.04"},{"index":5,"sense_key":"part%1:15:00::","synset_id":"region.n.01"},{"index":6,"sense_key":"part%1:04:01::","synset_id":"function.n.03"},{"index":7,"sense_key":"part%1:09:01::","synset_id":"character.n.04"},{"index":8,"sense_key":"part%1:21:00::","synset_id":"share.n.01"},{"index":9,"sense_key":"part%1:09:00::","synset_id":"part.n.09"},{"index":10,"sense_key":"part%1:08:01::","synset_id":"part.n.10"},{"index":11,"sense_key":"part%1:10:00::","synset_id":"part.n.11"},{"index":12,"sense_key":"part%1:04:03::","synset_id":"contribution.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"part%1:09:00::","call_ids":["semeval2015.d000.s003.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"part%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1213,"cached_input_tokens":0,"output_tokens":174,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0041659999999999996,"input_uncached_usd":0.0024259999999999998,"input_cached_usd":0.0,"output_usd":0.0017400000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.14378908299841}
{"item_id":"semeval2015.d000.s005.t000","gold_sense_keys":["powder%1:06:01::","powder%1:27:00::"],"candidates":[{"index":1,"sense_key":"powder%1:27:00::","synset_id":"powder.n.01"},{"index":2,"sense_key":"powder%1:27:01::","synset_id":"gunpowder.n.01"},{"index":3,"sense_key":"powder%1:06:01::","synset_id":"powder.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"powder%1:27:00::","call_ids":["semeval2015.d000.s005.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"powder%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":649,"cached_input_tokens":0,"output_tokens":195,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003248,"input_uncached_usd":0.001298,"input_cached_usd":0.0,"output_usd":0.0019500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.026170084020123}
{"item_id":"semeval2015.d000.s005.t002","gold_sense_keys":["solution%1:27:00::"],"candidates":[{"index":1,"sense_key":"solution%1:27:00::","synset_id":"solution.n.01"},{"index":2,"sense_key":"solution%1:10:00::","synset_id":"solution.n.02"},{"index":3,"sense_key":"solution%1:09:00::","synset_id":"solution.n.03"},{"index":4,"sense_key":"solution%1:14:00::","synset_id":"solution.n.04"},{"index":5,"sense_key":"solution%1:04:00::","synset_id":"solution.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"solution%1:27:00::","call_ids":["semeval2015.d000.s005.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"solution%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":743,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001606,"input_uncached_usd":0.001486,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":14.31340449990239}
{"item_id":"semeval2015.d000.s005.t003","gold_sense_keys":["infusion%1:04:00::"],"candidates":[{"index":1,"sense_key":"infusion%1:27:00::","synset_id":"infusion.n.01"},{"index":2,"sense_key":"infusion%1:22:00::","synset_id":"infusion.n.02"},{"index":3,"sense_key":"infusion%1:04:00::","synset_id":"infusion.n.03"},{"index":4,"sense_key":"infusion%1:04:01::","synset_id":"infusion.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"infusion%1:04:00::","call_ids":["semeval2015.d000.s005.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"infusion%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":699,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001518,"input_uncached_usd":0.001398,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.494587374967523}
{"item_id":"semeval2015.d000.s005.t004","gold_sense_keys":["drip%2:35:10::"],"candidates":[{"index":1,"sense_key":"drip%2:38:00::","synset_id":"drip.v.01"},{"index":2,"sense_key":"drip%2:35:10::","synset_id":"dribble.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"drip%2:35:10::","call_ids":["semeval2015.d000.s005.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"drip%2:35:10::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":546,"cached_input_tokens":0,"output_tokens":249,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003582,"input_uncached_usd":0.0010919999999999999,"input_cached_usd":0.0,"output_usd":0.00249,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":15.582235166919418}
{"item_id":"semeval2015.d000.s005.t005","gold_sense_keys":["vein%1:08:00::"],"candidates":[{"index":1,"sense_key":"vein%1:08:00::","synset_id":"vein.n.01"},{"index":2,"sense_key":"vein%1:10:00::","synset_id":"vein.n.02"},{"index":3,"sense_key":"vein%1:20:00::","synset_id":"vein.n.03"},{"index":4,"sense_key":"vein%1:17:00::","synset_id":"vein.n.04"},{"index":5,"sense_key":"vein%1:05:00::","synset_id":"vein.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"vein%1:08:00::","call_ids":["semeval2015.d000.s005.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"vein%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":748,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001616,"input_uncached_usd":0.001496,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.7173619590466842}
{"item_id":"semeval2015.d000.s006.t000","gold_sense_keys":["contain%2:42:00::"],"candidates":[{"index":1,"sense_key":"contain%2:42:00::","synset_id":"incorporate.v.02"},{"index":2,"sense_key":"contain%2:42:13::","synset_id":"hold.v.11"},{"index":3,"sense_key":"contain%2:41:00::","synset_id":"control.v.02"},{"index":4,"sense_key":"contain%2:42:06::","synset_id":"contain.v.04"},{"index":5,"sense_key":"contain%2:42:14::","synset_id":"contain.v.05"},{"index":6,"sense_key":"contain%2:33:00::","synset_id":"check.v.18"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"contain%2:42:00::","call_ids":["semeval2015.d000.s006.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"contain%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":865,"cached_input_tokens":0,"output_tokens":86,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0025900000000000003,"input_uncached_usd":0.00173,"input_cached_usd":0.0,"output_usd":0.0008600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.3759405419696122}
{"item_id":"semeval2015.d000.s006.t001","gold_sense_keys":["active%3:00:07::"],"candidates":[{"index":1,"sense_key":"active%3:00:02::","synset_id":"active.a.01"},{"index":2,"sense_key":"active%5:00:00:operational:00","synset_id":"active.s.01"},{"index":3,"sense_key":"active%3:00:03::","synset_id":"active.a.03"},{"index":4,"sense_key":"active%5:00:00:involved:00","synset_id":"active.s.02"},{"index":5,"sense_key":"active%3:00:01::","synset_id":"active.a.05"},{"index":6,"sense_key":"active%3:00:07::","synset_id":"active.a.06"},{"index":7,"sense_key":"active%3:00:06::","synset_id":"active.a.07"},{"index":8,"sense_key":"active%5:00:00:existent:00","synset_id":"active.s.03"},{"index":9,"sense_key":"active%3:00:10::","synset_id":"active.a.09"},{"index":10,"sense_key":"active%3:00:09::","synset_id":"active.a.10"},{"index":11,"sense_key":"active%3:00:00::","synset_id":"active.a.11"},{"index":12,"sense_key":"active%3:00:05::","synset_id":"active.a.12"},{"index":13,"sense_key":"active%3:00:04::","synset_id":"active.a.13"},{"index":14,"sense_key":"active%3:00:08::","synset_id":"active.a.14"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"active%3:00:07::","call_ids":["semeval2015.d000.s006.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"active%3:00:07::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1187,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0024939999999999997,"input_uncached_usd":0.002374,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4716867079259828}
{"item_id":"semeval2015.d000.s006.t002","gold_sense_keys":["substance%1:03:01::"],"candidates":[{"index":1,"sense_key":"substance%1:03:00::","synset_id":"substance.n.01"},{"index":2,"sense_key":"substance%1:09:01::","synset_id":"kernel.n.03"},{"index":3,"sense_key":"substance%1:09:00::","synset_id":"meaning.n.02"},{"index":4,"sense_key":"substance%1:27:00::","synset_id":"substance.n.04"},{"index":5,"sense_key":"substance%1:21:00::","synset_id":"means.n.03"},{"index":6,"sense_key":"substance%1:10:00::","synset_id":"message.n.02"},{"index":7,"sense_key":"substance%1:03:01::","synset_id":"substance.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"substance%1:03:01::","call_ids":["semeval2015.d000.s006.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"substance%1:03:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":837,"cached_input_tokens":0,"output_tokens":98,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002654,"input_uncached_usd":0.001674,"input_cached_usd":0.0,"output_usd":0.0009800000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.3593324999092147}
{"item_id":"semeval2015.d000.s007.t000","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":2,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":3,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":4,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":5,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":6,"sense_key":"use%2:41:03::","synset_id":"use.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%2:34:01::","call_ids":["semeval2015.d000.s007.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":818,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001756,"input_uncached_usd":0.0016359999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":16.02192137495149}
{"item_id":"semeval2015.d000.s008.t000","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":2,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":3,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":4,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":5,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":6,"sense_key":"use%2:41:03::","synset_id":"use.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%2:34:01::","call_ids":["semeval2015.d000.s008.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":896,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001912,"input_uncached_usd":0.001792,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":14.011444333009422}
{"item_id":"semeval2015.d000.s008.t001","gold_sense_keys":["treat%2:29:00::"],"candidates":[{"index":1,"sense_key":"treat%2:41:00::","synset_id":"treat.v.01"},{"index":2,"sense_key":"treat%2:30:01::","synset_id":"process.v.01"},{"index":3,"sense_key":"treat%2:29:00::","synset_id":"treat.v.03"},{"index":4,"sense_key":"treat%2:32:00::","synset_id":"cover.v.05"},{"index":5,"sense_key":"treat%2:40:00::","synset_id":"treat.v.05"},{"index":6,"sense_key":"treat%2:34:00::","synset_id":"regale.v.01"},{"index":7,"sense_key":"treat%2:32:03::","synset_id":"treat.v.07"},{"index":8,"sense_key":"treat%2:31:00::","synset_id":"treat.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"treat%2:29:00::","call_ids":["semeval2015.d000.s008.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"treat%2:29:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1035,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0021899999999999997,"input_uncached_usd":0.00207,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0069367079995573}
{"item_id":"semeval2015.d000.s008.t002","gold_sense_keys":["type%1:09:00::"],"candidates":[{"index":1,"sense_key":"type%1:09:00::","synset_id":"type.n.01"},{"index":2,"sense_key":"type%1:18:00::","synset_id":"character.n.05"},{"index":3,"sense_key":"type%1:14:01::","synset_id":"type.n.03"},{"index":4,"sense_key":"type%1:10:01::","synset_id":"type.n.04"},{"index":5,"sense_key":"type%1:10:00::","synset_id":"type.n.05"},{"index":6,"sense_key":"type%1:06:00::","synset_id":"type.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"type%1:09:00::","call_ids":["semeval2015.d000.s008.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"type%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":835,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00179,"input_uncached_usd":0.0016699999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.09916979202535}
{"item_id":"semeval2015.d000.s008.t003","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["semeval2015.d000.s008.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":828,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001776,"input_uncached_usd":0.001656,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.250938625074923}
{"item_id":"semeval2015.d000.s008.t006","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["semeval2015.d000.s008.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":828,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001776,"input_uncached_usd":0.001656,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0213716250145808}
{"item_id":"semeval2015.d000.s008.t007","gold_sense_keys":["lining%1:06:00::"],"candidates":[{"index":1,"sense_key":"lining%1:06:00::","synset_id":"lining.n.01"},{"index":2,"sense_key":"lining%1:06:01::","synset_id":"liner.n.03"},{"index":3,"sense_key":"lining%1:04:01::","synset_id":"lining.n.03"},{"index":4,"sense_key":"lining%1:04:00::","synset_id":"lining.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"lining%1:06:00::","call_ids":["semeval2015.d000.s008.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"lining%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":691,"cached_input_tokens":0,"output_tokens":95,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002332,"input_uncached_usd":0.001382,"input_cached_usd":0.0,"output_usd":0.0009500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6435168329626322}
{"item_id":"semeval2015.d000.s008.t010","gold_sense_keys":["cause%2:36:00::"],"candidates":[{"index":1,"sense_key":"cause%2:36:00::","synset_id":"cause.v.01"},{"index":2,"sense_key":"cause%2:32:00::","synset_id":"induce.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cause%2:36:00::","call_ids":["semeval2015.d000.s008.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cause%2:36:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":663,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001446,"input_uncached_usd":0.001326,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9223487919662148}
{"item_id":"semeval2015.d000.s009.t000","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":2,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":3,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":4,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":5,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":6,"sense_key":"use%2:41:03::","synset_id":"use.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%2:34:01::","call_ids":["semeval2015.d000.s009.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":905,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00193,"input_uncached_usd":0.00181,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1732060000067577}
{"item_id":"semeval2015.d000.s009.t003","gold_sense_keys":["medicine%1:06:00::"],"candidates":[{"index":1,"sense_key":"medicine%1:09:00::","synset_id":"medicine.n.01"},{"index":2,"sense_key":"medicine%1:06:00::","synset_id":"medicine.n.02"},{"index":3,"sense_key":"medicine%1:04:00::","synset_id":"medicine.n.03"},{"index":4,"sense_key":"medicine%1:04:02::","synset_id":"music.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"medicine%1:06:00::","call_ids":["semeval2015.d000.s009.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"medicine%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":793,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001706,"input_uncached_usd":0.001586,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":13.764086415991187}
{"item_id":"semeval2015.d000.s009.t004","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["semeval2015.d000.s009.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":837,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001794,"input_uncached_usd":0.001674,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9271862499881536}
{"item_id":"semeval2015.d000.s009.t005","gold_sense_keys":["remove%2:30:00::"],"candidates":[{"index":1,"sense_key":"remove%2:30:00::","synset_id":"remove.v.01"},{"index":2,"sense_key":"remove%2:41:00::","synset_id":"remove.v.02"},{"index":3,"sense_key":"remove%2:40:00::","synset_id":"get_rid_of.v.01"},{"index":4,"sense_key":"remove%2:41:02::","synset_id":"take_out.v.01"},{"index":5,"sense_key":"remove%2:38:00::","synset_id":"remove.v.05"},{"index":6,"sense_key":"remove%2:30:01::","synset_id":"absent.v.01"},{"index":7,"sense_key":"remove%2:41:01::","synset_id":"murder.v.01"},{"index":8,"sense_key":"remove%2:30:02::","synset_id":"remove.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"remove%2:30:00::","call_ids":["semeval2015.d000.s009.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"remove%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1029,"cached_input_tokens":0,"output_tokens":144,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0034980000000000002,"input_uncached_usd":0.002058,"input_cached_usd":0.0,"output_usd":0.00144,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8623371249996126}
{"item_id":"semeval2015.d000.s009.t006","gold_sense_keys":["surgery%1:04:01::"],"candidates":[{"index":1,"sense_key":"surgery%1:09:00::","synset_id":"surgery.n.01"},{"index":2,"sense_key":"surgery%1:06:01::","synset_id":"surgery.n.02"},{"index":3,"sense_key":"surgery%1:06:00::","synset_id":"operating_room.n.01"},{"index":4,"sense_key":"surgery%1:04:01::","synset_id":"operation.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"surgery%1:04:01::","call_ids":["semeval2015.d000.s009.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"surgery%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":848,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002306,"input_uncached_usd":0.001696,"input_cached_usd":0.0,"output_usd":0.0006100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4997883749892935}
{"item_id":"semeval2015.d000.s009.t007","gold_sense_keys":["alone%5:00:00:exclusive:00"],"candidates":[{"index":1,"sense_key":"alone%5:00:00:unsocial:00","synset_id":"alone.s.01"},{"index":2,"sense_key":"alone%5:00:00:unaccompanied:00","synset_id":"alone.s.02"},{"index":3,"sense_key":"alone%5:00:00:exclusive:00","synset_id":"alone.s.03"},{"index":4,"sense_key":"alone%5:00:00:incomparable:00","synset_id":"alone.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"alone%5:00:00:exclusive:00","call_ids":["semeval2015.d000.s009.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"alone%5:00:00:exclusive:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":817,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001754,"input_uncached_usd":0.0016339999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3635912080062553}
{"item_id":"semeval2015.d000.s009.t009","gold_sense_keys":["spread%2:35:07::"],"candidates":[{"index":1,"sense_key":"spread%2:35:00::","synset_id":"spread.v.01"},{"index":2,"sense_key":"spread%2:35:07::","synset_id":"spread.v.02"},{"index":3,"sense_key":"spread%2:42:00::","synset_id":"spread.v.03"},{"index":4,"sense_key":"spread%2:35:03::","synset_id":"unfold.v.04"},{"index":5,"sense_key":"spread%2:32:00::","synset_id":"circulate.v.02"},{"index":6,"sense_key":"spread%2:32:02::","synset_id":"go_around.v.02"},{"index":7,"sense_key":"spread%2:38:00::","synset_id":"spread.v.07"},{"index":8,"sense_key":"spread%2:38:02::","synset_id":"diffuse.v.01"},{"index":9,"sense_key":"spread%2:35:12::","synset_id":"spread.v.09"},{"index":10,"sense_key":"spread%2:35:13::","synset_id":"spread.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"spread%2:35:07::","call_ids":["semeval2015.d000.s009.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"spread%2:35:07::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1100,"cached_input_tokens":0,"output_tokens":75,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0029499999999999995,"input_uncached_usd":0.0021999999999999997,"input_cached_usd":0.0,"output_usd":0.00075,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0739163339603692}
{"item_id":"semeval2015.d000.s009.t010","gold_sense_keys":["likely%3:00:00::"],"candidates":[{"index":1,"sense_key":"likely%3:00:00::","synset_id":"likely.a.01"},{"index":2,"sense_key":"likely%3:00:04::","synset_id":"probable.a.01"},{"index":3,"sense_key":"likely%5:00:00:prospective:00","synset_id":"likely.s.01"},{"index":4,"sense_key":"likely%5:00:00:credible:00","synset_id":"likely.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"likely%3:00:00::","call_ids":["semeval2015.d000.s009.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"likely%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":743,"cached_input_tokens":0,"output_tokens":71,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002196,"input_uncached_usd":0.001486,"input_cached_usd":0.0,"output_usd":0.00071,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.424170083948411}
{"item_id":"semeval2015.d000.s009.t011","gold_sense_keys":["spread%2:35:07::"],"candidates":[{"index":1,"sense_key":"spread%2:35:00::","synset_id":"spread.v.01"},{"index":2,"sense_key":"spread%2:35:07::","synset_id":"spread.v.02"},{"index":3,"sense_key":"spread%2:42:00::","synset_id":"spread.v.03"},{"index":4,"sense_key":"spread%2:35:03::","synset_id":"unfold.v.04"},{"index":5,"sense_key":"spread%2:32:00::","synset_id":"circulate.v.02"},{"index":6,"sense_key":"spread%2:32:02::","synset_id":"go_around.v.02"},{"index":7,"sense_key":"spread%2:38:00::","synset_id":"spread.v.07"},{"index":8,"sense_key":"spread%2:38:02::","synset_id":"diffuse.v.01"},{"index":9,"sense_key":"spread%2:35:12::","synset_id":"spread.v.09"},{"index":10,"sense_key":"spread%2:35:13::","synset_id":"spread.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"spread%2:35:07::","call_ids":["semeval2015.d000.s009.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"spread%2:35:07::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1100,"cached_input_tokens":0,"output_tokens":80,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0029999999999999996,"input_uncached_usd":0.0021999999999999997,"input_cached_usd":0.0,"output_usd":0.0008,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4814836661098525}
{"item_id":"semeval2015.d000.s009.t012","gold_sense_keys":["easily%4:02:01::"],"candidates":[{"index":1,"sense_key":"easily%4:02:01::","synset_id":"easily.r.01"},{"index":2,"sense_key":"easily%4:02:06::","synset_id":"easily.r.02"},{"index":3,"sense_key":"easily%4:02:05::","synset_id":"well.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"easily%4:02:01::","call_ids":["semeval2015.d000.s009.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"easily%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":690,"cached_input_tokens":0,"output_tokens":109,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00247,"input_uncached_usd":0.00138,"input_cached_usd":0.0,"output_usd":0.00109,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.7014483750099316}
{"item_id":"semeval2015.d000.s009.t013","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":2,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":3,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"other%3:00:00::","call_ids":["semeval2015.d000.s009.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":753,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001626,"input_uncached_usd":0.001506,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1211173329502344}
{"item_id":"semeval2015.d000.s009.t014","gold_sense_keys":["part%1:15:00::","part%1:17:00::","part%1:24:00::"],"candidates":[{"index":1,"sense_key":"part%1:24:00::","synset_id":"part.n.01"},{"index":2,"sense_key":"part%1:06:00::","synset_id":"part.n.02"},{"index":3,"sense_key":"part%1:17:00::","synset_id":"part.n.03"},{"index":4,"sense_key":"part%1:09:02::","synset_id":"part.n.04"},{"index":5,"sense_key":"part%1:15:00::","synset_id":"region.n.01"},{"index":6,"sense_key":"part%1:04:01::","synset_id":"function.n.03"},{"index":7,"sense_key":"part%1:09:01::","synset_id":"character.n.04"},{"index":8,"sense_key":"part%1:21:00::","synset_id":"share.n.01"},{"index":9,"sense_key":"part%1:09:00::","synset_id":"part.n.09"},{"index":10,"sense_key":"part%1:08:01::","synset_id":"part.n.10"},{"index":11,"sense_key":"part%1:10:00::","synset_id":"part.n.11"},{"index":12,"sense_key":"part%1:04:03::","synset_id":"contribution.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"part%1:17:00::","call_ids":["semeval2015.d000.s009.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"part%1:17:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1330,"cached_input_tokens":0,"output_tokens":122,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00388,"input_uncached_usd":0.00266,"input_cached_usd":0.0,"output_usd":0.0012200000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8856779169291258}
{"item_id":"semeval2015.d000.s009.t015","gold_sense_keys":["body%1:08:00::"],"candidates":[{"index":1,"sense_key":"body%1:08:00::","synset_id":"body.n.01"},{"index":2,"sense_key":"body%1:14:00::","synset_id":"body.n.02"},{"index":3,"sense_key":"body%1:08:02::","synset_id":"body.n.03"},{"index":4,"sense_key":"body%1:17:00::","synset_id":"body.n.04"},{"index":5,"sense_key":"body%1:08:01::","synset_id":"torso.n.01"},{"index":6,"sense_key":"body%1:14:02::","synset_id":"body.n.06"},{"index":7,"sense_key":"body%1:07:00::","synset_id":"consistency.n.01"},{"index":8,"sense_key":"body%1:10:00::","synset_id":"body.n.08"},{"index":9,"sense_key":"body%1:07:01::","synset_id":"body.n.09"},{"index":10,"sense_key":"body%1:06:01::","synset_id":"soundbox.n.01"},{"index":11,"sense_key":"body%1:06:00::","synset_id":"body.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"body%1:08:00::","call_ids":["semeval2015.d000.s009.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"body%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1136,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0023919999999999996,"input_uncached_usd":0.0022719999999999997,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2124176250072196}
{"item_id":"semeval2015.d000.s009.t016","gold_sense_keys":["patient%1:18:00::"],"candidates":[{"index":1,"sense_key":"patient%1:18:00::","synset_id":"patient.n.01"},{"index":2,"sense_key":"patient%1:10:00::","synset_id":"affected_role.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"patient%1:18:00::","call_ids":["semeval2015.d000.s009.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"patient%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":644,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001408,"input_uncached_usd":0.0012879999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9300787079846486}
{"item_id":"semeval2015.d000.s009.t020","gold_sense_keys":["medicine%1:06:00::"],"candidates":[{"index":1,"sense_key":"medicine%1:09:00::","synset_id":"medicine.n.01"},{"index":2,"sense_key":"medicine%1:06:00::","synset_id":"medicine.n.02"},{"index":3,"sense_key":"medicine%1:04:00::","synset_id":"medicine.n.03"},{"index":4,"sense_key":"medicine%1:04:02::","synset_id":"music.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"medicine%1:06:00::","call_ids":["semeval2015.d000.s009.t020__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"medicine%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":793,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001706,"input_uncached_usd":0.001586,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0315589579986408}
{"item_id":"semeval2015.d000.s009.t021","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["semeval2015.d000.s009.t021__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":837,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001794,"input_uncached_usd":0.001674,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4457876670639962}
{"item_id":"semeval2015.d000.s009.t022","gold_sense_keys":["advanced%5:00:00:late:02"],"candidates":[{"index":1,"sense_key":"advanced%5:00:00:precocious:00","synset_id":"advanced.s.01"},{"index":2,"sense_key":"advanced%5:00:00:late:02","synset_id":"advanced.s.02"},{"index":3,"sense_key":"advanced%5:00:00:progressive:01","synset_id":"advanced.s.03"},{"index":4,"sense_key":"advanced%5:00:02:high:02","synset_id":"advanced.s.04"},{"index":5,"sense_key":"advanced%5:00:00:high-tech:00","synset_id":"advanced.s.05"},{"index":6,"sense_key":"advanced%5:00:00:late:00","synset_id":"advanced.s.06"},{"index":7,"sense_key":"advanced%5:00:00:civilized:00","synset_id":"advanced.s.07"},{"index":8,"sense_key":"advanced%5:00:00:front:00","synset_id":"advance.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"advanced%5:00:00:late:02","call_ids":["semeval2015.d000.s009.t022__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"advanced%5:00:00:late:02","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1067,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002764,"input_uncached_usd":0.002134,"input_cached_usd":0.0,"output_usd":0.00063,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9989638340193778}
{"item_id":"semeval2015.d000.s009.t024","gold_sense_keys":["cell%1:03:00::"],"candidates":[{"index":1,"sense_key":"cell%1:06:03::","synset_id":"cell.n.01"},{"index":2,"sense_key":"cell%1:03:00::","synset_id":"cell.n.02"},{"index":3,"sense_key":"cell%1:06:00::","synset_id":"cell.n.03"},{"index":4,"sense_key":"cell%1:14:00::","synset_id":"cell.n.04"},{"index":5,"sense_key":"cell%1:06:04::","synset_id":"cellular_telephone.n.01"},{"index":6,"sense_key":"cell%1:06:02::","synset_id":"cell.n.06"},{"index":7,"sense_key":"cell%1:06:01::","synset_id":"cell.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cell%1:03:00::","call_ids":["semeval2015.d000.s009.t024__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cell%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":929,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0019779999999999997,"input_uncached_usd":0.0018579999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":10.413391540991142}
{"item_id":"semeval2015.d000.s009.t027","gold_sense_keys":["affect%2:29:00::"],"candidates":[{"index":1,"sense_key":"affect%2:30:00::","synset_id":"affect.v.01"},{"index":2,"sense_key":"affect%2:29:00::","synset_id":"affect.v.02"},{"index":3,"sense_key":"affect%2:42:00::","synset_id":"involve.v.01"},{"index":4,"sense_key":"affect%2:32:00::","synset_id":"feign.v.01"},{"index":5,"sense_key":"affect%2:37:00::","synset_id":"affect.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"affect%2:29:00::","call_ids":["semeval2015.d000.s009.t027__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"affect%2:29:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":843,"cached_input_tokens":0,"output_tokens":122,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002906,"input_uncached_usd":0.001686,"input_cached_usd":0.0,"output_usd":0.0012200000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0426928750239313}
{"item_id":"semeval2015.d000.s010.t000","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":2,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":3,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":4,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":5,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":6,"sense_key":"use%2:41:03::","synset_id":"use.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%2:34:01::","call_ids":["semeval2015.d000.s010.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":914,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001948,"input_uncached_usd":0.001828,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":14.378983541973867}
{"item_id":"semeval2015.d000.s010.t002","gold_sense_keys":["patient%1:18:00::"],"candidates":[{"index":1,"sense_key":"patient%1:18:00::","synset_id":"patient.n.01"},{"index":2,"sense_key":"patient%1:10:00::","synset_id":"affected_role.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"patient%1:18:00::","call_ids":["semeval2015.d000.s010.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"patient%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":653,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001426,"input_uncached_usd":0.0013059999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":10.370163167011924}
{"item_id":"semeval2015.d000.s010.t004","gold_sense_keys":["treat%2:29:00::"],"candidates":[{"index":1,"sense_key":"treat%2:41:00::","synset_id":"treat.v.01"},{"index":2,"sense_key":"treat%2:30:01::","synset_id":"process.v.01"},{"index":3,"sense_key":"treat%2:29:00::","synset_id":"treat.v.03"},{"index":4,"sense_key":"treat%2:32:00::","synset_id":"cover.v.05"},{"index":5,"sense_key":"treat%2:40:00::","synset_id":"treat.v.05"},{"index":6,"sense_key":"treat%2:34:00::","synset_id":"regale.v.01"},{"index":7,"sense_key":"treat%2:32:03::","synset_id":"treat.v.07"},{"index":8,"sense_key":"treat%2:31:00::","synset_id":"treat.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"treat%2:29:00::","call_ids":["semeval2015.d000.s010.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"treat%2:29:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1053,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0022259999999999997,"input_uncached_usd":0.002106,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":10.370742458035238}
{"item_id":"semeval2015.d000.s010.t005","gold_sense_keys":["patient%1:18:00::"],"candidates":[{"index":1,"sense_key":"patient%1:18:00::","synset_id":"patient.n.01"},{"index":2,"sense_key":"patient%1:10:00::","synset_id":"affected_role.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"patient%1:18:00::","call_ids":["semeval2015.d000.s010.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"patient%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":653,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001426,"input_uncached_usd":0.0013059999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":14.174063542042859}
{"item_id":"semeval2015.d000.s010.t007","gold_sense_keys":["type%1:09:00::"],"candidates":[{"index":1,"sense_key":"type%1:09:00::","synset_id":"type.n.01"},{"index":2,"sense_key":"type%1:18:00::","synset_id":"character.n.05"},{"index":3,"sense_key":"type%1:14:01::","synset_id":"type.n.03"},{"index":4,"sense_key":"type%1:10:01::","synset_id":"type.n.04"},{"index":5,"sense_key":"type%1:10:00::","synset_id":"type.n.05"},{"index":6,"sense_key":"type%1:06:00::","synset_id":"type.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"type%1:09:00::","call_ids":["semeval2015.d000.s010.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"type%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":853,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001826,"input_uncached_usd":0.0017059999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2037218750920147}
{"item_id":"semeval2015.d000.s010.t009","gold_sense_keys":["past%1:28:00::"],"candidates":[{"index":1,"sense_key":"past%1:28:00::","synset_id":"past.n.01"},{"index":2,"sense_key":"past%1:28:01::","synset_id":"past.n.02"},{"index":3,"sense_key":"past%1:10:00::","synset_id":"past.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"past%1:28:00::","call_ids":["semeval2015.d000.s010.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"past%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":686,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001952,"input_uncached_usd":0.001372,"input_cached_usd":0.0,"output_usd":0.00058,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.400667833047919}
{"item_id":"semeval2015.d000.s011.t000","gold_sense_keys":["medicine%1:06:00::"],"candidates":[{"index":1,"sense_key":"medicine%1:09:00::","synset_id":"medicine.n.01"},{"index":2,"sense_key":"medicine%1:06:00::","synset_id":"medicine.n.02"},{"index":3,"sense_key":"medicine%1:04:00::","synset_id":"medicine.n.03"},{"index":4,"sense_key":"medicine%1:04:02::","synset_id":"music.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"medicine%1:06:00::","call_ids":["semeval2015.d000.s011.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"medicine%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":782,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001684,"input_uncached_usd":0.0015639999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1033654579659924}
{"item_id":"semeval2015.d000.s011.t001","gold_sense_keys":["only%4:02:01::"],"candidates":[{"index":1,"sense_key":"only%4:02:02::","synset_id":"merely.r.01"},{"index":2,"sense_key":"only%4:02:01::","synset_id":"entirely.r.02"},{"index":3,"sense_key":"only%4:02:00::","synset_id":"only.r.03"},{"index":4,"sense_key":"only%4:02:04::","synset_id":"only.r.04"},{"index":5,"sense_key":"only%4:02:06::","synset_id":"only.r.05"},{"index":6,"sense_key":"only%4:02:03::","synset_id":"only.r.06"},{"index":7,"sense_key":"only%4:02:05::","synset_id":"only.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"only%4:02:01::","call_ids":["semeval2015.d000.s011.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"only%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":874,"cached_input_tokens":0,"output_tokens":112,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002868,"input_uncached_usd":0.001748,"input_cached_usd":0.0,"output_usd":0.0011200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8819239169824868}
{"item_id":"semeval2015.d000.s011.t002","gold_sense_keys":["obtain%2:40:00::"],"candidates":[{"index":1,"sense_key":"obtain%2:40:00::","synset_id":"obtain.v.01"},{"index":2,"sense_key":"obtain%2:30:00::","synset_id":"receive.v.02"},{"index":3,"sense_key":"obtain%2:42:00::","synset_id":"prevail.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"obtain%2:40:00::","call_ids":["semeval2015.d000.s011.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"obtain%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":692,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0015040000000000001,"input_uncached_usd":0.001384,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.119754666928202}
{"item_id":"semeval2015.d000.s011.t003","gold_sense_keys":["prescription%1:10:01::"],"candidates":[{"index":1,"sense_key":"prescription%1:10:00::","synset_id":"prescription.n.01"},{"index":2,"sense_key":"prescription%1:06:00::","synset_id":"prescription_drug.n.01"},{"index":3,"sense_key":"prescription%1:10:02::","synset_id":"prescription.n.03"},{"index":4,"sense_key":"prescription%1:10:01::","synset_id":"prescription.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"prescription%1:10:01::","call_ids":["semeval2015.d000.s011.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"prescription%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":777,"cached_input_tokens":0,"output_tokens":127,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002824,"input_uncached_usd":0.0015539999999999998,"input_cached_usd":0.0,"output_usd":0.00127,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9107492079492658}
{"item_id":"semeval2015.d000.s012.t000","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":2,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":3,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":4,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":5,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":6,"sense_key":"use%2:41:03::","synset_id":"use.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%2:34:01::","call_ids":["semeval2015.d000.s012.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":918,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001956,"input_uncached_usd":0.001836,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4821171659277752}
{"item_id":"semeval2015.d000.s013.t000","gold_sense_keys":["only%4:02:03::"],"candidates":[{"index":1,"sense_key":"only%4:02:02::","synset_id":"merely.r.01"},{"index":2,"sense_key":"only%4:02:01::","synset_id":"entirely.r.02"},{"index":3,"sense_key":"only%4:02:00::","synset_id":"only.r.03"},{"index":4,"sense_key":"only%4:02:04::","synset_id":"only.r.04"},{"index":5,"sense_key":"only%4:02:06::","synset_id":"only.r.05"},{"index":6,"sense_key":"only%4:02:03::","synset_id":"only.r.06"},{"index":7,"sense_key":"only%4:02:05::","synset_id":"only.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"only%4:02:01::","call_ids":["semeval2015.d000.s013.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"only%4:02:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":929,"cached_input_tokens":0,"output_tokens":146,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0033179999999999998,"input_uncached_usd":0.0018579999999999998,"input_cached_usd":0.0,"output_usd":0.0014600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":15.133565958938561}
{"item_id":"semeval2015.d000.s013.t001","gold_sense_keys":["give%2:40:13::"],"candidates":[{"index":1,"sense_key":"give%2:40:03::","synset_id":"give.v.01"},{"index":2,"sense_key":"give%2:40:05::","synset_id":"yield.v.01"},{"index":3,"sense_key":"give%2:40:00::","synset_id":"give.v.03"},{"index":4,"sense_key":"give%2:40:07::","synset_id":"give.v.04"},{"index":5,"sense_key":"give%2:32:01::","synset_id":"give.v.05"},{"index":6,"sense_key":"give%2:36:02::","synset_id":"hold.v.03"},{"index":7,"sense_key":"give%2:32:00::","synset_id":"give.v.07"},{"index":8,"sense_key":"give%2:40:01::","synset_id":"give.v.08"},{"index":9,"sense_key":"give%2:36:03::","synset_id":"give.v.09"},{"index":10,"sense_key":"give%2:31:02::","synset_id":"give.v.10"},{"index":11,"sense_key":"give%2:36:00::","synset_id":"render.v.04"},{"index":12,"sense_key":"give%2:40:09::","synset_id":"impart.v.01"},{"index":13,"sense_key":"give%2:36:01::","synset_id":"establish.v.05"},{"index":14,"sense_key":"give%2:40:02::","synset_id":"give.v.14"},{"index":15,"sense_key":"give%2:41:00::","synset_id":"give.v.15"},{"index":16,"sense_key":"give%2:40:10::","synset_id":"sacrifice.v.01"},{"index":17,"sense_key":"give%2:40:06::","synset_id":"pass.v.05"},{"index":18,"sense_key":"give%2:32:03::","synset_id":"give.v.18"},{"index":19,"sense_key":"give%2:40:13::","synset_id":"give.v.19"},{"index":20,"sense_key":"give%2:40:14::","synset_id":"give.v.20"},{"index":21,"sense_key":"give%2:32:02::","synset_id":"give.v.21"},{"index":22,"sense_key":"give%2:40:11::","synset_id":"grant.v.05"},{"index":23,"sense_key":"give%2:38:00::","synset_id":"move_over.v.01"},{"index":24,"sense_key":"give%2:34:00::","synset_id":"feed.v.02"},{"index":25,"sense_key":"give%2:40:08::","synset_id":"contribute.v.02"},{"index":26,"sense_key":"give%2:38:13::","synset_id":"collapse.v.01"},{"index":27,"sense_key":"give%2:31:00::","synset_id":"give.v.27"},{"index":28,"sense_key":"give%2:41:10::","synset_id":"give.v.28"},{"index":29,"sense_key":"give%2:40:12::","synset_id":"give.v.29"},{"index":30,"sense_key":"give%2:40:15::","synset_id":"afford.v.04"},{"index":31,"sense_key":"give%2:36:13::","synset_id":"give.v.31"},{"index":32,"sense_key":"give%2:36:12::","synset_id":"give.v.32"},{"index":33,"sense_key":"give%2:35:00::","synset_id":"give.v.33"},{"index":34,"sense_key":"give%2:34:10::","synset_id":"give.v.34"},{"index":35,"sense_key":"give%2:32:11::","synset_id":"give.v.35"},{"index":36,"sense_key":"give%2:32:08::","synset_id":"give.v.36"},{"index":37,"sense_key":"give%2:32:15::","synset_id":"give.v.37"},{"index":38,"sense_key":"give%2:32:14::","synset_id":"give.v.38"},{"index":39,"sense_key":"give%2:32:09::","synset_id":"give.v.39"},{"index":40,"sense_key":"give%2:32:07::","synset_id":"give.v.40"},{"index":41,"sense_key":"give%2:32:10::","synset_id":"give.v.41"},{"index":42,"sense_key":"give%2:30:13::","synset_id":"give.v.42"},{"index":43,"sense_key":"give%2:29:11::","synset_id":"give.v.43"},{"index":44,"sense_key":"give%2:29:10::","synset_id":"give.v.44"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":19,"chosen_sense_key":"give%2:40:13::","call_ids":["semeval2015.d000.s013.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":19,"predicted_sense_key":"give%2:40:13::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2962,"cached_input_tokens":0,"output_tokens":62,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0065439999999999995,"input_uncached_usd":0.005924,"input_cached_usd":0.0,"output_usd":0.00062,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8613969590514898}
{"item_id":"semeval2015.d000.s013.t003","gold_sense_keys":["doctor%1:18:00::"],"candidates":[{"index":1,"sense_key":"doctor%1:18:00::","synset_id":"doctor.n.01"},{"index":2,"sense_key":"doctor%1:18:02::","synset_id":"doctor_of_the_church.n.01"},{"index":3,"sense_key":"doctor%1:04:00::","synset_id":"doctor.n.03"},{"index":4,"sense_key":"doctor%1:18:01::","synset_id":"doctor.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"doctor%1:18:00::","call_ids":["semeval2015.d000.s013.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"doctor%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":905,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00193,"input_uncached_usd":0.00181,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1432305000489578}
{"item_id":"semeval2015.d000.s013.t004","gold_sense_keys":["use%1:04:00::"],"candidates":[{"index":1,"sense_key":"use%1:04:00::","synset_id":"use.n.01"},{"index":2,"sense_key":"use%1:07:00::","synset_id":"function.n.02"},{"index":3,"sense_key":"use%1:07:02::","synset_id":"use.n.03"},{"index":4,"sense_key":"use%1:22:00::","synset_id":"consumption.n.03"},{"index":5,"sense_key":"use%1:04:01::","synset_id":"habit.n.02"},{"index":6,"sense_key":"use%1:04:02::","synset_id":"manipulation.n.01"},{"index":7,"sense_key":"use%1:07:03::","synset_id":"use.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%1:04:00::","call_ids":["semeval2015.d000.s013.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1090,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0023,"input_uncached_usd":0.00218,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0071356249973178}
{"item_id":"semeval2015.d000.s014.t000","gold_sense_keys":["dose%1:06:00::"],"candidates":[{"index":1,"sense_key":"dose%1:06:00::","synset_id":"dose.n.01"},{"index":2,"sense_key":"dose%1:23:00::","synset_id":"dose.n.02"},{"index":3,"sense_key":"dose%1:26:00::","synset_id":"venereal_disease.n.01"},{"index":4,"sense_key":"dose%1:06:01::","synset_id":"acid.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"dose%1:06:00::","call_ids":["semeval2015.d000.s014.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"dose%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":797,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001714,"input_uncached_usd":0.001594,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5267275830265135}
{"item_id":"semeval2015.d000.s014.t001","gold_sense_keys":["mg%1:23:00::"],"candidates":[{"index":1,"sense_key":"mg%1:23:00::","synset_id":"milligram.n.01"},{"index":2,"sense_key":"mg%1:27:00::","synset_id":"magnesium.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"mg%1:23:00::","call_ids":["semeval2015.d000.s014.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"mg%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":692,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0015040000000000001,"input_uncached_usd":0.001384,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2879519580164924}
{"item_id":"semeval2015.d000.s014.t003","gold_sense_keys":["body%1:08:00::"],"candidates":[{"index":1,"sense_key":"body%1:08:00::","synset_id":"body.n.01"},{"index":2,"sense_key":"body%1:14:00::","synset_id":"body.n.02"},{"index":3,"sense_key":"body%1:08:02::","synset_id":"body.n.03"},{"index":4,"sense_key":"body%1:17:00::","synset_id":"body.n.04"},{"index":5,"sense_key":"body%1:08:01::","synset_id":"torso.n.01"},{"index":6,"sense_key":"body%1:14:02::","synset_id":"body.n.06"},{"index":7,"sense_key":"body%1:07:00::","synset_id":"consistency.n.01"},{"index":8,"sense_key":"body%1:10:00::","synset_id":"body.n.08"},{"index":9,"sense_key":"body%1:07:01::","synset_id":"body.n.09"},{"index":10,"sense_key":"body%1:06:01::","synset_id":"soundbox.n.01"},{"index":11,"sense_key":"body%1:06:00::","synset_id":"body.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"body%1:08:00::","call_ids":["semeval2015.d000.s014.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"body%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1154,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0024279999999999996,"input_uncached_usd":0.0023079999999999997,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5089793748920783}
{"item_id":"semeval2015.d000.s014.t005","gold_sense_keys":["calculate%2:31:00::"],"candidates":[{"index":1,"sense_key":"calculate%2:31:00::","synset_id":"calculate.v.01"},{"index":2,"sense_key":"calculate%2:31:03::","synset_id":"calculate.v.02"},{"index":3,"sense_key":"calculate%2:40:00::","synset_id":"account.v.02"},{"index":4,"sense_key":"calculate%2:32:01::","synset_id":"forecast.v.01"},{"index":5,"sense_key":"calculate%2:31:02::","synset_id":"calculate.v.05"},{"index":6,"sense_key":"calculate%2:31:05::","synset_id":"count.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"calculate%2:31:00::","call_ids":["semeval2015.d000.s014.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"calculate%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":849,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001818,"input_uncached_usd":0.0016979999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8225612919777632}
{"item_id":"semeval2015.d000.s014.t006","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":2,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":3,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":4,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":5,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":6,"sense_key":"use%2:41:03::","synset_id":"use.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%2:34:01::","call_ids":["semeval2015.d000.s014.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":923,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001966,"input_uncached_usd":0.001846,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.224265333963558}
{"item_id":"semeval2015.d000.s014.t007","gold_sense_keys":["patient%1:18:00::"],"candidates":[{"index":1,"sense_key":"patient%1:18:00::","synset_id":"patient.n.01"},{"index":2,"sense_key":"patient%1:10:00::","synset_id":"affected_role.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"patient%1:18:00::","call_ids":["semeval2015.d000.s014.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"patient%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":662,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001444,"input_uncached_usd":0.0013239999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1608551250537857}
{"item_id":"semeval2015.d000.s014.t008","gold_sense_keys":["height%1:07:01::"],"candidates":[{"index":1,"sense_key":"height%1:07:00::","synset_id":"height.n.01"},{"index":2,"sense_key":"height%1:26:00::","synset_id":"acme.n.01"},{"index":3,"sense_key":"height%1:07:01::","synset_id":"stature.n.02"},{"index":4,"sense_key":"height%1:07:02::","synset_id":"altitude.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"height%1:07:01::","call_ids":["semeval2015.d000.s014.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"height%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":792,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001704,"input_uncached_usd":0.001584,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.209169500041753}
{"item_id":"semeval2015.d000.s014.t009","gold_sense_keys":["weight%1:07:00::"],"candidates":[{"index":1,"sense_key":"weight%1:07:00::","synset_id":"weight.n.01"},{"index":2,"sense_key":"weight%1:06:01::","synset_id":"weight.n.02"},{"index":3,"sense_key":"weight%1:07:01::","synset_id":"weight.n.03"},{"index":4,"sense_key":"weight%1:06:00::","synset_id":"weight.n.04"},{"index":5,"sense_key":"weight%1:12:00::","synset_id":"weight.n.05"},{"index":6,"sense_key":"weight%1:23:01::","synset_id":"system_of_weights.n.01"},{"index":7,"sense_key":"weight%1:23:00::","synset_id":"weight_unit.n.02"},{"index":8,"sense_key":"weight%1:23:02::","synset_id":"weight.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"weight%1:07:00::","call_ids":["semeval2015.d000.s014.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"weight%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":985,"cached_input_tokens":0,"output_tokens":103,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003,"input_uncached_usd":0.00197,"input_cached_usd":0.0,"output_usd":0.00103,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":12.203952874988317}
{"item_id":"semeval2015.d000.s015.t000","gold_sense_keys":["give%2:40:13::"],"candidates":[{"index":1,"sense_key":"give%2:40:03::","synset_id":"give.v.01"},{"index":2,"sense_key":"give%2:40:05::","synset_id":"yield.v.01"},{"index":3,"sense_key":"give%2:40:00::","synset_id":"give.v.03"},{"index":4,"sense_key":"give%2:40:07::","synset_id":"give.v.04"},{"index":5,"sense_key":"give%2:32:01::","synset_id":"give.v.05"},{"index":6,"sense_key":"give%2:36:02::","synset_id":"hold.v.03"},{"index":7,"sense_key":"give%2:32:00::","synset_id":"give.v.07"},{"index":8,"sense_key":"give%2:40:01::","synset_id":"give.v.08"},{"index":9,"sense_key":"give%2:36:03::","synset_id":"give.v.09"},{"index":10,"sense_key":"give%2:31:02::","synset_id":"give.v.10"},{"index":11,"sense_key":"give%2:36:00::","synset_id":"render.v.04"},{"index":12,"sense_key":"give%2:40:09::","synset_id":"impart.v.01"},{"index":13,"sense_key":"give%2:36:01::","synset_id":"establish.v.05"},{"index":14,"sense_key":"give%2:40:02::","synset_id":"give.v.14"},{"index":15,"sense_key":"give%2:41:00::","synset_id":"give.v.15"},{"index":16,"sense_key":"give%2:40:10::","synset_id":"sacrifice.v.01"},{"index":17,"sense_key":"give%2:40:06::","synset_id":"pass.v.05"},{"index":18,"sense_key":"give%2:32:03::","synset_id":"give.v.18"},{"index":19,"sense_key":"give%2:40:13::","synset_id":"give.v.19"},{"index":20,"sense_key":"give%2:40:14::","synset_id":"give.v.20"},{"index":21,"sense_key":"give%2:32:02::","synset_id":"give.v.21"},{"index":22,"sense_key":"give%2:40:11::","synset_id":"grant.v.05"},{"index":23,"sense_key":"give%2:38:00::","synset_id":"move_over.v.01"},{"index":24,"sense_key":"give%2:34:00::","synset_id":"feed.v.02"},{"index":25,"sense_key":"give%2:40:08::","synset_id":"contribute.v.02"},{"index":26,"sense_key":"give%2:38:13::","synset_id":"collapse.v.01"},{"index":27,"sense_key":"give%2:31:00::","synset_id":"give.v.27"},{"index":28,"sense_key":"give%2:41:10::","synset_id":"give.v.28"},{"index":29,"sense_key":"give%2:40:12::","synset_id":"give.v.29"},{"index":30,"sense_key":"give%2:40:15::","synset_id":"afford.v.04"},{"index":31,"sense_key":"give%2:36:13::","synset_id":"give.v.31"},{"index":32,"sense_key":"give%2:36:12::","synset_id":"give.v.32"},{"index":33,"sense_key":"give%2:35:00::","synset_id":"give.v.33"},{"index":34,"sense_key":"give%2:34:10::","synset_id":"give.v.34"},{"index":35,"sense_key":"give%2:32:11::","synset_id":"give.v.35"},{"index":36,"sense_key":"give%2:32:08::","synset_id":"give.v.36"},{"index":37,"sense_key":"give%2:32:15::","synset_id":"give.v.37"},{"index":38,"sense_key":"give%2:32:14::","synset_id":"give.v.38"},{"index":39,"sense_key":"give%2:32:09::","synset_id":"give.v.39"},{"index":40,"sense_key":"give%2:32:07::","synset_id":"give.v.40"},{"index":41,"sense_key":"give%2:32:10::","synset_id":"give.v.41"},{"index":42,"sense_key":"give%2:30:13::","synset_id":"give.v.42"},{"index":43,"sense_key":"give%2:29:11::","synset_id":"give.v.43"},{"index":44,"sense_key":"give%2:29:10::","synset_id":"give.v.44"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":19,"chosen_sense_key":"give%2:40:13::","call_ids":["semeval2015.d000.s015.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":19,"predicted_sense_key":"give%2:40:13::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2864,"cached_input_tokens":0,"output_tokens":49,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.006218,"input_uncached_usd":0.005728,"input_cached_usd":0.0,"output_usd":0.0004900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":9.419616792001761}
{"item_id":"semeval2015.d000.s015.t001","gold_sense_keys":["once%4:02:00::"],"candidates":[{"index":1,"sense_key":"once%4:02:00::","synset_id":"once.r.01"},{"index":2,"sense_key":"once%4:02:02::","synset_id":"once.r.02"},{"index":3,"sense_key":"once%4:02:01::","synset_id":"once.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"once%4:02:00::","call_ids":["semeval2015.d000.s015.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"once%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":618,"cached_input_tokens":0,"output_tokens":84,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0020759999999999997,"input_uncached_usd":0.001236,"input_cached_usd":0.0,"output_usd":0.00084,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4110422079684213}
{"item_id":"semeval2015.d000.s015.t002","gold_sense_keys":["every%5:00:02:all:00"],"candidates":[{"index":1,"sense_key":"every%5:00:00:all:00","synset_id":"every.s.01"},{"index":2,"sense_key":"every%5:00:02:all:00","synset_id":"every.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"every%5:00:02:all:00","call_ids":["semeval2015.d000.s015.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"every%5:00:02:all:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":583,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001286,"input_uncached_usd":0.001166,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2387291248887777}
{"item_id":"semeval2015.d000.s015.t003","gold_sense_keys":["week%1:28:00::"],"candidates":[{"index":1,"sense_key":"week%1:28:00::","synset_id":"week.n.01"},{"index":2,"sense_key":"week%1:28:01::","synset_id":"workweek.n.01"},{"index":3,"sense_key":"week%1:28:02::","synset_id":"week.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"week%1:28:00::","call_ids":["semeval2015.d000.s015.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"week%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":622,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001364,"input_uncached_usd":0.0012439999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0564776250394061}
{"item_id":"semeval2015.d000.s015.t005","gold_sense_keys":["infusion%1:04:00::"],"candidates":[{"index":1,"sense_key":"infusion%1:27:00::","synset_id":"infusion.n.01"},{"index":2,"sense_key":"infusion%1:22:00::","synset_id":"infusion.n.02"},{"index":3,"sense_key":"infusion%1:04:00::","synset_id":"infusion.n.03"},{"index":4,"sense_key":"infusion%1:04:01::","synset_id":"infusion.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"infusion%1:04:00::","call_ids":["semeval2015.d000.s015.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"infusion%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":719,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0015580000000000001,"input_uncached_usd":0.001438,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1428858749568462}
{"item_id":"semeval2015.d000.s015.t006","gold_sense_keys":["last%2:42:00::"],"candidates":[{"index":1,"sense_key":"last%2:42:00::","synset_id":"last.v.01"},{"index":2,"sense_key":"last%2:42:01::","synset_id":"survive.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"last%2:42:00::","call_ids":["semeval2015.d000.s015.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"last%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":605,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00133,"input_uncached_usd":0.00121,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9609427920076996}
{"item_id":"semeval2015.d000.s015.t007","gold_sense_keys":["minute%1:28:00::"],"candidates":[{"index":1,"sense_key":"minute%1:28:00::","synset_id":"minute.n.01"},{"index":2,"sense_key":"minute%1:28:01::","synset_id":"moment.n.02"},{"index":3,"sense_key":"minute%1:28:02::","synset_id":"moment.n.01"},{"index":4,"sense_key":"minute%1:23:00::","synset_id":"minute.n.04"},{"index":5,"sense_key":"minute%1:10:00::","synset_id":"minute.n.05"},{"index":6,"sense_key":"minute%1:07:00::","synset_id":"hour.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"minute%1:28:00::","call_ids":["semeval2015.d000.s015.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"minute%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":819,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001758,"input_uncached_usd":0.001638,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1316534170182422}
{"item_id":"semeval2015.d000.s016.t000","gold_sense_keys":["reduce%2:30:00::","reduce%2:30:01::","reduce%2:40:00::"],"candidates":[{"index":1,"sense_key":"reduce%2:30:00::","synset_id":"reduce.v.01"},{"index":2,"sense_key":"reduce%2:30:06::","synset_id":"reduce.v.02"},{"index":3,"sense_key":"reduce%2:41:00::","synset_id":"reduce.v.03"},{"index":4,"sense_key":"reduce%2:40:04::","synset_id":"reduce.v.04"},{"index":5,"sense_key":"reduce%2:37:00::","synset_id":"reduce.v.05"},{"index":6,"sense_key":"reduce%2:30:11::","synset_id":"reduce.v.06"},{"index":7,"sense_key":"reduce%2:30:05::","synset_id":"shrink.v.03"},{"index":8,"sense_key":"reduce%2:40:00::","synset_id":"reduce.v.08"},{"index":9,"sense_key":"reduce%2:30:04::","synset_id":"reduce.v.09"},{"index":10,"sense_key":"reduce%2:30:03::","synset_id":"deoxidize.v.01"},{"index":11,"sense_key":"reduce%2:30:01::","synset_id":"reduce.v.11"},{"index":12,"sense_key":"reduce%2:41:03::","synset_id":"repress.v.01"},{"index":13,"sense_key":"reduce%2:35:00::","synset_id":"reduce.v.13"},{"index":14,"sense_key":"reduce%2:35:01::","synset_id":"reduce.v.14"},{"index":15,"sense_key":"reduce%2:30:12::","synset_id":"reduce.v.15"},{"index":16,"sense_key":"reduce%2:30:08::","synset_id":"abridge.v.01"},{"index":17,"sense_key":"reduce%2:30:10::","synset_id":"boil_down.v.02"},{"index":18,"sense_key":"reduce%2:30:02::","synset_id":"reduce.v.18"},{"index":19,"sense_key":"reduce%2:30:09::","synset_id":"dilute.v.01"},{"index":20,"sense_key":"reduce%2:29:00::","synset_id":"reduce.v.20"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"reduce%2:30:00::","call_ids":["semeval2015.d000.s016.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"reduce%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1682,"cached_input_tokens":0,"output_tokens":126,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004624,"input_uncached_usd":0.003364,"input_cached_usd":0.0,"output_usd":0.00126,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.14369366702158}
{"item_id":"semeval2015.d000.s016.t001","gold_sense_keys":["side_effect%1:26:00::"],"candidates":[{"index":1,"sense_key":"side_effect%1:26:00::","synset_id":"side_effect.n.01"},{"index":2,"sense_key":"side_effect%1:19:00::","synset_id":"side_effect.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"side_effect%1:26:00::","call_ids":["semeval2015.d000.s016.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"side_effect%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":618,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001356,"input_uncached_usd":0.001236,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8811994589632377}
{"item_id":"semeval2015.d000.s016.t002","gold_sense_keys":["patient%1:18:00::"],"candidates":[{"index":1,"sense_key":"patient%1:18:00::","synset_id":"patient.n.01"},{"index":2,"sense_key":"patient%1:10:00::","synset_id":"affected_role.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"patient%1:18:00::","call_ids":["semeval2015.d000.s016.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"patient%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":605,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00133,"input_uncached_usd":0.00121,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":13.79531204199884}
{"item_id":"semeval2015.d000.s016.t006","gold_sense_keys":["type%1:09:00::"],"candidates":[{"index":1,"sense_key":"type%1:09:00::","synset_id":"type.n.01"},{"index":2,"sense_key":"type%1:18:00::","synset_id":"character.n.05"},{"index":3,"sense_key":"type%1:14:01::","synset_id":"type.n.03"},{"index":4,"sense_key":"type%1:10:01::","synset_id":"type.n.04"},{"index":5,"sense_key":"type%1:10:00::","synset_id":"type.n.05"},{"index":6,"sense_key":"type%1:06:00::","synset_id":"type.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"type%1:09:00::","call_ids":["semeval2015.d000.s016.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"type%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":805,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00173,"input_uncached_usd":0.0016099999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0255439160391688}
{"item_id":"semeval2015.d000.s016.t008","gold_sense_keys":["injection%1:04:01::"],"candidates":[{"index":1,"sense_key":"injection%1:04:00::","synset_id":"injection.n.01"},{"index":2,"sense_key":"injection%1:27:00::","synset_id":"injection.n.02"},{"index":3,"sense_key":"injection%1:04:01::","synset_id":"injection.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"injection%1:04:01::","call_ids":["semeval2015.d000.s016.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"injection%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":628,"cached_input_tokens":0,"output_tokens":90,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002156,"input_uncached_usd":0.001256,"input_cached_usd":0.0,"output_usd":0.0009000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":14.795604125014506}
{"item_id":"semeval2015.d000.s016.t010","gold_sense_keys":["treatment%1:04:00::"],"candidates":[{"index":1,"sense_key":"treatment%1:04:00::","synset_id":"treatment.n.01"},{"index":2,"sense_key":"treatment%1:04:01::","synset_id":"treatment.n.02"},{"index":3,"sense_key":"treatment%1:07:00::","synset_id":"treatment.n.03"},{"index":4,"sense_key":"treatment%1:10:00::","synset_id":"discussion.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"treatment%1:04:00::","call_ids":["semeval2015.d000.s016.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"treatment%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":754,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001628,"input_uncached_usd":0.001508,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.195282542030327}
{"item_id":"semeval2015.d000.s017.t000","gold_sense_keys":["give%2:40:13::"],"candidates":[{"index":1,"sense_key":"give%2:40:03::","synset_id":"give.v.01"},{"index":2,"sense_key":"give%2:40:05::","synset_id":"yield.v.01"},{"index":3,"sense_key":"give%2:40:00::","synset_id":"give.v.03"},{"index":4,"sense_key":"give%2:40:07::","synset_id":"give.v.04"},{"index":5,"sense_key":"give%2:32:01::","synset_id":"give.v.05"},{"index":6,"sense_key":"give%2:36:02::","synset_id":"hold.v.03"},{"index":7,"sense_key":"give%2:32:00::","synset_id":"give.v.07"},{"index":8,"sense_key":"give%2:40:01::","synset_id":"give.v.08"},{"index":9,"sense_key":"give%2:36:03::","synset_id":"give.v.09"},{"index":10,"sense_key":"give%2:31:02::","synset_id":"give.v.10"},{"index":11,"sense_key":"give%2:36:00::","synset_id":"render.v.04"},{"index":12,"sense_key":"give%2:40:09::","synset_id":"impart.v.01"},{"index":13,"sense_key":"give%2:36:01::","synset_id":"establish.v.05"},{"index":14,"sense_key":"give%2:40:02::","synset_id":"give.v.14"},{"index":15,"sense_key":"give%2:41:00::","synset_id":"give.v.15"},{"index":16,"sense_key":"give%2:40:10::","synset_id":"sacrifice.v.01"},{"index":17,"sense_key":"give%2:40:06::","synset_id":"pass.v.05"},{"index":18,"sense_key":"give%2:32:03::","synset_id":"give.v.18"},{"index":19,"sense_key":"give%2:40:13::","synset_id":"give.v.19"},{"index":20,"sense_key":"give%2:40:14::","synset_id":"give.v.20"},{"index":21,"sense_key":"give%2:32:02::","synset_id":"give.v.21"},{"index":22,"sense_key":"give%2:40:11::","synset_id":"grant.v.05"},{"index":23,"sense_key":"give%2:38:00::","synset_id":"move_over.v.01"},{"index":24,"sense_key":"give%2:34:00::","synset_id":"feed.v.02"},{"index":25,"sense_key":"give%2:40:08::","synset_id":"contribute.v.02"},{"index":26,"sense_key":"give%2:38:13::","synset_id":"collapse.v.01"},{"index":27,"sense_key":"give%2:31:00::","synset_id":"give.v.27"},{"index":28,"sense_key":"give%2:41:10::","synset_id":"give.v.28"},{"index":29,"sense_key":"give%2:40:12::","synset_id":"give.v.29"},{"index":30,"sense_key":"give%2:40:15::","synset_id":"afford.v.04"},{"index":31,"sense_key":"give%2:36:13::","synset_id":"give.v.31"},{"index":32,"sense_key":"give%2:36:12::","synset_id":"give.v.32"},{"index":33,"sense_key":"give%2:35:00::","synset_id":"give.v.33"},{"index":34,"sense_key":"give%2:34:10::","synset_id":"give.v.34"},{"index":35,"sense_key":"give%2:32:11::","synset_id":"give.v.35"},{"index":36,"sense_key":"give%2:32:08::","synset_id":"give.v.36"},{"index":37,"sense_key":"give%2:32:15::","synset_id":"give.v.37"},{"index":38,"sense_key":"give%2:32:14::","synset_id":"give.v.38"},{"index":39,"sense_key":"give%2:32:09::","synset_id":"give.v.39"},{"index":40,"sense_key":"give%2:32:07::","synset_id":"give.v.40"},{"index":41,"sense_key":"give%2:32:10::","synset_id":"give.v.41"},{"index":42,"sense_key":"give%2:30:13::","synset_id":"give.v.42"},{"index":43,"sense_key":"give%2:29:11::","synset_id":"give.v.43"},{"index":44,"sense_key":"give%2:29:10::","synset_id":"give.v.44"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":19,"chosen_sense_key":"give%2:40:13::","call_ids":["semeval2015.d000.s017.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":19,"predicted_sense_key":"give%2:40:13::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2910,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.006489999999999999,"input_uncached_usd":0.00582,"input_cached_usd":0.0,"output_usd":0.00067,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9321477500488982}
{"item_id":"semeval2015.d000.s017.t001","gold_sense_keys":["medicine%1:06:00::"],"candidates":[{"index":1,"sense_key":"medicine%1:09:00::","synset_id":"medicine.n.01"},{"index":2,"sense_key":"medicine%1:06:00::","synset_id":"medicine.n.02"},{"index":3,"sense_key":"medicine%1:04:00::","synset_id":"medicine.n.03"},{"index":4,"sense_key":"medicine%1:04:02::","synset_id":"music.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"medicine%1:06:00::","call_ids":["semeval2015.d000.s017.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"medicine%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":785,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00169,"input_uncached_usd":0.00157,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4578500830102712}
{"item_id":"semeval2015.d000.s017.t002","gold_sense_keys":["prevent%2:41:01::"],"candidates":[{"index":1,"sense_key":"prevent%2:41:01::","synset_id":"prevent.v.01"},{"index":2,"sense_key":"prevent%2:41:00::","synset_id":"prevent.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"prevent%2:41:01::","call_ids":["semeval2015.d000.s017.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"prevent%2:41:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":692,"cached_input_tokens":0,"output_tokens":153,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002914,"input_uncached_usd":0.001384,"input_cached_usd":0.0,"output_usd":0.0015300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":15.714961083023809}
{"item_id":"semeval2015.d000.s017.t004","gold_sense_keys":["fluid%1:27:02::"],"candidates":[{"index":1,"sense_key":"fluid%1:27:02::","synset_id":"fluid.n.01"},{"index":2,"sense_key":"fluid%1:27:00::","synset_id":"fluid.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"fluid%1:27:02::","call_ids":["semeval2015.d000.s017.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"fluid%1:27:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":612,"cached_input_tokens":0,"output_tokens":149,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0027140000000000003,"input_uncached_usd":0.001224,"input_cached_usd":0.0,"output_usd":0.0014900000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.6855302089825273}
{"item_id":"semeval2015.d000.s017.t005","gold_sense_keys":["prevent%2:41:01::"],"candidates":[{"index":1,"sense_key":"prevent%2:41:01::","synset_id":"prevent.v.01"},{"index":2,"sense_key":"prevent%2:41:00::","synset_id":"prevent.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"prevent%2:41:01::","call_ids":["semeval2015.d000.s017.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"prevent%2:41:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":692,"cached_input_tokens":0,"output_tokens":126,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002644,"input_uncached_usd":0.001384,"input_cached_usd":0.0,"output_usd":0.00126,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.099709500093013}
{"item_id":"semeval2015.d000.s017.t006","gold_sense_keys":["dehydration%1:26:01::"],"candidates":[{"index":1,"sense_key":"dehydration%1:26:00::","synset_id":"dehydration.n.01"},{"index":2,"sense_key":"dehydration%1:26:01::","synset_id":"dehydration.n.02"},{"index":3,"sense_key":"dehydration%1:22:00::","synset_id":"dehydration.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"dehydration%1:26:01::","call_ids":["semeval2015.d000.s017.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"dehydration%1:26:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":648,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001416,"input_uncached_usd":0.0012959999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1141327910590917}
{"item_id":"semeval2015.d000.s017.t008","gold_sense_keys":["give%2:40:13::"],"candidates":[{"index":1,"sense_key":"give%2:40:03::","synset_id":"give.v.01"},{"index":2,"sense_key":"give%2:40:05::","synset_id":"yield.v.01"},{"index":3,"sense_key":"give%2:40:00::","synset_id":"give.v.03"},{"index":4,"sense_key":"give%2:40:07::","synset_id":"give.v.04"},{"index":5,"sense_key":"give%2:32:01::","synset_id":"give.v.05"},{"index":6,"sense_key":"give%2:36:02::","synset_id":"hold.v.03"},{"index":7,"sense_key":"give%2:32:00::","synset_id":"give.v.07"},{"index":8,"sense_key":"give%2:40:01::","synset_id":"give.v.08"},{"index":9,"sense_key":"give%2:36:03::","synset_id":"give.v.09"},{"index":10,"sense_key":"give%2:31:02::","synset_id":"give.v.10"},{"index":11,"sense_key":"give%2:36:00::","synset_id":"render.v.04"},{"index":12,"sense_key":"give%2:40:09::","synset_id":"impart.v.01"},{"index":13,"sense_key":"give%2:36:01::","synset_id":"establish.v.05"},{"index":14,"sense_key":"give%2:40:02::","synset_id":"give.v.14"},{"index":15,"sense_key":"give%2:41:00::","synset_id":"give.v.15"},{"index":16,"sense_key":"give%2:40:10::","synset_id":"sacrifice.v.01"},{"index":17,"sense_key":"give%2:40:06::","synset_id":"pass.v.05"},{"index":18,"sense_key":"give%2:32:03::","synset_id":"give.v.18"},{"index":19,"sense_key":"give%2:40:13::","synset_id":"give.v.19"},{"index":20,"sense_key":"give%2:40:14::","synset_id":"give.v.20"},{"index":21,"sense_key":"give%2:32:02::","synset_id":"give.v.21"},{"index":22,"sense_key":"give%2:40:11::","synset_id":"grant.v.05"},{"index":23,"sense_key":"give%2:38:00::","synset_id":"move_over.v.01"},{"index":24,"sense_key":"give%2:34:00::","synset_id":"feed.v.02"},{"index":25,"sense_key":"give%2:40:08::","synset_id":"contribute.v.02"},{"index":26,"sense_key":"give%2:38:13::","synset_id":"collapse.v.01"},{"index":27,"sense_key":"give%2:31:00::","synset_id":"give.v.27"},{"index":28,"sense_key":"give%2:41:10::","synset_id":"give.v.28"},{"index":29,"sense_key":"give%2:40:12::","synset_id":"give.v.29"},{"index":30,"sense_key":"give%2:40:15::","synset_id":"afford.v.04"},{"index":31,"sense_key":"give%2:36:13::","synset_id":"give.v.31"},{"index":32,"sense_key":"give%2:36:12::","synset_id":"give.v.32"},{"index":33,"sense_key":"give%2:35:00::","synset_id":"give.v.33"},{"index":34,"sense_key":"give%2:34:10::","synset_id":"give.v.34"},{"index":35,"sense_key":"give%2:32:11::","synset_id":"give.v.35"},{"index":36,"sense_key":"give%2:32:08::","synset_id":"give.v.36"},{"index":37,"sense_key":"give%2:32:15::","synset_id":"give.v.37"},{"index":38,"sense_key":"give%2:32:14::","synset_id":"give.v.38"},{"index":39,"sense_key":"give%2:32:09::","synset_id":"give.v.39"},{"index":40,"sense_key":"give%2:32:07::","synset_id":"give.v.40"},{"index":41,"sense_key":"give%2:32:10::","synset_id":"give.v.41"},{"index":42,"sense_key":"give%2:30:13::","synset_id":"give.v.42"},{"index":43,"sense_key":"give%2:29:11::","synset_id":"give.v.43"},{"index":44,"sense_key":"give%2:29:10::","synset_id":"give.v.44"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":19,"chosen_sense_key":"give%2:40:13::","call_ids":["semeval2015.d000.s017.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":19,"predicted_sense_key":"give%2:40:13::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2910,"cached_input_tokens":0,"output_tokens":82,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00664,"input_uncached_usd":0.00582,"input_cached_usd":0.0,"output_usd":0.0008200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":11.088779333978891}
{"item_id":"semeval2015.d000.s017.t009","gold_sense_keys":["before%4:02:03::"],"candidates":[{"index":1,"sense_key":"before%4:02:03::","synset_id":"earlier.r.01"},{"index":2,"sense_key":"before%4:02:01::","synset_id":"ahead.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"before%4:02:03::","call_ids":["semeval2015.d000.s017.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"before%4:02:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":625,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0013700000000000001,"input_uncached_usd":0.00125,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4152549169957638}
{"item_id":"semeval2015.d000.s017.t010","gold_sense_keys":["after%4:02:01::"],"candidates":[{"index":1,"sense_key":"after%4:02:01::","synset_id":"subsequently.r.01"},{"index":2,"sense_key":"after%4:02:00::","synset_id":"after.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"after%4:02:01::","call_ids":["semeval2015.d000.s017.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"after%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":638,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001396,"input_uncached_usd":0.001276,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2004540829220787}
{"item_id":"semeval2015.d000.s017.t011","gold_sense_keys":["dose%1:06:00::"],"candidates":[{"index":1,"sense_key":"dose%1:06:00::","synset_id":"dose.n.01"},{"index":2,"sense_key":"dose%1:23:00::","synset_id":"dose.n.02"},{"index":3,"sense_key":"dose%1:26:00::","synset_id":"venereal_disease.n.01"},{"index":4,"sense_key":"dose%1:06:01::","synset_id":"acid.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"dose%1:06:00::","call_ids":["semeval2015.d000.s017.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"dose%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":771,"cached_input_tokens":0,"output_tokens":107,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0026119999999999997,"input_uncached_usd":0.001542,"input_cached_usd":0.0,"output_usd":0.00107,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6291077500209212}
{"item_id":"semeval2015.d000.s018.t000","gold_sense_keys":["treatment%1:04:00::"],"candidates":[{"index":1,"sense_key":"treatment%1:04:00::","synset_id":"treatment.n.01"},{"index":2,"sense_key":"treatment%1:04:01::","synset_id":"treatment.n.02"},{"index":3,"sense_key":"treatment%1:07:00::","synset_id":"treatment.n.03"},{"index":4,"sense_key":"treatment%1:10:00::","synset_id":"discussion.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"treatment%1:04:00::","call_ids":["semeval2015.d000.s018.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"treatment%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":805,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00173,"input_uncached_usd":0.0016099999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5793876249808818}
{"item_id":"semeval2015.d000.s018.t001","gold_sense_keys":["delay%2:30:00::"],"candidates":[{"index":1,"sense_key":"delay%2:30:00::","synset_id":"delay.v.01"},{"index":2,"sense_key":"delay%2:42:00::","synset_id":"delay.v.02"},{"index":3,"sense_key":"delay%2:30:01::","synset_id":"stay.v.06"},{"index":4,"sense_key":"delay%2:30:05::","synset_id":"check.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"delay%2:30:00::","call_ids":["semeval2015.d000.s018.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"delay%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":781,"cached_input_tokens":0,"output_tokens":168,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003242,"input_uncached_usd":0.001562,"input_cached_usd":0.0,"output_usd":0.00168,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.7433542090002447}
{"item_id":"semeval2015.d000.s018.t002","gold_sense_keys":["discontinue%2:42:00::"],"candidates":[{"index":1,"sense_key":"discontinue%2:42:00::","synset_id":"discontinue.v.01"},{"index":2,"sense_key":"discontinue%2:42:01::","synset_id":"discontinue.v.02"},{"index":3,"sense_key":"discontinue%2:30:00::","synset_id":"break.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"discontinue%2:42:00::","call_ids":["semeval2015.d000.s018.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"discontinue%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":717,"cached_input_tokens":0,"output_tokens":138,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002814,"input_uncached_usd":0.001434,"input_cached_usd":0.0,"output_usd":0.0013800000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9298982919426635}
{"item_id":"semeval2015.d000.s018.t003","gold_sense_keys":["dose%1:06:00::"],"candidates":[{"index":1,"sense_key":"dose%1:06:00::","synset_id":"dose.n.01"},{"index":2,"sense_key":"dose%1:23:00::","synset_id":"dose.n.02"},{"index":3,"sense_key":"dose%1:26:00::","synset_id":"venereal_disease.n.01"},{"index":4,"sense_key":"dose%1:06:01::","synset_id":"acid.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"dose%1:06:00::","call_ids":["semeval2015.d000.s018.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"dose%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":790,"cached_input_tokens":0,"output_tokens":98,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00256,"input_uncached_usd":0.00158,"input_cached_usd":0.0,"output_usd":0.0009800000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4776903329184279}
{"item_id":"semeval2015.d000.s018.t004","gold_sense_keys":["reduce%2:30:00::"],"candidates":[{"index":1,"sense_key":"reduce%2:30:00::","synset_id":"reduce.v.01"},{"index":2,"sense_key":"reduce%2:30:06::","synset_id":"reduce.v.02"},{"index":3,"sense_key":"reduce%2:41:00::","synset_id":"reduce.v.03"},{"index":4,"sense_key":"reduce%2:40:04::","synset_id":"reduce.v.04"},{"index":5,"sense_key":"reduce%2:37:00::","synset_id":"reduce.v.05"},{"index":6,"sense_key":"reduce%2:30:11::","synset_id":"reduce.v.06"},{"index":7,"sense_key":"reduce%2:30:05::","synset_id":"shrink.v.03"},{"index":8,"sense_key":"reduce%2:40:00::","synset_id":"reduce.v.08"},{"index":9,"sense_key":"reduce%2:30:04::","synset_id":"reduce.v.09"},{"index":10,"sense_key":"reduce%2:30:03::","synset_id":"deoxidize.v.01"},{"index":11,"sense_key":"reduce%2:30:01::","synset_id":"reduce.v.11"},{"index":12,"sense_key":"reduce%2:41:03::","synset_id":"repress.v.01"},{"index":13,"sense_key":"reduce%2:35:00::","synset_id":"reduce.v.13"},{"index":14,"sense_key":"reduce%2:35:01::","synset_id":"reduce.v.14"},{"index":15,"sense_key":"reduce%2:30:12::","synset_id":"reduce.v.15"},{"index":16,"sense_key":"reduce%2:30:08::","synset_id":"abridge.v.01"},{"index":17,"sense_key":"reduce%2:30:10::","synset_id":"boil_down.v.02"},{"index":18,"sense_key":"reduce%2:30:02::","synset_id":"reduce.v.18"},{"index":19,"sense_key":"reduce%2:30:09::","synset_id":"dilute.v.01"},{"index":20,"sense_key":"reduce%2:29:00::","synset_id":"reduce.v.20"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"reduce%2:30:00::","call_ids":["semeval2015.d000.s018.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"reduce%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1732,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0035839999999999995,"input_uncached_usd":0.0034639999999999996,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.7580674999626353}
{"item_id":"semeval2015.d000.s018.t005","gold_sense_keys":["patient%1:18:00::"],"candidates":[{"index":1,"sense_key":"patient%1:18:00::","synset_id":"patient.n.01"},{"index":2,"sense_key":"patient%1:10:00::","synset_id":"affected_role.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"patient%1:18:00::","call_ids":["semeval2015.d000.s018.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"patient%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":655,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00143,"input_uncached_usd":0.00131,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1710343329468742}
{"item_id":"semeval2015.d000.s018.t006","gold_sense_keys":["blood_count%1:23:00::"],"candidates":[{"index":1,"sense_key":"blood_count%1:23:00::","synset_id":"blood_count.n.01"},{"index":2,"sense_key":"blood_count%1:04:00::","synset_id":"blood_count.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"blood_count%1:23:00::","call_ids":["semeval2015.d000.s018.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"blood_count%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":623,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001926,"input_uncached_usd":0.001246,"input_cached_usd":0.0,"output_usd":0.00068,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.987632291042246}
{"item_id":"semeval2015.d000.s018.t007","gold_sense_keys":["abnormal%3:00:00::"],"candidates":[{"index":1,"sense_key":"abnormal%3:00:00::","synset_id":"abnormal.a.01"},{"index":2,"sense_key":"abnormal%3:00:03::","synset_id":"abnormal.a.02"},{"index":3,"sense_key":"abnormal%5:00:00:immoderate:00","synset_id":"abnormal.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"abnormal%3:00:00::","call_ids":["semeval2015.d000.s018.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"abnormal%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":741,"cached_input_tokens":0,"output_tokens":85,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002332,"input_uncached_usd":0.001482,"input_cached_usd":0.0,"output_usd":0.0008500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6389343340415508}
{"item_id":"semeval2015.d000.s018.t008","gold_sense_keys":["have%2:30:01::","have%2:39:06::"],"candidates":[{"index":1,"sense_key":"have%2:40:00::","synset_id":"have.v.01"},{"index":2,"sense_key":"have%2:42:00::","synset_id":"have.v.02"},{"index":3,"sense_key":"have%2:39:06::","synset_id":"experience.v.03"},{"index":4,"sense_key":"have%2:40:04::","synset_id":"own.v.01"},{"index":5,"sense_key":"have%2:30:00::","synset_id":"get.v.03"},{"index":6,"sense_key":"have%2:34:00::","synset_id":"consume.v.02"},{"index":7,"sense_key":"have%2:41:00::","synset_id":"have.v.07"},{"index":8,"sense_key":"have%2:36:00::","synset_id":"hold.v.03"},{"index":9,"sense_key":"have%2:40:01::","synset_id":"have.v.09"},{"index":10,"sense_key":"have%2:42:12::","synset_id":"have.v.10"},{"index":11,"sense_key":"have%2:30:01::","synset_id":"have.v.11"},{"index":12,"sense_key":"have%2:29:02::","synset_id":"have.v.12"},{"index":13,"sense_key":"have%2:32:00::","synset_id":"induce.v.02"},{"index":14,"sense_key":"have%2:40:05::","synset_id":"accept.v.02"},{"index":15,"sense_key":"have%2:40:03::","synset_id":"receive.v.01"},{"index":16,"sense_key":"have%2:29:05::","synset_id":"suffer.v.02"},{"index":17,"sense_key":"have%2:40:02::","synset_id":"have.v.17"},{"index":18,"sense_key":"have%2:29:00::","synset_id":"give_birth.v.01"},{"index":19,"sense_key":"have%2:35:00::","synset_id":"take.v.35"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":16,"chosen_sense_key":"have%2:29:05::","call_ids":["semeval2015.d000.s018.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":16,"predicted_sense_key":"have%2:29:05::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1698,"cached_input_tokens":0,"output_tokens":282,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.006215999999999999,"input_uncached_usd":0.0033959999999999997,"input_cached_usd":0.0,"output_usd":0.00282,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":15.923630042001605}
{"item_id":"semeval2015.d000.s018.t009","gold_sense_keys":["certain%5:00:00:definite:00"],"candidates":[{"index":1,"sense_key":"certain%5:00:00:definite:00","synset_id":"certain.s.01"},{"index":2,"sense_key":"certain%3:00:02::","synset_id":"certain.a.02"},{"index":3,"sense_key":"certain%3:00:01::","synset_id":"certain.a.03"},{"index":4,"sense_key":"certain%3:00:03::","synset_id":"certain.a.04"},{"index":5,"sense_key":"certain%3:00:05::","synset_id":"sealed.a.01"},{"index":6,"sense_key":"certain%5:00:00:reliable:00","synset_id":"certain.s.02"},{"index":7,"sense_key":"certain%5:00:00:careful:00","synset_id":"certain.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"certain%5:00:00:definite:00","call_ids":["semeval2015.d000.s018.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"certain%5:00:00:definite:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1000,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00212,"input_uncached_usd":0.002,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9739311248995364}
{"item_id":"semeval2015.d000.s018.t010","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":2,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":3,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"other%3:00:00::","call_ids":["semeval2015.d000.s018.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":764,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001648,"input_uncached_usd":0.0015279999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9868607909884304}
{"item_id":"semeval2015.d000.s018.t011","gold_sense_keys":["side_effect%1:26:00::"],"candidates":[{"index":1,"sense_key":"side_effect%1:26:00::","synset_id":"side_effect.n.01"},{"index":2,"sense_key":"side_effect%1:19:00::","synset_id":"side_effect.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"side_effect%1:26:00::","call_ids":["semeval2015.d000.s018.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"side_effect%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":668,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001456,"input_uncached_usd":0.001336,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":9.664736917009577}
{"item_id":"semeval2015.d000.s019.t002","gold_sense_keys":["use%1:04:00::"],"candidates":[{"index":1,"sense_key":"use%1:04:00::","synset_id":"use.n.01"},{"index":2,"sense_key":"use%1:07:00::","synset_id":"function.n.02"},{"index":3,"sense_key":"use%1:07:02::","synset_id":"use.n.03"},{"index":4,"sense_key":"use%1:22:00::","synset_id":"consumption.n.03"},{"index":5,"sense_key":"use%1:04:01::","synset_id":"habit.n.02"},{"index":6,"sense_key":"use%1:04:02::","synset_id":"manipulation.n.01"},{"index":7,"sense_key":"use%1:07:03::","synset_id":"use.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%1:04:00::","call_ids":["semeval2015.d000.s019.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1046,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0027020000000000004,"input_uncached_usd":0.002092,"input_cached_usd":0.0,"output_usd":0.0006100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6538674999028444}
{"item_id":"semeval2015.d000.s019.t003","gold_sense_keys":["patient%1:18:00::"],"candidates":[{"index":1,"sense_key":"patient%1:18:00::","synset_id":"patient.n.01"},{"index":2,"sense_key":"patient%1:10:00::","synset_id":"affected_role.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"patient%1:18:00::","call_ids":["semeval2015.d000.s019.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"patient%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":644,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001408,"input_uncached_usd":0.0012879999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9778346669627354}
{"item_id":"semeval2015.d000.s019.t004","gold_sense_keys":["moderate%3:00:00::","moderate%5:00:00:mild:00"],"candidates":[{"index":1,"sense_key":"moderate%3:00:00::","synset_id":"moderate.a.01"},{"index":2,"sense_key":"moderate%5:00:00:mild:00","synset_id":"moderate.s.01"},{"index":3,"sense_key":"moderate%5:00:00:temperate:00","synset_id":"moderate.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"moderate%5:00:00:mild:00","call_ids":["semeval2015.d000.s019.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"moderate%5:00:00:mild:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":712,"cached_input_tokens":0,"output_tokens":144,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0028640000000000002,"input_uncached_usd":0.001424,"input_cached_usd":0.0,"output_usd":0.00144,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":10.621616541990079}
{"item_id":"semeval2015.d000.s019.t005","gold_sense_keys":["severe%5:00:00:critical:03","severe%5:00:00:intense:00","severe%5:00:01:bad:00"],"candidates":[{"index":1,"sense_key":"severe%5:00:00:intense:00","synset_id":"severe.s.01"},{"index":2,"sense_key":"severe%5:00:00:strong:00","synset_id":"hard.s.01"},{"index":3,"sense_key":"severe%5:00:00:plain:01","synset_id":"austere.s.01"},{"index":4,"sense_key":"severe%5:00:02:nonindulgent:00","synset_id":"severe.s.04"},{"index":5,"sense_key":"severe%5:00:00:critical:03","synset_id":"dangerous.s.01"},{"index":6,"sense_key":"severe%5:00:01:bad:00","synset_id":"severe.s.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"severe%5:00:00:intense:00","call_ids":["semeval2015.d000.s019.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"severe%5:00:00:intense:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":956,"cached_input_tokens":0,"output_tokens":175,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003662,"input_uncached_usd":0.0019119999999999999,"input_cached_usd":0.0,"output_usd":0.00175,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.6470180419273674}
{"item_id":"semeval2015.d000.s019.t007","gold_sense_keys":["problem%1:09:00::","problem%1:26:00::"],"candidates":[{"index":1,"sense_key":"problem%1:26:00::","synset_id":"problem.n.01"},{"index":2,"sense_key":"problem%1:10:00::","synset_id":"problem.n.02"},{"index":3,"sense_key":"problem%1:09:00::","synset_id":"trouble.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"problem%1:09:00::","call_ids":["semeval2015.d000.s019.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"problem%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":698,"cached_input_tokens":0,"output_tokens":178,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003176,"input_uncached_usd":0.0013959999999999999,"input_cached_usd":0.0,"output_usd":0.0017800000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.439871708978899}
{"item_id":"semeval2015.d000.s020.t000","gold_sense_keys":["more%3:00:01::"],"candidates":[{"index":1,"sense_key":"more%3:00:01::","synset_id":"more.a.01"},{"index":2,"sense_key":"more%3:00:02::","synset_id":"more.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"more%3:00:01::","call_ids":["semeval2015.d000.s020.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"more%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":627,"cached_input_tokens":0,"output_tokens":46,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0017139999999999998,"input_uncached_usd":0.001254,"input_cached_usd":0.0,"output_usd":0.00046,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2156945000169799}
{"item_id":"semeval2015.d000.s020.t001","gold_sense_keys":["information%1:09:00::"],"candidates":[{"index":1,"sense_key":"information%1:10:00::","synset_id":"information.n.01"},{"index":2,"sense_key":"information%1:09:00::","synset_id":"information.n.02"},{"index":3,"sense_key":"information%1:10:02::","synset_id":"information.n.03"},{"index":4,"sense_key":"information%1:14:00::","synset_id":"data.n.01"},{"index":5,"sense_key":"information%1:07:00::","synset_id":"information.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"information%1:10:00::","call_ids":["semeval2015.d000.s020.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"information%1:10:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":715,"cached_input_tokens":0,"output_tokens":118,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00261,"input_uncached_usd":0.0014299999999999998,"input_cached_usd":0.0,"output_usd":0.00118,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.694691541022621}
{"item_id":"semeval2015.d000.s020.t002","gold_sense_keys":["see%2:39:13::"],"candidates":[{"index":1,"sense_key":"see%2:39:00::","synset_id":"see.v.01"},{"index":2,"sense_key":"see%2:31:01::","synset_id":"understand.v.02"},{"index":3,"sense_key":"see%2:39:02::","synset_id":"witness.v.02"},{"index":4,"sense_key":"see%2:36:00::","synset_id":"visualize.v.01"},{"index":5,"sense_key":"see%2:31:00::","synset_id":"see.v.05"},{"index":6,"sense_key":"see%2:31:02::","synset_id":"learn.v.02"},{"index":7,"sense_key":"see%2:39:03::","synset_id":"watch.v.03"},{"index":8,"sense_key":"see%2:38:00::","synset_id":"meet.v.01"},{"index":9,"sense_key":"see%2:32:00::","synset_id":"determine.v.08"},{"index":10,"sense_key":"see%2:31:03::","synset_id":"see.v.10"},{"index":11,"sense_key":"see%2:41:11::","synset_id":"see.v.11"},{"index":12,"sense_key":"see%2:41:12::","synset_id":"see.v.12"},{"index":13,"sense_key":"see%2:41:10::","synset_id":"visit.v.01"},{"index":14,"sense_key":"see%2:41:14::","synset_id":"attend.v.02"},{"index":15,"sense_key":"see%2:35:09::","synset_id":"see.v.15"},{"index":16,"sense_key":"see%2:41:00::","synset_id":"go_steady.v.01"},{"index":17,"sense_key":"see%2:39:05::","synset_id":"see.v.17"},{"index":18,"sense_key":"see%2:31:08::","synset_id":"see.v.18"},{"index":19,"sense_key":"see%2:39:11::","synset_id":"see.v.19"},{"index":20,"sense_key":"see%2:39:13::","synset_id":"examine.v.02"},{"index":21,"sense_key":"see%2:39:12::","synset_id":"experience.v.01"},{"index":22,"sense_key":"see%2:38:03::","synset_id":"see.v.22"},{"index":23,"sense_key":"see%2:33:00::","synset_id":"see.v.23"},{"index":24,"sense_key":"see%2:31:13::","synset_id":"interpret.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":20,"chosen_sense_key":"see%2:39:13::","call_ids":["semeval2015.d000.s020.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":20,"predicted_sense_key":"see%2:39:13::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2041,"cached_input_tokens":0,"output_tokens":215,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.006232,"input_uncached_usd":0.004082,"input_cached_usd":0.0,"output_usd":0.00215,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":12.393117083003744}
{"item_id":"semeval2015.d000.s020.t004","gold_sense_keys":["product%1:06:01::"],"candidates":[{"index":1,"sense_key":"product%1:06:01::","synset_id":"merchandise.n.01"},{"index":2,"sense_key":"product%1:06:00::","synset_id":"product.n.02"},{"index":3,"sense_key":"product%1:09:00::","synset_id":"product.n.03"},{"index":4,"sense_key":"product%1:27:00::","synset_id":"product.n.04"},{"index":5,"sense_key":"product%1:19:00::","synset_id":"product.n.05"},{"index":6,"sense_key":"product%1:14:00::","synset_id":"intersection.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"product%1:06:01::","call_ids":["semeval2015.d000.s020.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"product%1:06:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":885,"cached_input_tokens":0,"output_tokens":103,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0028,"input_uncached_usd":0.0017699999999999999,"input_cached_usd":0.0,"output_usd":0.00103,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6439337079646066}
{"item_id":"semeval2015.d000.s020.t006","gold_sense_keys":["part%1:09:00::","part%1:24:00::"],"candidates":[{"index":1,"sense_key":"part%1:24:00::","synset_id":"part.n.01"},{"index":2,"sense_key":"part%1:06:00::","synset_id":"part.n.02"},{"index":3,"sense_key":"part%1:17:00::","synset_id":"part.n.03"},{"index":4,"sense_key":"part%1:09:02::","synset_id":"part.n.04"},{"index":5,"sense_key":"part%1:15:00::","synset_id":"region.n.01"},{"index":6,"sense_key":"part%1:04:01::","synset_id":"function.n.03"},{"index":7,"sense_key":"part%1:09:01::","synset_id":"character.n.04"},{"index":8,"sense_key":"part%1:21:00::","synset_id":"share.n.01"},{"index":9,"sense_key":"part%1:09:00::","synset_id":"part.n.09"},{"index":10,"sense_key":"part%1:08:01::","synset_id":"part.n.10"},{"index":11,"sense_key":"part%1:10:00::","synset_id":"part.n.11"},{"index":12,"sense_key":"part%1:04:03::","synset_id":"contribution.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"part%1:09:00::","call_ids":["semeval2015.d000.s020.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"part%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1297,"cached_input_tokens":0,"output_tokens":145,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004044,"input_uncached_usd":0.002594,"input_cached_usd":0.0,"output_usd":0.0014500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8523002919973806}
{"item_id":"semeval2015.d000.s022.t000","gold_sense_keys":["active%3:00:07::"],"candidates":[{"index":1,"sense_key":"active%3:00:02::","synset_id":"active.a.01"},{"index":2,"sense_key":"active%5:00:00:operational:00","synset_id":"active.s.01"},{"index":3,"sense_key":"active%3:00:03::","synset_id":"active.a.03"},{"index":4,"sense_key":"active%5:00:00:involved:00","synset_id":"active.s.02"},{"index":5,"sense_key":"active%3:00:01::","synset_id":"active.a.05"},{"index":6,"sense_key":"active%3:00:07::","synset_id":"active.a.06"},{"index":7,"sense_key":"active%3:00:06::","synset_id":"active.a.07"},{"index":8,"sense_key":"active%5:00:00:existent:00","synset_id":"active.s.03"},{"index":9,"sense_key":"active%3:00:10::","synset_id":"active.a.09"},{"index":10,"sense_key":"active%3:00:09::","synset_id":"active.a.10"},{"index":11,"sense_key":"active%3:00:00::","synset_id":"active.a.11"},{"index":12,"sense_key":"active%3:00:05::","synset_id":"active.a.12"},{"index":13,"sense_key":"active%3:00:04::","synset_id":"active.a.13"},{"index":14,"sense_key":"active%3:00:08::","synset_id":"active.a.14"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"active%3:00:07::","call_ids":["semeval2015.d000.s022.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"active%3:00:07::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1291,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002702,"input_uncached_usd":0.002582,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0742703329306096}
{"item_id":"semeval2015.d000.s022.t001","gold_sense_keys":["substance%1:03:01::"],"candidates":[{"index":1,"sense_key":"substance%1:03:00::","synset_id":"substance.n.01"},{"index":2,"sense_key":"substance%1:09:01::","synset_id":"kernel.n.03"},{"index":3,"sense_key":"substance%1:09:00::","synset_id":"meaning.n.02"},{"index":4,"sense_key":"substance%1:27:00::","synset_id":"substance.n.04"},{"index":5,"sense_key":"substance%1:21:00::","synset_id":"means.n.03"},{"index":6,"sense_key":"substance%1:10:00::","synset_id":"message.n.02"},{"index":7,"sense_key":"substance%1:03:01::","synset_id":"substance.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"substance%1:03:01::","call_ids":["semeval2015.d000.s022.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"substance%1:03:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":941,"cached_input_tokens":0,"output_tokens":136,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003242,"input_uncached_usd":0.001882,"input_cached_usd":0.0,"output_usd":0.00136,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":5.790081624989398}
{"item_id":"semeval2015.d000.s022.t003","gold_sense_keys":["medicine%1:06:00::"],"candidates":[{"index":1,"sense_key":"medicine%1:09:00::","synset_id":"medicine.n.01"},{"index":2,"sense_key":"medicine%1:06:00::","synset_id":"medicine.n.02"},{"index":3,"sense_key":"medicine%1:04:00::","synset_id":"medicine.n.03"},{"index":4,"sense_key":"medicine%1:04:02::","synset_id":"music.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"medicine%1:06:00::","call_ids":["semeval2015.d000.s022.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"medicine%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":809,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001738,"input_uncached_usd":0.0016179999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1015649159671739}
{"item_id":"semeval2015.d000.s022.t004","gold_sense_keys":["medicine%1:06:00::"],"candidates":[{"index":1,"sense_key":"medicine%1:09:00::","synset_id":"medicine.n.01"},{"index":2,"sense_key":"medicine%1:06:00::","synset_id":"medicine.n.02"},{"index":3,"sense_key":"medicine%1:04:00::","synset_id":"medicine.n.03"},{"index":4,"sense_key":"medicine%1:04:02::","synset_id":"music.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"medicine%1:06:00::","call_ids":["semeval2015.d000.s022.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"medicine%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":809,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001738,"input_uncached_usd":0.0016179999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4816004999447614}
{"item_id":"semeval2015.d000.s022.t005","gold_sense_keys":["kill%2:35:00::","kill%2:35:01::","kill%2:35:02::"],"candidates":[{"index":1,"sense_key":"kill%2:35:00::","synset_id":"kill.v.01"},{"index":2,"sense_key":"kill%2:41:01::","synset_id":"kill.v.02"},{"index":3,"sense_key":"kill%2:30:08::","synset_id":"stamp_out.v.01"},{"index":4,"sense_key":"kill%2:42:00::","synset_id":"kill.v.04"},{"index":5,"sense_key":"kill%2:39:00::","synset_id":"kill.v.05"},{"index":6,"sense_key":"kill%2:37:00::","synset_id":"kill.v.06"},{"index":7,"sense_key":"kill%2:35:06::","synset_id":"kill.v.07"},{"index":8,"sense_key":"kill%2:35:05::","synset_id":"kill.v.08"},{"index":9,"sense_key":"kill%2:35:02::","synset_id":"kill.v.09"},{"index":10,"sense_key":"kill%2:35:01::","synset_id":"kill.v.10"},{"index":11,"sense_key":"kill%2:34:00::","synset_id":"toss_off.v.02"},{"index":12,"sense_key":"kill%2:30:00::","synset_id":"kill.v.12"},{"index":13,"sense_key":"kill%2:30:04::","synset_id":"kill.v.13"},{"index":14,"sense_key":"kill%2:30:03::","synset_id":"kill.v.14"},{"index":15,"sense_key":"kill%2:30:02::","synset_id":"kill.v.15"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"kill%2:35:02::","call_ids":["semeval2015.d000.s022.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"kill%2:35:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1358,"cached_input_tokens":0,"output_tokens":238,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.005096,"input_uncached_usd":0.0027159999999999997,"input_cached_usd":0.0,"output_usd":0.00238,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.8887623749906197}
{"item_id":"semeval2015.d000.s022.t006","gold_sense_keys":["cell%1:03:00::"],"candidates":[{"index":1,"sense_key":"cell%1:06:03::","synset_id":"cell.n.01"},{"index":2,"sense_key":"cell%1:03:00::","synset_id":"cell.n.02"},{"index":3,"sense_key":"cell%1:06:00::","synset_id":"cell.n.03"},{"index":4,"sense_key":"cell%1:14:00::","synset_id":"cell.n.04"},{"index":5,"sense_key":"cell%1:06:04::","synset_id":"cellular_telephone.n.01"},{"index":6,"sense_key":"cell%1:06:02::","synset_id":"cell.n.06"},{"index":7,"sense_key":"cell%1:06:01::","synset_id":"cell.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cell%1:03:00::","call_ids":["semeval2015.d000.s022.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cell%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":945,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00201,"input_uncached_usd":0.00189,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":10.796717833960429}
{"item_id":"semeval2015.d000.s022.t007","gold_sense_keys":["divide%2:35:00::"],"candidates":[{"index":1,"sense_key":"divide%2:41:00::","synset_id":"divide.v.01"},{"index":2,"sense_key":"divide%2:31:00::","synset_id":"divide.v.02"},{"index":3,"sense_key":"divide%2:42:00::","synset_id":"separate.v.01"},{"index":4,"sense_key":"divide%2:35:00::","synset_id":"separate.v.12"},{"index":5,"sense_key":"divide%2:38:00::","synset_id":"separate.v.07"},{"index":6,"sense_key":"divide%2:35:01::","synset_id":"separate.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"divide%2:35:00::","call_ids":["semeval2015.d000.s022.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"divide%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":885,"cached_input_tokens":0,"output_tokens":183,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0036,"input_uncached_usd":0.0017699999999999999,"input_cached_usd":0.0,"output_usd":0.0018300000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0077705000294372}
{"item_id":"semeval2015.d000.s023.t000","gold_sense_keys":["body%1:08:00::"],"candidates":[{"index":1,"sense_key":"body%1:08:00::","synset_id":"body.n.01"},{"index":2,"sense_key":"body%1:14:00::","synset_id":"body.n.02"},{"index":3,"sense_key":"body%1:08:02::","synset_id":"body.n.03"},{"index":4,"sense_key":"body%1:17:00::","synset_id":"body.n.04"},{"index":5,"sense_key":"body%1:08:01::","synset_id":"torso.n.01"},{"index":6,"sense_key":"body%1:14:02::","synset_id":"body.n.06"},{"index":7,"sense_key":"body%1:07:00::","synset_id":"consistency.n.01"},{"index":8,"sense_key":"body%1:10:00::","synset_id":"body.n.08"},{"index":9,"sense_key":"body%1:07:01::","synset_id":"body.n.09"},{"index":10,"sense_key":"body%1:06:01::","synset_id":"soundbox.n.01"},{"index":11,"sense_key":"body%1:06:00::","synset_id":"body.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"body%1:08:00::","call_ids":["semeval2015.d000.s023.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"body%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1098,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002316,"input_uncached_usd":0.002196,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9694392499513924}
{"item_id":"semeval2015.d000.s023.t001","gold_sense_keys":["convert%2:30:03::"],"candidates":[{"index":1,"sense_key":"convert%2:30:00::","synset_id":"convert.v.01"},{"index":2,"sense_key":"convert%2:30:03::","synset_id":"convert.v.02"},{"index":3,"sense_key":"convert%2:30:01::","synset_id":"convert.v.03"},{"index":4,"sense_key":"convert%2:30:02::","synset_id":"change.v.06"},{"index":5,"sense_key":"convert%2:30:07::","synset_id":"convert.v.05"},{"index":6,"sense_key":"convert%2:33:02::","synset_id":"convert.v.06"},{"index":7,"sense_key":"convert%2:33:01::","synset_id":"convert.v.07"},{"index":8,"sense_key":"convert%2:33:00::","synset_id":"convert.v.08"},{"index":9,"sense_key":"convert%2:32:00::","synset_id":"convert.v.09"},{"index":10,"sense_key":"convert%2:30:09::","synset_id":"commute.v.04"},{"index":11,"sense_key":"convert%2:30:04::","synset_id":"convert.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"convert%2:30:04::","call_ids":["semeval2015.d000.s023.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"convert%2:30:04::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1100,"cached_input_tokens":0,"output_tokens":138,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00358,"input_uncached_usd":0.0021999999999999997,"input_cached_usd":0.0,"output_usd":0.0013800000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9284387910738587}
{"item_id":"semeval2015.d000.s023.t002","gold_sense_keys":["active%3:00:07::"],"candidates":[{"index":1,"sense_key":"active%3:00:02::","synset_id":"active.a.01"},{"index":2,"sense_key":"active%5:00:00:operational:00","synset_id":"active.s.01"},{"index":3,"sense_key":"active%3:00:03::","synset_id":"active.a.03"},{"index":4,"sense_key":"active%5:00:00:involved:00","synset_id":"active.s.02"},{"index":5,"sense_key":"active%3:00:01::","synset_id":"active.a.05"},{"index":6,"sense_key":"active%3:00:07::","synset_id":"active.a.06"},{"index":7,"sense_key":"active%3:00:06::","synset_id":"active.a.07"},{"index":8,"sense_key":"active%5:00:00:existent:00","synset_id":"active.s.03"},{"index":9,"sense_key":"active%3:00:10::","synset_id":"active.a.09"},{"index":10,"sense_key":"active%3:00:09::","synset_id":"active.a.10"},{"index":11,"sense_key":"active%3:00:00::","synset_id":"active.a.11"},{"index":12,"sense_key":"active%3:00:05::","synset_id":"active.a.12"},{"index":13,"sense_key":"active%3:00:04::","synset_id":"active.a.13"},{"index":14,"sense_key":"active%3:00:08::","synset_id":"active.a.14"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"active%3:00:07::","call_ids":["semeval2015.d000.s023.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"active%3:00:07::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1237,"cached_input_tokens":0,"output_tokens":74,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0032140000000000003,"input_uncached_usd":0.002474,"input_cached_usd":0.0,"output_usd":0.0007400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4732594169909135}
{"item_id":"semeval2015.d000.s023.t003","gold_sense_keys":["form%1:07:03::"],"candidates":[{"index":1,"sense_key":"form%1:10:00::","synset_id":"form.n.01"},{"index":2,"sense_key":"form%1:09:01::","synset_id":"kind.n.01"},{"index":3,"sense_key":"form%1:09:00::","synset_id":"form.n.03"},{"index":4,"sense_key":"form%1:07:01::","synset_id":"shape.n.01"},{"index":5,"sense_key":"form%1:08:00::","synset_id":"human_body.n.01"},{"index":6,"sense_key":"form%1:03:00::","synset_id":"shape.n.02"},{"index":7,"sense_key":"form%1:07:00::","synset_id":"form.n.07"},{"index":8,"sense_key":"form%1:10:01::","synset_id":"form.n.08"},{"index":9,"sense_key":"form%1:14:02::","synset_id":"form.n.09"},{"index":10,"sense_key":"form%1:10:02::","synset_id":"form.n.10"},{"index":11,"sense_key":"form%1:07:03::","synset_id":"form.n.11"},{"index":12,"sense_key":"form%1:26:00::","synset_id":"phase.n.02"},{"index":13,"sense_key":"form%1:14:00::","synset_id":"class.n.02"},{"index":14,"sense_key":"form%1:07:02::","synset_id":"form.n.14"},{"index":15,"sense_key":"form%1:06:00::","synset_id":"mannequin.n.02"},{"index":16,"sense_key":"form%1:06:01::","synset_id":"form.n.16"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"form%1:07:03::","call_ids":["semeval2015.d000.s023.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"form%1:07:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1553,"cached_input_tokens":0,"output_tokens":131,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004416,"input_uncached_usd":0.003106,"input_cached_usd":0.0,"output_usd":0.0013100000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.2128365410026163}
{"item_id":"semeval2015.d000.s023.t004","gold_sense_keys":["block%2:41:01::"],"candidates":[{"index":1,"sense_key":"block%2:35:00::","synset_id":"barricade.v.01"},{"index":2,"sense_key":"block%2:41:00::","synset_id":"obstruct.v.01"},{"index":3,"sense_key":"block%2:41:01::","synset_id":"stop.v.03"},{"index":4,"sense_key":"block%2:32:00::","synset_id":"jam.v.04"},{"index":5,"sense_key":"block%2:41:03::","synset_id":"block.v.05"},{"index":6,"sense_key":"block%2:39:05::","synset_id":"block.v.06"},{"index":7,"sense_key":"block%2:39:00::","synset_id":"obstruct.v.03"},{"index":8,"sense_key":"block%2:35:03::","synset_id":"block.v.08"},{"index":9,"sense_key":"block%2:35:02::","synset_id":"stuff.v.03"},{"index":10,"sense_key":"block%2:35:12::","synset_id":"obstruct.v.02"},{"index":11,"sense_key":"block%2:35:06::","synset_id":"block.v.11"},{"index":12,"sense_key":"block%2:33:00::","synset_id":"parry.v.01"},{"index":13,"sense_key":"block%2:31:00::","synset_id":"forget.v.02"},{"index":14,"sense_key":"block%2:30:01::","synset_id":"block.v.14"},{"index":15,"sense_key":"block%2:30:00::","synset_id":"block.v.15"},{"index":16,"sense_key":"block%2:30:02::","synset_id":"freeze.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"block%2:41:00::","call_ids":["semeval2015.d000.s023.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"block%2:41:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1452,"cached_input_tokens":0,"output_tokens":155,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004454,"input_uncached_usd":0.002904,"input_cached_usd":0.0,"output_usd":0.0015500000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1320485420292243}
{"item_id":"semeval2015.d000.s023.t007","gold_sense_keys":["involved%3:00:00::"],"candidates":[{"index":1,"sense_key":"involved%3:00:00::","synset_id":"involved.a.01"},{"index":2,"sense_key":"involved%5:00:00:encumbered:00","synset_id":"involved.s.01"},{"index":3,"sense_key":"involved%5:00:00:attached:01","synset_id":"involved.s.02"},{"index":4,"sense_key":"involved%5:00:00:complex:00","synset_id":"byzantine.s.01"},{"index":5,"sense_key":"involved%5:00:00:enclosed:00","synset_id":"involved.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"involved%3:00:00::","call_ids":["semeval2015.d000.s023.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"involved%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":844,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001808,"input_uncached_usd":0.0016879999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":14.148904333007522}
{"item_id":"semeval2015.d000.s023.t008","gold_sense_keys":["produce%2:36:02::","produce%2:36:03::"],"candidates":[{"index":1,"sense_key":"produce%2:36:02::","synset_id":"produce.v.01"},{"index":2,"sense_key":"produce%2:36:00::","synset_id":"produce.v.02"},{"index":3,"sense_key":"produce%2:36:03::","synset_id":"produce.v.03"},{"index":4,"sense_key":"produce%2:39:01::","synset_id":"produce.v.04"},{"index":5,"sense_key":"produce%2:36:05::","synset_id":"grow.v.07"},{"index":6,"sense_key":"produce%2:39:00::","synset_id":"produce.v.06"},{"index":7,"sense_key":"produce%2:29:00::","synset_id":"grow.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"produce%2:36:00::","call_ids":["semeval2015.d000.s023.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"produce%2:36:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":983,"cached_input_tokens":0,"output_tokens":161,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003576,"input_uncached_usd":0.001966,"input_cached_usd":0.0,"output_usd":0.00161,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":15.359963417053223}
{"item_id":"semeval2015.d000.s023.t010","gold_sense_keys":["building_block%1:17:00::"],"candidates":[{"index":1,"sense_key":"building_block%1:17:00::","synset_id":"unit.n.05"},{"index":2,"sense_key":"building_block%1:06:00::","synset_id":"building_block.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"building_block%1:17:00::","call_ids":["semeval2015.d000.s023.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"building_block%1:17:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":579,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001278,"input_uncached_usd":0.001158,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9862061670282856}
{"item_id":"semeval2015.d000.s023.t013","gold_sense_keys":["genetic%3:01:02::"],"candidates":[{"index":1,"sense_key":"genetic%5:00:00:inheritable:00","synset_id":"familial.s.01"},{"index":2,"sense_key":"genetic%3:01:02::","synset_id":"genic.a.01"},{"index":3,"sense_key":"genetic%3:01:01::","synset_id":"genetic.a.03"},{"index":4,"sense_key":"genetic%3:01:00::","synset_id":"genetic.a.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"genetic%3:01:02::","call_ids":["semeval2015.d000.s023.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"genetic%3:01:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":738,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0022459999999999997,"input_uncached_usd":0.0014759999999999999,"input_cached_usd":0.0,"output_usd":0.0007700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5212627080036327}
{"item_id":"semeval2015.d000.s023.t014","gold_sense_keys":["material%1:27:00::"],"candidates":[{"index":1,"sense_key":"material%1:27:00::","synset_id":"material.n.01"},{"index":2,"sense_key":"material%1:10:00::","synset_id":"material.n.02"},{"index":3,"sense_key":"material%1:06:00::","synset_id":"fabric.n.01"},{"index":4,"sense_key":"material%1:06:01::","synset_id":"material.n.04"},{"index":5,"sense_key":"material%1:18:00::","synset_id":"material.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"material%1:27:00::","call_ids":["semeval2015.d000.s023.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"material%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":829,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001778,"input_uncached_usd":0.001658,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.40905970800668}
{"item_id":"semeval2015.d000.s023.t015","gold_sense_keys":["cell%1:03:00::"],"candidates":[{"index":1,"sense_key":"cell%1:06:03::","synset_id":"cell.n.01"},{"index":2,"sense_key":"cell%1:03:00::","synset_id":"cell.n.02"},{"index":3,"sense_key":"cell%1:06:00::","synset_id":"cell.n.03"},{"index":4,"sense_key":"cell%1:14:00::","synset_id":"cell.n.04"},{"index":5,"sense_key":"cell%1:06:04::","synset_id":"cellular_telephone.n.01"},{"index":6,"sense_key":"cell%1:06:02::","synset_id":"cell.n.06"},{"index":7,"sense_key":"cell%1:06:01::","synset_id":"cell.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cell%1:03:00::","call_ids":["semeval2015.d000.s023.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cell%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":891,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001902,"input_uncached_usd":0.001782,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.40700912498869}
{"item_id":"semeval2015.d000.s024.t000","gold_sense_keys":["study%2:31:02::"],"candidates":[{"index":1,"sense_key":"study%2:31:02::","synset_id":"analyze.v.01"},{"index":2,"sense_key":"study%2:31:03::","synset_id":"study.v.02"},{"index":3,"sense_key":"study%2:39:00::","synset_id":"study.v.03"},{"index":4,"sense_key":"study%2:31:00::","synset_id":"learn.v.04"},{"index":5,"sense_key":"study%2:31:01::","synset_id":"study.v.05"},{"index":6,"sense_key":"study%2:31:04::","synset_id":"study.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"study%2:31:02::","call_ids":["semeval2015.d000.s024.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"study%2:31:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":869,"cached_input_tokens":0,"output_tokens":74,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002478,"input_uncached_usd":0.001738,"input_cached_usd":0.0,"output_usd":0.0007400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7797623330261558}
{"item_id":"semeval2015.d000.s025.t000","gold_sense_keys":["treatment%1:04:00::"],"candidates":[{"index":1,"sense_key":"treatment%1:04:00::","synset_id":"treatment.n.01"},{"index":2,"sense_key":"treatment%1:04:01::","synset_id":"treatment.n.02"},{"index":3,"sense_key":"treatment%1:07:00::","synset_id":"treatment.n.03"},{"index":4,"sense_key":"treatment%1:10:00::","synset_id":"discussion.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"treatment%1:04:00::","call_ids":["semeval2015.d000.s025.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"treatment%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":765,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00165,"input_uncached_usd":0.00153,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4363589169224724}
{"item_id":"semeval2015.d000.s025.t004","gold_sense_keys":["study%2:31:02::"],"candidates":[{"index":1,"sense_key":"study%2:31:02::","synset_id":"analyze.v.01"},{"index":2,"sense_key":"study%2:31:03::","synset_id":"study.v.02"},{"index":3,"sense_key":"study%2:39:00::","synset_id":"study.v.03"},{"index":4,"sense_key":"study%2:31:00::","synset_id":"learn.v.04"},{"index":5,"sense_key":"study%2:31:01::","synset_id":"study.v.05"},{"index":6,"sense_key":"study%2:31:04::","synset_id":"study.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"study%2:31:02::","call_ids":["semeval2015.d000.s025.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"study%2:31:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":873,"cached_input_tokens":0,"output_tokens":79,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002536,"input_uncached_usd":0.001746,"input_cached_usd":0.0,"output_usd":0.00079,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":14.12251383299008}
{"item_id":"semeval2015.d000.s025.t005","gold_sense_keys":["main%5:00:00:important:00"],"candidates":[{"index":1,"sense_key":"main%5:00:00:important:00","synset_id":"chief.s.01"},{"index":2,"sense_key":"main%3:00:02::","synset_id":"independent.a.03"},{"index":3,"sense_key":"main%5:00:00:intense:00","synset_id":"main.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"main%5:00:00:important:00","call_ids":["semeval2015.d000.s025.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"main%5:00:00:important:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":684,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001488,"input_uncached_usd":0.0013679999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9727358749369159}
{"item_id":"semeval2015.d000.s025.t006","gold_sense_keys":["study%1:04:00::"],"candidates":[{"index":1,"sense_key":"study%1:04:00::","synset_id":"survey.n.01"},{"index":2,"sense_key":"study%1:09:03::","synset_id":"study.n.02"},{"index":3,"sense_key":"study%1:10:00::","synset_id":"report.n.01"},{"index":4,"sense_key":"study%1:09:04::","synset_id":"study.n.04"},{"index":5,"sense_key":"study%1:06:00::","synset_id":"study.n.05"},{"index":6,"sense_key":"study%1:09:02::","synset_id":"discipline.n.01"},{"index":7,"sense_key":"study%1:06:01::","synset_id":"sketch.n.01"},{"index":8,"sense_key":"study%1:09:01::","synset_id":"cogitation.n.02"},{"index":9,"sense_key":"study%1:18:00::","synset_id":"study.n.09"},{"index":10,"sense_key":"study%1:10:01::","synset_id":"study.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"study%1:04:00::","call_ids":["semeval2015.d000.s025.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"study%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1078,"cached_input_tokens":0,"output_tokens":178,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003936,"input_uncached_usd":0.002156,"input_cached_usd":0.0,"output_usd":0.0017800000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.9988344999728724}
{"item_id":"semeval2015.d000.s025.t007","gold_sense_keys":["patient%1:18:00::"],"candidates":[{"index":1,"sense_key":"patient%1:18:00::","synset_id":"patient.n.01"},{"index":2,"sense_key":"patient%1:10:00::","synset_id":"affected_role.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"patient%1:18:00::","call_ids":["semeval2015.d000.s025.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"patient%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":616,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0013520000000000001,"input_uncached_usd":0.001232,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0787337078945711}
{"item_id":"semeval2015.d000.s025.t012","gold_sense_keys":["before%4:02:03::"],"candidates":[{"index":1,"sense_key":"before%4:02:03::","synset_id":"earlier.r.01"},{"index":2,"sense_key":"before%4:02:01::","synset_id":"ahead.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"before%4:02:03::","call_ids":["semeval2015.d000.s025.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"before%4:02:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":605,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00133,"input_uncached_usd":0.00121,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.065688458038494}
{"item_id":"semeval2015.d000.s026.t000","gold_sense_keys":["effect%1:26:00::"],"candidates":[{"index":1,"sense_key":"effect%1:19:00::","synset_id":"consequence.n.01"},{"index":2,"sense_key":"effect%1:07:00::","synset_id":"impression.n.02"},{"index":3,"sense_key":"effect%1:09:00::","synset_id":"effect.n.03"},{"index":4,"sense_key":"effect%1:10:00::","synset_id":"effect.n.04"},{"index":5,"sense_key":"effect%1:07:01::","synset_id":"effect.n.05"},{"index":6,"sense_key":"effect%1:26:00::","synset_id":"effect.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"effect%1:26:00::","call_ids":["semeval2015.d000.s026.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"effect%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":924,"cached_input_tokens":0,"output_tokens":116,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003008,"input_uncached_usd":0.0018479999999999998,"input_cached_usd":0.0,"output_usd":0.00116,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.881462667020969}
{"item_id":"semeval2015.d000.s026.t002","gold_sense_keys":["compare%2:31:00::"],"candidates":[{"index":1,"sense_key":"compare%2:31:00::","synset_id":"compare.v.01"},{"index":2,"sense_key":"compare%2:42:00::","synset_id":"compare.v.02"},{"index":3,"sense_key":"compare%2:31:01::","synset_id":"compare.v.03"},{"index":4,"sense_key":"compare%2:32:00::","synset_id":"compare.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"compare%2:31:00::","call_ids":["semeval2015.d000.s026.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"compare%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":784,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001688,"input_uncached_usd":0.001568,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.803230666089803}
{"item_id":"semeval2015.d000.s027.t000","gold_sense_keys":["treatment%1:04:00::"],"candidates":[{"index":1,"sense_key":"treatment%1:04:00::","synset_id":"treatment.n.01"},{"index":2,"sense_key":"treatment%1:04:01::","synset_id":"treatment.n.02"},{"index":3,"sense_key":"treatment%1:07:00::","synset_id":"treatment.n.03"},{"index":4,"sense_key":"treatment%1:10:00::","synset_id":"discussion.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"treatment%1:04:00::","call_ids":["semeval2015.d000.s027.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"treatment%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":880,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00188,"input_uncached_usd":0.0017599999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0156457918928936}
{"item_id":"semeval2015.d000.s027.t001","gold_sense_keys":["cell%1:03:00::"],"candidates":[{"index":1,"sense_key":"cell%1:06:03::","synset_id":"cell.n.01"},{"index":2,"sense_key":"cell%1:03:00::","synset_id":"cell.n.02"},{"index":3,"sense_key":"cell%1:06:00::","synset_id":"cell.n.03"},{"index":4,"sense_key":"cell%1:14:00::","synset_id":"cell.n.04"},{"index":5,"sense_key":"cell%1:06:04::","synset_id":"cellular_telephone.n.01"},{"index":6,"sense_key":"cell%1:06:02::","synset_id":"cell.n.06"},{"index":7,"sense_key":"cell%1:06:01::","synset_id":"cell.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cell%1:03:00::","call_ids":["semeval2015.d000.s027.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cell%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1016,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002152,"input_uncached_usd":0.002032,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1438110830495134}
{"item_id":"semeval2015.d000.s027.t003","gold_sense_keys":["effect%1:26:00::"],"candidates":[{"index":1,"sense_key":"effect%1:19:00::","synset_id":"consequence.n.01"},{"index":2,"sense_key":"effect%1:07:00::","synset_id":"impression.n.02"},{"index":3,"sense_key":"effect%1:09:00::","synset_id":"effect.n.03"},{"index":4,"sense_key":"effect%1:10:00::","synset_id":"effect.n.04"},{"index":5,"sense_key":"effect%1:07:01::","synset_id":"effect.n.05"},{"index":6,"sense_key":"effect%1:26:00::","synset_id":"effect.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"effect%1:26:00::","call_ids":["semeval2015.d000.s027.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"effect%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":984,"cached_input_tokens":0,"output_tokens":113,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003098,"input_uncached_usd":0.0019679999999999997,"input_cached_usd":0.0,"output_usd":0.0011300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.104536416940391}
{"item_id":"semeval2015.d000.s027.t004","gold_sense_keys":["compare%2:31:00::"],"candidates":[{"index":1,"sense_key":"compare%2:31:00::","synset_id":"compare.v.01"},{"index":2,"sense_key":"compare%2:42:00::","synset_id":"compare.v.02"},{"index":3,"sense_key":"compare%2:31:01::","synset_id":"compare.v.03"},{"index":4,"sense_key":"compare%2:32:00::","synset_id":"compare.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"compare%2:31:00::","call_ids":["semeval2015.d000.s027.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"compare%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":844,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0023179999999999997,"input_uncached_usd":0.0016879999999999998,"input_cached_usd":0.0,"output_usd":0.00063,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2939667910104617}
{"item_id":"semeval2015.d000.s027.t006","gold_sense_keys":["medicine%1:06:00::"],"candidates":[{"index":1,"sense_key":"medicine%1:09:00::","synset_id":"medicine.n.01"},{"index":2,"sense_key":"medicine%1:06:00::","synset_id":"medicine.n.02"},{"index":3,"sense_key":"medicine%1:04:00::","synset_id":"medicine.n.03"},{"index":4,"sense_key":"medicine%1:04:02::","synset_id":"music.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"medicine%1:06:00::","call_ids":["semeval2015.d000.s027.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"medicine%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":880,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00188,"input_uncached_usd":0.0017599999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3253307499689981}
{"item_id":"semeval2015.d000.s027.t007","gold_sense_keys":["study%1:04:00::"],"candidates":[{"index":1,"sense_key":"study%1:04:00::","synset_id":"survey.n.01"},{"index":2,"sense_key":"study%1:09:03::","synset_id":"study.n.02"},{"index":3,"sense_key":"study%1:10:00::","synset_id":"report.n.01"},{"index":4,"sense_key":"study%1:09:04::","synset_id":"study.n.04"},{"index":5,"sense_key":"study%1:06:00::","synset_id":"study.n.05"},{"index":6,"sense_key":"study%1:09:02::","synset_id":"discipline.n.01"},{"index":7,"sense_key":"study%1:06:01::","synset_id":"sketch.n.01"},{"index":8,"sense_key":"study%1:09:01::","synset_id":"cogitation.n.02"},{"index":9,"sense_key":"study%1:18:00::","synset_id":"study.n.09"},{"index":10,"sense_key":"study%1:10:01::","synset_id":"study.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"study%1:04:00::","call_ids":["semeval2015.d000.s027.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"study%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1193,"cached_input_tokens":0,"output_tokens":185,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004236,"input_uncached_usd":0.002386,"input_cached_usd":0.0,"output_usd":0.00185,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2420442500151694}
{"item_id":"semeval2015.d000.s027.t008","gold_sense_keys":["involve%2:42:04::"],"candidates":[{"index":1,"sense_key":"involve%2:42:01::","synset_id":"involve.v.01"},{"index":2,"sense_key":"involve%2:42:04::","synset_id":"involve.v.02"},{"index":3,"sense_key":"involve%2:42:00::","synset_id":"imply.v.05"},{"index":4,"sense_key":"involve%2:42:07::","synset_id":"necessitate.v.01"},{"index":5,"sense_key":"involve%2:42:02::","synset_id":"involve.v.05"},{"index":6,"sense_key":"involve%2:31:00::","synset_id":"involve.v.06"},{"index":7,"sense_key":"involve%2:30:00::","synset_id":"involve.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"involve%2:42:04::","call_ids":["semeval2015.d000.s027.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"involve%2:42:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":983,"cached_input_tokens":0,"output_tokens":162,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0035859999999999998,"input_uncached_usd":0.001966,"input_cached_usd":0.0,"output_usd":0.0016200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2779916250146925}
{"item_id":"semeval2015.d000.s027.t009","gold_sense_keys":["patient%1:18:00::"],"candidates":[{"index":1,"sense_key":"patient%1:18:00::","synset_id":"patient.n.01"},{"index":2,"sense_key":"patient%1:10:00::","synset_id":"affected_role.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"patient%1:18:00::","call_ids":["semeval2015.d000.s027.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"patient%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":731,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001582,"input_uncached_usd":0.001462,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6845082499785349}
{"item_id":"semeval2015.d000.s027.t010","gold_sense_keys":["locally%4:02:01::"],"candidates":[{"index":1,"sense_key":"locally%4:02:00::","synset_id":"locally.r.01"},{"index":2,"sense_key":"locally%4:02:01::","synset_id":"locally.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"locally%4:02:01::","call_ids":["semeval2015.d000.s027.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"locally%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":696,"cached_input_tokens":0,"output_tokens":87,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002262,"input_uncached_usd":0.001392,"input_cached_usd":0.0,"output_usd":0.0008700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":11.397680375026539}
{"item_id":"semeval2015.d000.s027.t011","gold_sense_keys":["advanced%5:00:00:late:02"],"candidates":[{"index":1,"sense_key":"advanced%5:00:00:precocious:00","synset_id":"advanced.s.01"},{"index":2,"sense_key":"advanced%5:00:00:late:02","synset_id":"advanced.s.02"},{"index":3,"sense_key":"advanced%5:00:00:progressive:01","synset_id":"advanced.s.03"},{"index":4,"sense_key":"advanced%5:00:02:high:02","synset_id":"advanced.s.04"},{"index":5,"sense_key":"advanced%5:00:00:high-tech:00","synset_id":"advanced.s.05"},{"index":6,"sense_key":"advanced%5:00:00:late:00","synset_id":"advanced.s.06"},{"index":7,"sense_key":"advanced%5:00:00:civilized:00","synset_id":"advanced.s.07"},{"index":8,"sense_key":"advanced%5:00:00:front:00","synset_id":"advance.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"advanced%5:00:00:late:02","call_ids":["semeval2015.d000.s027.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"advanced%5:00:00:late:02","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1154,"cached_input_tokens":0,"output_tokens":81,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0031179999999999997,"input_uncached_usd":0.0023079999999999997,"input_cached_usd":0.0,"output_usd":0.0008100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":15.274815249955282}
{"item_id":"semeval2015.d000.s027.t016","gold_sense_keys":["past%1:28:00::"],"candidates":[{"index":1,"sense_key":"past%1:28:00::","synset_id":"past.n.01"},{"index":2,"sense_key":"past%1:28:01::","synset_id":"past.n.02"},{"index":3,"sense_key":"past%1:10:00::","synset_id":"past.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"past%1:28:00::","call_ids":["semeval2015.d000.s027.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"past%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":764,"cached_input_tokens":0,"output_tokens":78,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002308,"input_uncached_usd":0.0015279999999999998,"input_cached_usd":0.0,"output_usd":0.0007800000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":14.857291708001867}
{"item_id":"semeval2015.d000.s028.t001","gold_sense_keys":["compare%2:31:00::"],"candidates":[{"index":1,"sense_key":"compare%2:31:00::","synset_id":"compare.v.01"},{"index":2,"sense_key":"compare%2:42:00::","synset_id":"compare.v.02"},{"index":3,"sense_key":"compare%2:31:01::","synset_id":"compare.v.03"},{"index":4,"sense_key":"compare%2:32:00::","synset_id":"compare.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"compare%2:31:00::","call_ids":["semeval2015.d000.s028.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"compare%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":803,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001726,"input_uncached_usd":0.001606,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.514172959025018}
{"item_id":"semeval2015.d000.s028.t003","gold_sense_keys":["medicine%1:06:00::"],"candidates":[{"index":1,"sense_key":"medicine%1:09:00::","synset_id":"medicine.n.01"},{"index":2,"sense_key":"medicine%1:06:00::","synset_id":"medicine.n.02"},{"index":3,"sense_key":"medicine%1:04:00::","synset_id":"medicine.n.03"},{"index":4,"sense_key":"medicine%1:04:02::","synset_id":"music.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"medicine%1:06:00::","call_ids":["semeval2015.d000.s028.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"medicine%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":839,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0017980000000000001,"input_uncached_usd":0.001678,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.020244290935807}
{"item_id":"semeval2015.d000.s028.t005","gold_sense_keys":["study%1:04:00::"],"candidates":[{"index":1,"sense_key":"study%1:04:00::","synset_id":"survey.n.01"},{"index":2,"sense_key":"study%1:09:03::","synset_id":"study.n.02"},{"index":3,"sense_key":"study%1:10:00::","synset_id":"report.n.01"},{"index":4,"sense_key":"study%1:09:04::","synset_id":"study.n.04"},{"index":5,"sense_key":"study%1:06:00::","synset_id":"study.n.05"},{"index":6,"sense_key":"study%1:09:02::","synset_id":"discipline.n.01"},{"index":7,"sense_key":"study%1:06:01::","synset_id":"sketch.n.01"},{"index":8,"sense_key":"study%1:09:01::","synset_id":"cogitation.n.02"},{"index":9,"sense_key":"study%1:18:00::","synset_id":"study.n.09"},{"index":10,"sense_key":"study%1:10:01::","synset_id":"study.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"study%1:10:00::","call_ids":["semeval2015.d000.s028.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"study%1:10:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1152,"cached_input_tokens":0,"output_tokens":145,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0037540000000000004,"input_uncached_usd":0.002304,"input_cached_usd":0.0,"output_usd":0.0014500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9624470000853762}
{"item_id":"semeval2015.d000.s028.t006","gold_sense_keys":["involve%2:42:04::"],"candidates":[{"index":1,"sense_key":"involve%2:42:01::","synset_id":"involve.v.01"},{"index":2,"sense_key":"involve%2:42:04::","synset_id":"involve.v.02"},{"index":3,"sense_key":"involve%2:42:00::","synset_id":"imply.v.05"},{"index":4,"sense_key":"involve%2:42:07::","synset_id":"necessitate.v.01"},{"index":5,"sense_key":"involve%2:42:02::","synset_id":"involve.v.05"},{"index":6,"sense_key":"involve%2:31:00::","synset_id":"involve.v.06"},{"index":7,"sense_key":"involve%2:30:00::","synset_id":"involve.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"involve%2:42:02::","call_ids":["semeval2015.d000.s028.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"involve%2:42:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":942,"cached_input_tokens":0,"output_tokens":205,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003934,"input_uncached_usd":0.0018839999999999998,"input_cached_usd":0.0,"output_usd":0.00205,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.0628425419563428}
{"item_id":"semeval2015.d000.s028.t007","gold_sense_keys":["patient%1:18:00::"],"candidates":[{"index":1,"sense_key":"patient%1:18:00::","synset_id":"patient.n.01"},{"index":2,"sense_key":"patient%1:10:00::","synset_id":"affected_role.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"patient%1:18:00::","call_ids":["semeval2015.d000.s028.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"patient%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":690,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0015,"input_uncached_usd":0.00138,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6378985829651356}
{"item_id":"semeval2015.d000.s028.t012","gold_sense_keys":["past%1:28:00::"],"candidates":[{"index":1,"sense_key":"past%1:28:00::","synset_id":"past.n.01"},{"index":2,"sense_key":"past%1:28:01::","synset_id":"past.n.02"},{"index":3,"sense_key":"past%1:10:00::","synset_id":"past.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"past%1:28:00::","call_ids":["semeval2015.d000.s028.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"past%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":723,"cached_input_tokens":0,"output_tokens":81,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002256,"input_uncached_usd":0.001446,"input_cached_usd":0.0,"output_usd":0.0008100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5509128749836236}
{"item_id":"semeval2015.d000.s029.t000","gold_sense_keys":["study%1:04:00::"],"candidates":[{"index":1,"sense_key":"study%1:04:00::","synset_id":"survey.n.01"},{"index":2,"sense_key":"study%1:09:03::","synset_id":"study.n.02"},{"index":3,"sense_key":"study%1:10:00::","synset_id":"report.n.01"},{"index":4,"sense_key":"study%1:09:04::","synset_id":"study.n.04"},{"index":5,"sense_key":"study%1:06:00::","synset_id":"study.n.05"},{"index":6,"sense_key":"study%1:09:02::","synset_id":"discipline.n.01"},{"index":7,"sense_key":"study%1:06:01::","synset_id":"sketch.n.01"},{"index":8,"sense_key":"study%1:09:01::","synset_id":"cogitation.n.02"},{"index":9,"sense_key":"study%1:18:00::","synset_id":"study.n.09"},{"index":10,"sense_key":"study%1:10:01::","synset_id":"study.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"study%1:04:00::","call_ids":["semeval2015.d000.s029.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"study%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1103,"cached_input_tokens":0,"output_tokens":170,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003906,"input_uncached_usd":0.002206,"input_cached_usd":0.0,"output_usd":0.0017000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.781136417062953}
{"item_id":"semeval2015.d000.s029.t001","gold_sense_keys":["main%5:00:00:important:00"],"candidates":[{"index":1,"sense_key":"main%5:00:00:important:00","synset_id":"chief.s.01"},{"index":2,"sense_key":"main%3:00:02::","synset_id":"independent.a.03"},{"index":3,"sense_key":"main%5:00:00:intense:00","synset_id":"main.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"main%5:00:00:important:00","call_ids":["semeval2015.d000.s029.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"main%5:00:00:important:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":708,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001536,"input_uncached_usd":0.001416,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2479945410741493}
{"item_id":"semeval2015.d000.s029.t002","gold_sense_keys":["measure%1:10:03::"],"candidates":[{"index":1,"sense_key":"measure%1:04:01::","synset_id":"measure.n.01"},{"index":2,"sense_key":"measure%1:03:00::","synset_id":"measure.n.02"},{"index":3,"sense_key":"measure%1:10:02::","synset_id":"bill.n.01"},{"index":4,"sense_key":"measure%1:04:00::","synset_id":"measurement.n.01"},{"index":5,"sense_key":"measure%1:10:03::","synset_id":"standard.n.01"},{"index":6,"sense_key":"measure%1:10:01::","synset_id":"meter.n.03"},{"index":7,"sense_key":"measure%1:10:00::","synset_id":"measure.n.07"},{"index":8,"sense_key":"measure%1:06:00::","synset_id":"measuring_stick.n.01"},{"index":9,"sense_key":"measure%1:06:01::","synset_id":"measure.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"measure%1:10:03::","call_ids":["semeval2015.d000.s029.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"measure%1:10:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1117,"cached_input_tokens":0,"output_tokens":76,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002994,"input_uncached_usd":0.002234,"input_cached_usd":0.0,"output_usd":0.00076,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":14.657430249964818}
{"item_id":"semeval2015.d000.s029.t004","gold_sense_keys":["long%4:02:00::"],"candidates":[{"index":1,"sense_key":"long%4:02:00::","synset_id":"long.r.01"},{"index":2,"sense_key":"long%4:02:03::","synset_id":"long.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"long%4:02:00::","call_ids":["semeval2015.d000.s029.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"long%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":592,"cached_input_tokens":0,"output_tokens":44,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001624,"input_uncached_usd":0.001184,"input_cached_usd":0.0,"output_usd":0.00044,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2005702499300241}
{"item_id":"semeval2015.d000.s029.t005","gold_sense_keys":["patient%1:18:00::"],"candidates":[{"index":1,"sense_key":"patient%1:18:00::","synset_id":"patient.n.01"},{"index":2,"sense_key":"patient%1:10:00::","synset_id":"affected_role.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"patient%1:18:00::","call_ids":["semeval2015.d000.s029.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"patient%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":640,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0014,"input_uncached_usd":0.0012799999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0906207920052111}
{"item_id":"semeval2015.d000.s029.t006","gold_sense_keys":["survive%2:42:01::"],"candidates":[{"index":1,"sense_key":"survive%2:42:01::","synset_id":"survive.v.01"},{"index":2,"sense_key":"survive%2:42:02::","synset_id":"survive.v.02"},{"index":3,"sense_key":"survive%2:42:00::","synset_id":"exist.v.02"},{"index":4,"sense_key":"survive%2:42:03::","synset_id":"outlive.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"survive%2:42:01::","call_ids":["semeval2015.d000.s029.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"survive%2:42:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":800,"cached_input_tokens":0,"output_tokens":202,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00362,"input_uncached_usd":0.0015999999999999999,"input_cached_usd":0.0,"output_usd":0.00202,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.62262629205361}
{"item_id":"semeval2015.d000.s030.t000","gold_sense_keys":["benefit%1:07:00::"],"candidates":[{"index":1,"sense_key":"benefit%1:21:00::","synset_id":"benefit.n.01"},{"index":2,"sense_key":"benefit%1:07:00::","synset_id":"benefit.n.02"},{"index":3,"sense_key":"benefit%1:10:00::","synset_id":"benefit.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"benefit%1:07:00::","call_ids":["semeval2015.d000.s030.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"benefit%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":658,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001436,"input_uncached_usd":0.001316,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":9.713988250005059}
{"item_id":"semeval2015.d000.s030.t001","gold_sense_keys":["show%2:32:00::","show%2:39:00::"],"candidates":[{"index":1,"sense_key":"show%2:39:02::","synset_id":"show.v.01"},{"index":2,"sense_key":"show%2:31:00::","synset_id":"prove.v.02"},{"index":3,"sense_key":"show%2:32:00::","synset_id":"testify.v.02"},{"index":4,"sense_key":"show%2:39:00::","synset_id":"show.v.04"},{"index":5,"sense_key":"show%2:36:00::","synset_id":"picture.v.02"},{"index":6,"sense_key":"show%2:32:02::","synset_id":"express.v.01"},{"index":7,"sense_key":"show%2:32:01::","synset_id":"indicate.v.02"},{"index":8,"sense_key":"show%2:39:01::","synset_id":"show.v.08"},{"index":9,"sense_key":"show%2:32:04::","synset_id":"read.v.08"},{"index":10,"sense_key":"show%2:32:03::","synset_id":"show.v.10"},{"index":11,"sense_key":"show%2:38:00::","synset_id":"usher.v.01"},{"index":12,"sense_key":"show%2:33:00::","synset_id":"show.v.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"show%2:32:00::","call_ids":["semeval2015.d000.s030.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"show%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1346,"cached_input_tokens":0,"output_tokens":205,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004742,"input_uncached_usd":0.002692,"input_cached_usd":0.0,"output_usd":0.00205,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":10.99680912506301}
{"item_id":"semeval2015.d000.s030.t002","gold_sense_keys":["study%1:04:00::"],"candidates":[{"index":1,"sense_key":"study%1:04:00::","synset_id":"survey.n.01"},{"index":2,"sense_key":"study%1:09:03::","synset_id":"study.n.02"},{"index":3,"sense_key":"study%1:10:00::","synset_id":"report.n.01"},{"index":4,"sense_key":"study%1:09:04::","synset_id":"study.n.04"},{"index":5,"sense_key":"study%1:06:00::","synset_id":"study.n.05"},{"index":6,"sense_key":"study%1:09:02::","synset_id":"discipline.n.01"},{"index":7,"sense_key":"study%1:06:01::","synset_id":"sketch.n.01"},{"index":8,"sense_key":"study%1:09:01::","synset_id":"cogitation.n.02"},{"index":9,"sense_key":"study%1:18:00::","synset_id":"study.n.09"},{"index":10,"sense_key":"study%1:10:01::","synset_id":"study.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"study%1:04:00::","call_ids":["semeval2015.d000.s030.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"study%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1120,"cached_input_tokens":0,"output_tokens":161,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00385,"input_uncached_usd":0.00224,"input_cached_usd":0.0,"output_usd":0.00161,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0735078339930624}
{"item_id":"semeval2015.d000.s031.t000","gold_sense_keys":["increase%2:30:02::"],"candidates":[{"index":1,"sense_key":"increase%2:30:00::","synset_id":"increase.v.01"},{"index":2,"sense_key":"increase%2:30:02::","synset_id":"increase.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"increase%2:30:02::","call_ids":["semeval2015.d000.s031.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"increase%2:30:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":629,"cached_input_tokens":0,"output_tokens":50,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001758,"input_uncached_usd":0.001258,"input_cached_usd":0.0,"output_usd":0.0005,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":5.689194458071142}
{"item_id":"semeval2015.d000.s031.t001","gold_sense_keys":["survival%1:26:00::"],"candidates":[{"index":1,"sense_key":"survival%1:26:00::","synset_id":"survival.n.01"},{"index":2,"sense_key":"survival%1:22:00::","synset_id":"survival.n.02"},{"index":3,"sense_key":"survival%1:04:00::","synset_id":"survival.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"survival%1:26:00::","call_ids":["semeval2015.d000.s031.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"survival%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":667,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001454,"input_uncached_usd":0.001334,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4846682499628514}
{"item_id":"semeval2015.d000.s031.t002","gold_sense_keys":["time%1:28:00::"],"candidates":[{"index":1,"sense_key":"time%1:11:00::","synset_id":"time.n.01"},{"index":2,"sense_key":"time%1:28:05::","synset_id":"time.n.02"},{"index":3,"sense_key":"time%1:28:00::","synset_id":"time.n.03"},{"index":4,"sense_key":"time%1:28:06::","synset_id":"time.n.04"},{"index":5,"sense_key":"time%1:03:00::","synset_id":"time.n.05"},{"index":6,"sense_key":"time%1:11:01::","synset_id":"time.n.06"},{"index":7,"sense_key":"time%1:28:03::","synset_id":"clock_time.n.01"},{"index":8,"sense_key":"time%1:28:04::","synset_id":"fourth_dimension.n.01"},{"index":9,"sense_key":"time%1:07:01::","synset_id":"meter.n.04"},{"index":10,"sense_key":"time%1:28:01::","synset_id":"prison_term.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"time%1:28:00::","call_ids":["semeval2015.d000.s031.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"time%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1100,"cached_input_tokens":0,"output_tokens":46,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0026599999999999996,"input_uncached_usd":0.0021999999999999997,"input_cached_usd":0.0,"output_usd":0.00046,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0472435828996822}
{"item_id":"semeval2015.d000.s031.t003","gold_sense_keys":["patient%1:18:00::"],"candidates":[{"index":1,"sense_key":"patient%1:18:00::","synset_id":"patient.n.01"},{"index":2,"sense_key":"patient%1:10:00::","synset_id":"affected_role.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"patient%1:18:00::","call_ids":["semeval2015.d000.s031.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"patient%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":655,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00143,"input_uncached_usd":0.00131,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0682229170342907}
{"item_id":"semeval2015.d000.s032.t000","gold_sense_keys":["patient%1:18:00::"],"candidates":[{"index":1,"sense_key":"patient%1:18:00::","synset_id":"patient.n.01"},{"index":2,"sense_key":"patient%1:10:00::","synset_id":"affected_role.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"patient%1:18:00::","call_ids":["semeval2015.d000.s032.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"patient%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":659,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001438,"input_uncached_usd":0.001318,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.100402083946392}
{"item_id":"semeval2015.d000.s032.t002","gold_sense_keys":["survive%2:42:01::"],"candidates":[{"index":1,"sense_key":"survive%2:42:01::","synset_id":"survive.v.01"},{"index":2,"sense_key":"survive%2:42:02::","synset_id":"survive.v.02"},{"index":3,"sense_key":"survive%2:42:00::","synset_id":"exist.v.02"},{"index":4,"sense_key":"survive%2:42:03::","synset_id":"outlive.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"survive%2:42:01::","call_ids":["semeval2015.d000.s032.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"survive%2:42:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":818,"cached_input_tokens":0,"output_tokens":257,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004206,"input_uncached_usd":0.0016359999999999999,"input_cached_usd":0.0,"output_usd":0.0025700000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.707686875015497}
{"item_id":"semeval2015.d000.s032.t003","gold_sense_keys":["average%1:09:00::"],"candidates":[{"index":1,"sense_key":"average%1:09:00::","synset_id":"average.n.01"},{"index":2,"sense_key":"average%1:24:00::","synset_id":"average.n.02"},{"index":3,"sense_key":"average%1:09:01::","synset_id":"average.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"average%1:09:00::","call_ids":["semeval2015.d000.s032.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"average%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":685,"cached_input_tokens":0,"output_tokens":117,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0025399999999999997,"input_uncached_usd":0.00137,"input_cached_usd":0.0,"output_usd":0.00117,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7736899158917367}
{"item_id":"semeval2015.d000.s032.t004","gold_sense_keys":["month%1:28:00::"],"candidates":[{"index":1,"sense_key":"month%1:28:01::","synset_id":"calendar_month.n.01"},{"index":2,"sense_key":"month%1:28:00::","synset_id":"month.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"month%1:28:00::","call_ids":["semeval2015.d000.s032.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"month%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":638,"cached_input_tokens":0,"output_tokens":70,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0019760000000000003,"input_uncached_usd":0.001276,"input_cached_usd":0.0,"output_usd":0.0007000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9018168749753386}
{"item_id":"semeval2015.d000.s032.t005","gold_sense_keys":["compare%2:31:00::"],"candidates":[{"index":1,"sense_key":"compare%2:31:00::","synset_id":"compare.v.01"},{"index":2,"sense_key":"compare%2:42:00::","synset_id":"compare.v.02"},{"index":3,"sense_key":"compare%2:31:01::","synset_id":"compare.v.03"},{"index":4,"sense_key":"compare%2:32:00::","synset_id":"compare.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"compare%2:31:00::","call_ids":["semeval2015.d000.s032.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"compare%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":771,"cached_input_tokens":0,"output_tokens":112,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0026620000000000003,"input_uncached_usd":0.001542,"input_cached_usd":0.0,"output_usd":0.0011200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5707671249983832}
{"item_id":"semeval2015.d000.s032.t006","gold_sense_keys":["month%1:28:00::"],"candidates":[{"index":1,"sense_key":"month%1:28:01::","synset_id":"calendar_month.n.01"},{"index":2,"sense_key":"month%1:28:00::","synset_id":"month.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"month%1:28:00::","call_ids":["semeval2015.d000.s032.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"month%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":638,"cached_input_tokens":0,"output_tokens":71,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001986,"input_uncached_usd":0.001276,"input_cached_usd":0.0,"output_usd":0.00071,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1869849159847945}
{"item_id":"semeval2015.d000.s033.t000","gold_sense_keys":["treatment%1:04:00::"],"candidates":[{"index":1,"sense_key":"treatment%1:04:00::","synset_id":"treatment.n.01"},{"index":2,"sense_key":"treatment%1:04:01::","synset_id":"treatment.n.02"},{"index":3,"sense_key":"treatment%1:07:00::","synset_id":"treatment.n.03"},{"index":4,"sense_key":"treatment%1:10:00::","synset_id":"discussion.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"treatment%1:04:00::","call_ids":["semeval2015.d000.s033.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"treatment%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":775,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00167,"input_uncached_usd":0.00155,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4634928749874234}
{"item_id":"semeval2015.d000.s033.t001","gold_sense_keys":["patient%1:18:00::"],"candidates":[{"index":1,"sense_key":"patient%1:18:00::","synset_id":"patient.n.01"},{"index":2,"sense_key":"patient%1:10:00::","synset_id":"affected_role.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"patient%1:18:00::","call_ids":["semeval2015.d000.s033.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"patient%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":626,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001372,"input_uncached_usd":0.0012519999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.111557750031352}
{"item_id":"semeval2015.d000.s033.t002","gold_sense_keys":["cell%1:03:00::"],"candidates":[{"index":1,"sense_key":"cell%1:06:03::","synset_id":"cell.n.01"},{"index":2,"sense_key":"cell%1:03:00::","synset_id":"cell.n.02"},{"index":3,"sense_key":"cell%1:06:00::","synset_id":"cell.n.03"},{"index":4,"sense_key":"cell%1:14:00::","synset_id":"cell.n.04"},{"index":5,"sense_key":"cell%1:06:04::","synset_id":"cellular_telephone.n.01"},{"index":6,"sense_key":"cell%1:06:02::","synset_id":"cell.n.06"},{"index":7,"sense_key":"cell%1:06:01::","synset_id":"cell.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cell%1:03:00::","call_ids":["semeval2015.d000.s033.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cell%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":911,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001942,"input_uncached_usd":0.0018219999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9444144589360803}
{"item_id":"semeval2015.d000.s033.t005","gold_sense_keys":["effective%3:00:00::","effective%5:00:00:competent:00","effective%5:00:00:efficacious:00"],"candidates":[{"index":1,"sense_key":"effective%3:00:00::","synset_id":"effective.a.01"},{"index":2,"sense_key":"effective%5:00:00:competent:00","synset_id":"effective.s.01"},{"index":3,"sense_key":"effective%5:00:00:efficacious:00","synset_id":"effective.s.02"},{"index":4,"sense_key":"effective%5:00:00:operative:00","synset_id":"effective.s.03"},{"index":5,"sense_key":"effective%5:00:00:actual:00","synset_id":"effective.s.04"},{"index":6,"sense_key":"effective%5:00:00:operational:00","synset_id":"effective.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"effective%5:00:00:efficacious:00","call_ids":["semeval2015.d000.s033.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"effective%5:00:00:efficacious:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":968,"cached_input_tokens":0,"output_tokens":134,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0032760000000000003,"input_uncached_usd":0.001936,"input_cached_usd":0.0,"output_usd":0.00134,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.3490576250478625}
{"item_id":"semeval2015.d000.s034.t000","gold_sense_keys":["patient%1:18:00::"],"candidates":[{"index":1,"sense_key":"patient%1:18:00::","synset_id":"patient.n.01"},{"index":2,"sense_key":"patient%1:10:00::","synset_id":"affected_role.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"patient%1:18:00::","call_ids":["semeval2015.d000.s034.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"patient%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":592,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001304,"input_uncached_usd":0.001184,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9413497910136357}
{"item_id":"semeval2015.d000.s034.t003","gold_sense_keys":["past%1:28:00::"],"candidates":[{"index":1,"sense_key":"past%1:28:00::","synset_id":"past.n.01"},{"index":2,"sense_key":"past%1:28:01::","synset_id":"past.n.02"},{"index":3,"sense_key":"past%1:10:00::","synset_id":"past.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"past%1:28:00::","call_ids":["semeval2015.d000.s034.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"past%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":625,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0013700000000000001,"input_uncached_usd":0.00125,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.101816082955338}
{"item_id":"semeval2015.d000.s034.t004","gold_sense_keys":["average%5:00:01:normal:01","average%5:00:02:normal:01","average%5:00:03:normal:01"],"candidates":[{"index":1,"sense_key":"average%5:00:02:normal:01","synset_id":"average.s.01"},{"index":2,"sense_key":"average%5:00:00:common:01","synset_id":"average.s.02"},{"index":3,"sense_key":"average%5:00:00:ordinary:00","synset_id":"average.s.03"},{"index":4,"sense_key":"average%5:00:00:moderate:00","synset_id":"average.s.04"},{"index":5,"sense_key":"average%5:00:03:normal:01","synset_id":"modal.s.01"},{"index":6,"sense_key":"average%5:00:01:normal:01","synset_id":"median.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"average%5:00:02:normal:01","call_ids":["semeval2015.d000.s034.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"average%5:00:02:normal:01","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":936,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002592,"input_uncached_usd":0.001872,"input_cached_usd":0.0,"output_usd":0.00072,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.913330917013809}
{"item_id":"semeval2015.d000.s034.t005","gold_sense_keys":["survival%1:26:00::"],"candidates":[{"index":1,"sense_key":"survival%1:26:00::","synset_id":"survival.n.01"},{"index":2,"sense_key":"survival%1:22:00::","synset_id":"survival.n.02"},{"index":3,"sense_key":"survival%1:04:00::","synset_id":"survival.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"survival%1:26:00::","call_ids":["semeval2015.d000.s034.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"survival%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":604,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001328,"input_uncached_usd":0.0012079999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9821277080336586}
{"item_id":"semeval2015.d000.s034.t006","gold_sense_keys":["time%1:28:00::"],"candidates":[{"index":1,"sense_key":"time%1:11:00::","synset_id":"time.n.01"},{"index":2,"sense_key":"time%1:28:05::","synset_id":"time.n.02"},{"index":3,"sense_key":"time%1:28:00::","synset_id":"time.n.03"},{"index":4,"sense_key":"time%1:28:06::","synset_id":"time.n.04"},{"index":5,"sense_key":"time%1:03:00::","synset_id":"time.n.05"},{"index":6,"sense_key":"time%1:11:01::","synset_id":"time.n.06"},{"index":7,"sense_key":"time%1:28:03::","synset_id":"clock_time.n.01"},{"index":8,"sense_key":"time%1:28:04::","synset_id":"fourth_dimension.n.01"},{"index":9,"sense_key":"time%1:07:01::","synset_id":"meter.n.04"},{"index":10,"sense_key":"time%1:28:01::","synset_id":"prison_term.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"time%1:28:00::","call_ids":["semeval2015.d000.s034.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"time%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1037,"cached_input_tokens":0,"output_tokens":90,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002974,"input_uncached_usd":0.002074,"input_cached_usd":0.0,"output_usd":0.0009000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7409560419619083}
{"item_id":"semeval2015.d000.s034.t007","gold_sense_keys":["month%1:28:00::"],"candidates":[{"index":1,"sense_key":"month%1:28:01::","synset_id":"calendar_month.n.01"},{"index":2,"sense_key":"month%1:28:00::","synset_id":"month.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"month%1:28:00::","call_ids":["semeval2015.d000.s034.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"month%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":572,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001824,"input_uncached_usd":0.001144,"input_cached_usd":0.0,"output_usd":0.00068,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.076645166031085}
{"item_id":"semeval2015.d000.s034.t008","gold_sense_keys":["compare%2:31:00::"],"candidates":[{"index":1,"sense_key":"compare%2:31:00::","synset_id":"compare.v.01"},{"index":2,"sense_key":"compare%2:42:00::","synset_id":"compare.v.02"},{"index":3,"sense_key":"compare%2:31:01::","synset_id":"compare.v.03"},{"index":4,"sense_key":"compare%2:32:00::","synset_id":"compare.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"compare%2:31:00::","call_ids":["semeval2015.d000.s034.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"compare%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":705,"cached_input_tokens":0,"output_tokens":107,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00248,"input_uncached_usd":0.00141,"input_cached_usd":0.0,"output_usd":0.00107,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.3351735830074176}
{"item_id":"semeval2015.d000.s034.t009","gold_sense_keys":["month%1:28:00::"],"candidates":[{"index":1,"sense_key":"month%1:28:01::","synset_id":"calendar_month.n.01"},{"index":2,"sense_key":"month%1:28:00::","synset_id":"month.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"month%1:28:00::","call_ids":["semeval2015.d000.s034.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"month%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":572,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001824,"input_uncached_usd":0.001144,"input_cached_usd":0.0,"output_usd":0.00068,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5468220410402864}
{"item_id":"semeval2015.d000.s035.t000","gold_sense_keys":["patient%1:18:00::"],"candidates":[{"index":1,"sense_key":"patient%1:18:00::","synset_id":"patient.n.01"},{"index":2,"sense_key":"patient%1:10:00::","synset_id":"affected_role.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"patient%1:18:00::","call_ids":["semeval2015.d000.s035.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"patient%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":613,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001346,"input_uncached_usd":0.0012259999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0980303749674931}
{"item_id":"semeval2015.d000.s035.t004","gold_sense_keys":["past%1:28:00::"],"candidates":[{"index":1,"sense_key":"past%1:28:00::","synset_id":"past.n.01"},{"index":2,"sense_key":"past%1:28:01::","synset_id":"past.n.02"},{"index":3,"sense_key":"past%1:10:00::","synset_id":"past.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"past%1:28:00::","call_ids":["semeval2015.d000.s035.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"past%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":646,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001412,"input_uncached_usd":0.001292,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3669289579847828}
{"item_id":"semeval2015.d000.s035.t005","gold_sense_keys":["average%5:00:01:normal:01","average%5:00:02:normal:01","average%5:00:03:normal:01"],"candidates":[{"index":1,"sense_key":"average%5:00:02:normal:01","synset_id":"average.s.01"},{"index":2,"sense_key":"average%5:00:00:common:01","synset_id":"average.s.02"},{"index":3,"sense_key":"average%5:00:00:ordinary:00","synset_id":"average.s.03"},{"index":4,"sense_key":"average%5:00:00:moderate:00","synset_id":"average.s.04"},{"index":5,"sense_key":"average%5:00:03:normal:01","synset_id":"modal.s.01"},{"index":6,"sense_key":"average%5:00:01:normal:01","synset_id":"median.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"average%5:00:02:normal:01","call_ids":["semeval2015.d000.s035.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"average%5:00:02:normal:01","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":957,"cached_input_tokens":0,"output_tokens":69,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002604,"input_uncached_usd":0.001914,"input_cached_usd":0.0,"output_usd":0.0006900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4000575840473175}
{"item_id":"semeval2015.d000.s035.t006","gold_sense_keys":["survival%1:26:00::"],"candidates":[{"index":1,"sense_key":"survival%1:26:00::","synset_id":"survival.n.01"},{"index":2,"sense_key":"survival%1:22:00::","synset_id":"survival.n.02"},{"index":3,"sense_key":"survival%1:04:00::","synset_id":"survival.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"survival%1:26:00::","call_ids":["semeval2015.d000.s035.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"survival%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":625,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0013700000000000001,"input_uncached_usd":0.00125,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9905942080076784}
{"item_id":"semeval2015.d000.s035.t007","gold_sense_keys":["time%1:28:00::"],"candidates":[{"index":1,"sense_key":"time%1:11:00::","synset_id":"time.n.01"},{"index":2,"sense_key":"time%1:28:05::","synset_id":"time.n.02"},{"index":3,"sense_key":"time%1:28:00::","synset_id":"time.n.03"},{"index":4,"sense_key":"time%1:28:06::","synset_id":"time.n.04"},{"index":5,"sense_key":"time%1:03:00::","synset_id":"time.n.05"},{"index":6,"sense_key":"time%1:11:01::","synset_id":"time.n.06"},{"index":7,"sense_key":"time%1:28:03::","synset_id":"clock_time.n.01"},{"index":8,"sense_key":"time%1:28:04::","synset_id":"fourth_dimension.n.01"},{"index":9,"sense_key":"time%1:07:01::","synset_id":"meter.n.04"},{"index":10,"sense_key":"time%1:28:01::","synset_id":"prison_term.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"time%1:28:00::","call_ids":["semeval2015.d000.s035.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"time%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1058,"cached_input_tokens":0,"output_tokens":76,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002876,"input_uncached_usd":0.002116,"input_cached_usd":0.0,"output_usd":0.00076,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":9.709493541973643}
{"item_id":"semeval2015.d000.s035.t008","gold_sense_keys":["month%1:28:00::"],"candidates":[{"index":1,"sense_key":"month%1:28:01::","synset_id":"calendar_month.n.01"},{"index":2,"sense_key":"month%1:28:00::","synset_id":"month.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"month%1:28:00::","call_ids":["semeval2015.d000.s035.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"month%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":593,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0017560000000000002,"input_uncached_usd":0.001186,"input_cached_usd":0.0,"output_usd":0.0005700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":14.449436916969717}
{"item_id":"semeval2015.d000.s035.t009","gold_sense_keys":["group%1:03:00::"],"candidates":[{"index":1,"sense_key":"group%1:03:00::","synset_id":"group.n.01"},{"index":2,"sense_key":"group%1:27:00::","synset_id":"group.n.02"},{"index":3,"sense_key":"group%1:09:00::","synset_id":"group.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"group%1:03:00::","call_ids":["semeval2015.d000.s035.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"group%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":650,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00142,"input_uncached_usd":0.0013,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.7000257499748841}
{"item_id":"semeval2015.d000.s036.t000","gold_sense_keys":["however%4:02:04::"],"candidates":[{"index":1,"sense_key":"however%4:02:00::","synset_id":"however.r.01"},{"index":2,"sense_key":"however%4:02:04::","synset_id":"however.r.02"},{"index":3,"sense_key":"however%4:02:02::","synset_id":"however.r.03"},{"index":4,"sense_key":"however%4:02:01::","synset_id":"however.r.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"however%4:02:04::","call_ids":["semeval2015.d000.s036.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"however%4:02:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":798,"cached_input_tokens":0,"output_tokens":215,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003746,"input_uncached_usd":0.001596,"input_cached_usd":0.0,"output_usd":0.00215,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.4197500000009313}
{"item_id":"semeval2015.d000.s036.t001","gold_sense_keys":["study%1:04:00::"],"candidates":[{"index":1,"sense_key":"study%1:04:00::","synset_id":"survey.n.01"},{"index":2,"sense_key":"study%1:09:03::","synset_id":"study.n.02"},{"index":3,"sense_key":"study%1:10:00::","synset_id":"report.n.01"},{"index":4,"sense_key":"study%1:09:04::","synset_id":"study.n.04"},{"index":5,"sense_key":"study%1:06:00::","synset_id":"study.n.05"},{"index":6,"sense_key":"study%1:09:02::","synset_id":"discipline.n.01"},{"index":7,"sense_key":"study%1:06:01::","synset_id":"sketch.n.01"},{"index":8,"sense_key":"study%1:09:01::","synset_id":"cogitation.n.02"},{"index":9,"sense_key":"study%1:18:00::","synset_id":"study.n.09"},{"index":10,"sense_key":"study%1:10:01::","synset_id":"study.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"study%1:10:00::","call_ids":["semeval2015.d000.s036.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"study%1:10:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1096,"cached_input_tokens":0,"output_tokens":135,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003542,"input_uncached_usd":0.002192,"input_cached_usd":0.0,"output_usd":0.00135,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":14.858713832916692}
{"item_id":"semeval2015.d000.s036.t002","gold_sense_keys":["patient%1:18:00::"],"candidates":[{"index":1,"sense_key":"patient%1:18:00::","synset_id":"patient.n.01"},{"index":2,"sense_key":"patient%1:10:00::","synset_id":"affected_role.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"patient%1:18:00::","call_ids":["semeval2015.d000.s036.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"patient%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":633,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001386,"input_uncached_usd":0.001266,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7687285840511322}
{"item_id":"semeval2015.d000.s036.t003","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["semeval2015.d000.s036.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":826,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001772,"input_uncached_usd":0.0016519999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.041766916983761}
{"item_id":"semeval2015.d000.s036.t005","gold_sense_keys":["affect%2:29:00::"],"candidates":[{"index":1,"sense_key":"affect%2:30:00::","synset_id":"affect.v.01"},{"index":2,"sense_key":"affect%2:29:00::","synset_id":"affect.v.02"},{"index":3,"sense_key":"affect%2:42:00::","synset_id":"involve.v.01"},{"index":4,"sense_key":"affect%2:32:00::","synset_id":"feign.v.01"},{"index":5,"sense_key":"affect%2:37:00::","synset_id":"affect.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"affect%2:29:00::","call_ids":["semeval2015.d000.s036.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"affect%2:29:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":832,"cached_input_tokens":0,"output_tokens":118,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002844,"input_uncached_usd":0.001664,"input_cached_usd":0.0,"output_usd":0.00118,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":10.166214999975637}
{"item_id":"semeval2015.d000.s036.t008","gold_sense_keys":["long%3:00:02::"],"candidates":[{"index":1,"sense_key":"long%3:00:02::","synset_id":"long.a.01"},{"index":2,"sense_key":"long%3:00:01::","synset_id":"long.a.02"},{"index":3,"sense_key":"long%5:00:00:tall:00","synset_id":"long.s.01"},{"index":4,"sense_key":"long%3:00:00::","synset_id":"retentive.a.01"},{"index":5,"sense_key":"long%3:00:05::","synset_id":"long.a.05"},{"index":6,"sense_key":"long%3:00:04::","synset_id":"long.a.06"},{"index":7,"sense_key":"long%5:00:00:unsound:00","synset_id":"long.s.02"},{"index":8,"sense_key":"long%5:00:00:provident:00","synset_id":"farseeing.s.02"},{"index":9,"sense_key":"long%5:00:00:abundant:00","synset_id":"long.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"long%3:00:02::","call_ids":["semeval2015.d000.s036.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"long%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1089,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0022979999999999997,"input_uncached_usd":0.002178,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1781809590756893}
{"item_id":"semeval2015.d000.s036.t009","gold_sense_keys":["survival%1:26:00::"],"candidates":[{"index":1,"sense_key":"survival%1:26:00::","synset_id":"survival.n.01"},{"index":2,"sense_key":"survival%1:22:00::","synset_id":"survival.n.02"},{"index":3,"sense_key":"survival%1:04:00::","synset_id":"survival.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"survival%1:26:00::","call_ids":["semeval2015.d000.s036.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"survival%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":645,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00141,"input_uncached_usd":0.00129,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.083078540978022}
{"item_id":"semeval2015.d000.s036.t010","gold_sense_keys":["time%1:28:00::"],"candidates":[{"index":1,"sense_key":"time%1:11:00::","synset_id":"time.n.01"},{"index":2,"sense_key":"time%1:28:05::","synset_id":"time.n.02"},{"index":3,"sense_key":"time%1:28:00::","synset_id":"time.n.03"},{"index":4,"sense_key":"time%1:28:06::","synset_id":"time.n.04"},{"index":5,"sense_key":"time%1:03:00::","synset_id":"time.n.05"},{"index":6,"sense_key":"time%1:11:01::","synset_id":"time.n.06"},{"index":7,"sense_key":"time%1:28:03::","synset_id":"clock_time.n.01"},{"index":8,"sense_key":"time%1:28:04::","synset_id":"fourth_dimension.n.01"},{"index":9,"sense_key":"time%1:07:01::","synset_id":"meter.n.04"},{"index":10,"sense_key":"time%1:28:01::","synset_id":"prison_term.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"time%1:28:00::","call_ids":["semeval2015.d000.s036.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"time%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1078,"cached_input_tokens":0,"output_tokens":126,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0034159999999999998,"input_uncached_usd":0.002156,"input_cached_usd":0.0,"output_usd":0.00126,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.008414415991865}
{"item_id":"semeval2015.d000.s037.t000","gold_sense_keys":["contrast%1:24:00::"],"candidates":[{"index":1,"sense_key":"contrast%1:24:00::","synset_id":"contrast.n.01"},{"index":2,"sense_key":"contrast%1:04:00::","synset_id":"contrast.n.02"},{"index":3,"sense_key":"contrast%1:09:01::","synset_id":"line.n.29"},{"index":4,"sense_key":"contrast%1:09:00::","synset_id":"contrast.n.04"},{"index":5,"sense_key":"contrast%1:07:00::","synset_id":"contrast.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"contrast%1:24:00::","call_ids":["semeval2015.d000.s037.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"contrast%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":767,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001654,"input_uncached_usd":0.001534,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1998152500018477}
{"item_id":"semeval2015.d000.s037.t001","gold_sense_keys":["patient%1:18:00::"],"candidates":[{"index":1,"sense_key":"patient%1:18:00::","synset_id":"patient.n.01"},{"index":2,"sense_key":"patient%1:10:00::","synset_id":"affected_role.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"patient%1:18:00::","call_ids":["semeval2015.d000.s037.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"patient%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":606,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001332,"input_uncached_usd":0.001212,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0562478339998052}
{"item_id":"semeval2015.d000.s037.t002","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["semeval2015.d000.s037.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":799,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0017180000000000001,"input_uncached_usd":0.001598,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1415449579944834}
{"item_id":"semeval2015.d000.s037.t003","gold_sense_keys":["affect%2:29:00::"],"candidates":[{"index":1,"sense_key":"affect%2:30:00::","synset_id":"affect.v.01"},{"index":2,"sense_key":"affect%2:29:00::","synset_id":"affect.v.02"},{"index":3,"sense_key":"affect%2:42:00::","synset_id":"involve.v.01"},{"index":4,"sense_key":"affect%2:32:00::","synset_id":"feign.v.01"},{"index":5,"sense_key":"affect%2:37:00::","synset_id":"affect.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"affect%2:29:00::","call_ids":["semeval2015.d000.s037.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"affect%2:29:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":805,"cached_input_tokens":0,"output_tokens":84,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00245,"input_uncached_usd":0.0016099999999999999,"input_cached_usd":0.0,"output_usd":0.00084,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":14.752747416961938}
{"item_id":"semeval2015.d000.s037.t006","gold_sense_keys":["short%3:00:02::"],"candidates":[{"index":1,"sense_key":"short%3:00:02::","synset_id":"short.a.01"},{"index":2,"sense_key":"short%3:00:01::","synset_id":"short.a.02"},{"index":3,"sense_key":"short%3:00:03::","synset_id":"short.a.03"},{"index":4,"sense_key":"short%5:00:00:insufficient:00","synset_id":"inadequate.s.01"},{"index":5,"sense_key":"short%3:00:00::","synset_id":"unretentive.a.01"},{"index":6,"sense_key":"short%3:00:05::","synset_id":"short.a.06"},{"index":7,"sense_key":"short%3:00:04::","synset_id":"short.a.07"},{"index":8,"sense_key":"short%5:00:02:insufficient:00","synset_id":"light.s.10"},{"index":9,"sense_key":"short%5:00:00:improvident:00","synset_id":"short.s.03"},{"index":10,"sense_key":"short%5:00:00:breakable:00","synset_id":"short.s.04"},{"index":11,"sense_key":"short%5:00:00:discourteous:00","synset_id":"brusque.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"short%3:00:02::","call_ids":["semeval2015.d000.s037.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"short%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1204,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002528,"input_uncached_usd":0.002408,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":14.53444220800884}
{"item_id":"semeval2015.d000.s037.t007","gold_sense_keys":["survival%1:26:00::"],"candidates":[{"index":1,"sense_key":"survival%1:26:00::","synset_id":"survival.n.01"},{"index":2,"sense_key":"survival%1:22:00::","synset_id":"survival.n.02"},{"index":3,"sense_key":"survival%1:04:00::","synset_id":"survival.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"survival%1:26:00::","call_ids":["semeval2015.d000.s037.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"survival%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":618,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001356,"input_uncached_usd":0.001236,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":11.016601916984655}
{"item_id":"semeval2015.d000.s037.t008","gold_sense_keys":["time%1:28:00::"],"candidates":[{"index":1,"sense_key":"time%1:11:00::","synset_id":"time.n.01"},{"index":2,"sense_key":"time%1:28:05::","synset_id":"time.n.02"},{"index":3,"sense_key":"time%1:28:00::","synset_id":"time.n.03"},{"index":4,"sense_key":"time%1:28:06::","synset_id":"time.n.04"},{"index":5,"sense_key":"time%1:03:00::","synset_id":"time.n.05"},{"index":6,"sense_key":"time%1:11:01::","synset_id":"time.n.06"},{"index":7,"sense_key":"time%1:28:03::","synset_id":"clock_time.n.01"},{"index":8,"sense_key":"time%1:28:04::","synset_id":"fourth_dimension.n.01"},{"index":9,"sense_key":"time%1:07:01::","synset_id":"meter.n.04"},{"index":10,"sense_key":"time%1:28:01::","synset_id":"prison_term.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"time%1:28:00::","call_ids":["semeval2015.d000.s037.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"time%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1051,"cached_input_tokens":0,"output_tokens":120,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003302,"input_uncached_usd":0.0021019999999999997,"input_cached_usd":0.0,"output_usd":0.0012000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6274966660421342}
{"item_id":"semeval2015.d001.s000.t000","gold_sense_keys":["mathematical%3:01:00::"],"candidates":[{"index":1,"sense_key":"mathematical%3:01:00::","synset_id":"mathematical.a.01"},{"index":2,"sense_key":"mathematical%3:00:00::","synset_id":"numerical.a.03"},{"index":3,"sense_key":"mathematical%5:00:00:unquestionable:00","synset_id":"mathematical.s.01"},{"index":4,"sense_key":"mathematical%5:00:00:possible:00","synset_id":"mathematical.s.02"},{"index":5,"sense_key":"mathematical%5:00:00:exact:00","synset_id":"mathematical.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"mathematical%3:01:00::","call_ids":["semeval2015.d001.s000.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"mathematical%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":602,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001324,"input_uncached_usd":0.001204,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.550625207950361}
{"item_id":"semeval2015.d001.s000.t001","gold_sense_keys":["base%2:31:00::"],"candidates":[{"index":1,"sense_key":"base%2:31:00::","synset_id":"establish.v.08"},{"index":2,"sense_key":"base%2:42:04::","synset_id":"base.v.02"},{"index":3,"sense_key":"base%2:34:00::","synset_id":"free-base.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"base%2:31:00::","call_ids":["semeval2015.d001.s000.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"base%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":446,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001012,"input_uncached_usd":0.000892,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6429285421036184}
{"item_id":"semeval2015.d001.s001.t001","gold_sense_keys":["capable%3:00:00::"],"candidates":[{"index":1,"sense_key":"capable%3:00:00::","synset_id":"capable.a.01"},{"index":2,"sense_key":"capable%5:00:00:susceptible:00","synset_id":"capable.s.01"},{"index":3,"sense_key":"capable%3:00:02::","synset_id":"capable.a.03"},{"index":4,"sense_key":"capable%5:00:00:adequate:00","synset_id":"adequate_to.s.01"},{"index":5,"sense_key":"capable%5:00:00:competent:00","synset_id":"able.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"capable%3:00:00::","call_ids":["semeval2015.d001.s001.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"capable%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":633,"cached_input_tokens":0,"output_tokens":115,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002416,"input_uncached_usd":0.001266,"input_cached_usd":0.0,"output_usd":0.0011500000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":10.102221624925733}
{"item_id":"semeval2015.d001.s001.t002","gold_sense_keys":["make%2:36:12::"],"candidates":[{"index":1,"sense_key":"make%2:41:00::","synset_id":"make.v.01"},{"index":2,"sense_key":"make%2:30:00::","synset_id":"make.v.02"},{"index":3,"sense_key":"make%2:36:00::","synset_id":"make.v.03"},{"index":4,"sense_key":"make%2:32:00::","synset_id":"induce.v.02"},{"index":5,"sense_key":"make%2:36:08::","synset_id":"cause.v.01"},{"index":6,"sense_key":"make%2:36:01::","synset_id":"produce.v.02"},{"index":7,"sense_key":"make%2:31:00::","synset_id":"draw.v.04"},{"index":8,"sense_key":"make%2:36:15::","synset_id":"make.v.08"},{"index":9,"sense_key":"make%2:36:09::","synset_id":"create.v.05"},{"index":10,"sense_key":"make%2:40:01::","synset_id":"gain.v.08"},{"index":11,"sense_key":"make%2:36:05::","synset_id":"do.v.08"},{"index":12,"sense_key":"make%2:42:05::","synset_id":"form.v.02"},{"index":13,"sense_key":"make%2:38:02::","synset_id":"reach.v.07"},{"index":14,"sense_key":"make%2:42:00::","synset_id":"make.v.14"},{"index":15,"sense_key":"make%2:36:11::","synset_id":"make.v.15"},{"index":16,"sense_key":"make%2:36:12::","synset_id":"make.v.16"},{"index":17,"sense_key":"make%2:36:13::","synset_id":"construct.v.01"},{"index":18,"sense_key":"make%2:30:15::","synset_id":"make.v.18"},{"index":19,"sense_key":"make%2:29:08::","synset_id":"make.v.19"},{"index":20,"sense_key":"make%2:41:03::","synset_id":"name.v.03"},{"index":21,"sense_key":"make%2:40:02::","synset_id":"have.v.17"},{"index":22,"sense_key":"make%2:38:00::","synset_id":"reach.v.01"},{"index":23,"sense_key":"make%2:31:13::","synset_id":"lay_down.v.01"},{"index":24,"sense_key":"make%2:41:13::","synset_id":"make.v.24"},{"index":25,"sense_key":"make%2:41:01::","synset_id":"make.v.25"},{"index":26,"sense_key":"make%2:36:04::","synset_id":"hold.v.03"},{"index":27,"sense_key":"make%2:30:02::","synset_id":"make.v.27"},{"index":28,"sense_key":"make%2:38:05::","synset_id":"take.v.27"},{"index":29,"sense_key":"make%2:29:00::","synset_id":"stool.v.04"},{"index":30,"sense_key":"make%2:42:09::","synset_id":"make.v.30"},{"index":31,"sense_key":"make%2:42:13::","synset_id":"make.v.31"},{"index":32,"sense_key":"make%2:42:12::","synset_id":"make.v.32"},{"index":33,"sense_key":"make%2:42:11::","synset_id":"make.v.33"},{"index":34,"sense_key":"make%2:42:15::","synset_id":"make.v.34"},{"index":35,"sense_key":"make%2:39:12::","synset_id":"make.v.35"},{"index":36,"sense_key":"make%2:38:10::","synset_id":"make.v.36"},{"index":37,"sense_key":"make%2:38:15::","synset_id":"make.v.37"},{"index":38,"sense_key":"make%2:36:10::","synset_id":"make.v.38"},{"index":39,"sense_key":"make%2:36:07::","synset_id":"cook.v.02"},{"index":40,"sense_key":"make%2:35:13::","synset_id":"seduce.v.01"},{"index":41,"sense_key":"make%2:32:14::","synset_id":"make.v.41"},{"index":42,"sense_key":"make%2:32:09::","synset_id":"make.v.42"},{"index":43,"sense_key":"make%2:31:10::","synset_id":"make.v.43"},{"index":44,"sense_key":"make%2:31:12::","synset_id":"make.v.44"},{"index":45,"sense_key":"make%2:30:10::","synset_id":"make.v.45"},{"index":46,"sense_key":"make%2:30:14::","synset_id":"make.v.46"},{"index":47,"sense_key":"make%2:30:12::","synset_id":"make.v.47"},{"index":48,"sense_key":"make%2:29:06::","synset_id":"make.v.48"},{"index":49,"sense_key":"make%2:29:01::","synset_id":"make.v.49"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":16,"chosen_sense_key":"make%2:36:12::","call_ids":["semeval2015.d001.s001.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":16,"predicted_sense_key":"make%2:36:12::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2984,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.006558,"input_uncached_usd":0.005967999999999999,"input_cached_usd":0.0,"output_usd":0.00059,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":14.699218541034497}
{"item_id":"semeval2015.d001.s001.t003","gold_sense_keys":["simple%3:00:02::","simple%5:00:00:easy:01"],"candidates":[{"index":1,"sense_key":"simple%3:00:02::","synset_id":"simple.a.01"},{"index":2,"sense_key":"simple%5:00:00:easy:01","synset_id":"elementary.s.01"},{"index":3,"sense_key":"simple%5:00:02:plain:01","synset_id":"bare.s.04"},{"index":4,"sense_key":"simple%5:00:00:naive:00","synset_id":"childlike.s.02"},{"index":5,"sense_key":"simple%5:00:00:retarded:00","synset_id":"dim-witted.s.01"},{"index":6,"sense_key":"simple%3:00:01::","synset_id":"simple.a.06"},{"index":7,"sense_key":"simple%5:00:00:plain:01","synset_id":"simple.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"simple%3:00:02::","call_ids":["semeval2015.d001.s001.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"simple%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":793,"cached_input_tokens":0,"output_tokens":174,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003326,"input_uncached_usd":0.001586,"input_cached_usd":0.0,"output_usd":0.0017400000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.8026139999274164}
{"item_id":"semeval2015.d001.s001.t004","gold_sense_keys":["operation%1:04:01::"],"candidates":[{"index":1,"sense_key":"operation%1:26:00::","synset_id":"operation.n.01"},{"index":2,"sense_key":"operation%1:04:06::","synset_id":"operation.n.02"},{"index":3,"sense_key":"operation%1:04:03::","synset_id":"operation.n.03"},{"index":4,"sense_key":"operation%1:22:00::","synset_id":"operation.n.04"},{"index":5,"sense_key":"operation%1:04:02::","synset_id":"operation.n.05"},{"index":6,"sense_key":"operation%1:04:00::","synset_id":"operation.n.06"},{"index":7,"sense_key":"operation%1:04:05::","synset_id":"operation.n.07"},{"index":8,"sense_key":"operation%1:22:01::","synset_id":"operation.n.08"},{"index":9,"sense_key":"operation%1:09:00::","synset_id":"process.n.02"},{"index":10,"sense_key":"operation%1:04:01::","synset_id":"mathematical_process.n.01"},{"index":11,"sense_key":"operation%1:04:07::","synset_id":"operation.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"operation%1:04:01::","call_ids":["semeval2015.d001.s001.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"operation%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1161,"cached_input_tokens":0,"output_tokens":126,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003582,"input_uncached_usd":0.002322,"input_cached_usd":0.0,"output_usd":0.00126,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":15.179635250009596}
{"item_id":"semeval2015.d001.s001.t006","gold_sense_keys":["3d%1:09:00::"],"candidates":[{"index":1,"sense_key":"3-d%1:10:00::","synset_id":"three-d.n.01"},{"index":2,"sense_key":"3-d%1:09:00::","synset_id":"three-d.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"3-d%1:09:00::","call_ids":["semeval2015.d001.s001.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"3-d%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":430,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00141,"input_uncached_usd":0.00086,"input_cached_usd":0.0,"output_usd":0.00055,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":11.298227207968011}
{"item_id":"semeval2015.d001.s002.t000","gold_sense_keys":["actually%4:02:00::","actually%4:02:02::"],"candidates":[{"index":1,"sense_key":"actually%4:02:01::","synset_id":"actually.r.01"},{"index":2,"sense_key":"actually%4:02:00::","synset_id":"actually.r.02"},{"index":3,"sense_key":"actually%4:02:03::","synset_id":"actually.r.03"},{"index":4,"sense_key":"actually%4:02:02::","synset_id":"actually.r.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"actually%4:02:00::","call_ids":["semeval2015.d001.s002.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"actually%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":599,"cached_input_tokens":0,"output_tokens":165,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002848,"input_uncached_usd":0.0011979999999999998,"input_cached_usd":0.0,"output_usd":0.0016500000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":14.71719699993264}
{"item_id":"semeval2015.d001.s002.t002","gold_sense_keys":["necessary%3:00:00::"],"candidates":[{"index":1,"sense_key":"necessary%3:00:00::","synset_id":"necessary.a.01"},{"index":2,"sense_key":"necessary%5:00:00:inevitable:00","synset_id":"necessary.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"necessary%3:00:00::","call_ids":["semeval2015.d001.s002.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"necessary%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":426,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.000972,"input_uncached_usd":0.000852,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4989530829479918}
{"item_id":"semeval2015.d001.s002.t003","gold_sense_keys":["know%2:31:15::"],"candidates":[{"index":1,"sense_key":"know%2:31:01::","synset_id":"know.v.01"},{"index":2,"sense_key":"know%2:31:03::","synset_id":"know.v.02"},{"index":3,"sense_key":"know%2:31:02::","synset_id":"know.v.03"},{"index":4,"sense_key":"know%2:31:00::","synset_id":"know.v.04"},{"index":5,"sense_key":"know%2:31:04::","synset_id":"know.v.05"},{"index":6,"sense_key":"know%2:31:06::","synset_id":"acknowledge.v.06"},{"index":7,"sense_key":"know%2:31:15::","synset_id":"know.v.07"},{"index":8,"sense_key":"know%2:35:00::","synset_id":"sleep_together.v.01"},{"index":9,"sense_key":"know%2:31:11::","synset_id":"know.v.09"},{"index":10,"sense_key":"know%2:31:12::","synset_id":"know.v.10"},{"index":11,"sense_key":"know%2:31:14::","synset_id":"know.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"know%2:31:15::","call_ids":["semeval2015.d001.s002.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"know%2:31:15::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1056,"cached_input_tokens":0,"output_tokens":49,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0026019999999999997,"input_uncached_usd":0.0021119999999999997,"input_cached_usd":0.0,"output_usd":0.0004900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8290812079794705}
{"item_id":"semeval2015.d001.s002.t004","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":2,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":3,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":4,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":5,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":6,"sense_key":"use%2:41:03::","synset_id":"use.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%2:34:01::","call_ids":["semeval2015.d001.s002.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":721,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001562,"input_uncached_usd":0.001442,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5244506659219041}
{"item_id":"semeval2015.d001.s004.t002","gold_sense_keys":["now%4:02:00::","now%4:02:04::"],"candidates":[{"index":1,"sense_key":"now%4:02:05::","synset_id":"now.r.01"},{"index":2,"sense_key":"now%4:02:00::","synset_id":"nowadays.r.01"},{"index":3,"sense_key":"now%4:02:02::","synset_id":"now.r.03"},{"index":4,"sense_key":"now%4:02:04::","synset_id":"now.r.04"},{"index":5,"sense_key":"now%4:02:01::","synset_id":"immediately.r.01"},{"index":6,"sense_key":"now%4:02:07::","synset_id":"now.r.06"},{"index":7,"sense_key":"now%4:02:06::","synset_id":"now.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"now%4:02:00::","call_ids":["semeval2015.d001.s004.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"now%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":822,"cached_input_tokens":0,"output_tokens":119,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0028339999999999997,"input_uncached_usd":0.0016439999999999998,"input_cached_usd":0.0,"output_usd":0.00119,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":7.244198750006035}
{"item_id":"semeval2015.d001.s004.t003","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":2,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":3,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":4,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":5,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":6,"sense_key":"use%2:41:03::","synset_id":"use.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%2:34:01::","call_ids":["semeval2015.d001.s004.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":769,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001658,"input_uncached_usd":0.0015379999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":13.87190762499813}
{"item_id":"semeval2015.d001.s004.t004","gold_sense_keys":["little%3:00:03::"],"candidates":[{"index":1,"sense_key":"little%3:00:01::","synset_id":"small.a.01"},{"index":2,"sense_key":"little%3:00:03::","synset_id":"little.a.02"},{"index":3,"sense_key":"little%5:00:00:young:00","synset_id":"little.s.01"},{"index":4,"sense_key":"little%5:00:00:unimportant:00","synset_id":"fiddling.s.01"},{"index":5,"sense_key":"little%5:00:00:soft:04","synset_id":"little.s.03"},{"index":6,"sense_key":"little%3:00:00::","synset_id":"short.a.03"},{"index":7,"sense_key":"little%5:00:00:lowercase:00","synset_id":"little.s.04"},{"index":8,"sense_key":"little%5:00:00:emotional:00","synset_id":"little.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"little%3:00:03::","call_ids":["semeval2015.d001.s004.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"little%3:00:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":899,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001918,"input_uncached_usd":0.001798,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5268926658900455}
{"item_id":"semeval2015.d001.s004.t005","gold_sense_keys":["mathematical%3:01:00::"],"candidates":[{"index":1,"sense_key":"mathematical%3:01:00::","synset_id":"mathematical.a.01"},{"index":2,"sense_key":"mathematical%3:00:00::","synset_id":"numerical.a.03"},{"index":3,"sense_key":"mathematical%5:00:00:unquestionable:00","synset_id":"mathematical.s.01"},{"index":4,"sense_key":"mathematical%5:00:00:possible:00","synset_id":"mathematical.s.02"},{"index":5,"sense_key":"mathematical%5:00:00:exact:00","synset_id":"mathematical.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"mathematical%3:01:00::","call_ids":["semeval2015.d001.s004.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"mathematical%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":688,"cached_input_tokens":0,"output_tokens":98,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002356,"input_uncached_usd":0.0013759999999999998,"input_cached_usd":0.0,"output_usd":0.0009800000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1603827090002596}
{"item_id":"semeval2015.d001.s005.t000","gold_sense_keys":["here%4:02:00::"],"candidates":[{"index":1,"sense_key":"here%4:02:00::","synset_id":"here.r.01"},{"index":2,"sense_key":"here%4:02:02::","synset_id":"here.r.02"},{"index":3,"sense_key":"here%4:02:01::","synset_id":"here.r.03"},{"index":4,"sense_key":"here%4:02:04::","synset_id":"here.r.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"here%4:02:00::","call_ids":["semeval2015.d001.s005.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"here%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":599,"cached_input_tokens":0,"output_tokens":147,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0026680000000000002,"input_uncached_usd":0.0011979999999999998,"input_cached_usd":0.0,"output_usd":0.0014700000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.132272250019014}
{"item_id":"semeval2015.d001.s005.t001","gold_sense_keys":["main%5:00:00:important:00"],"candidates":[{"index":1,"sense_key":"main%5:00:00:important:00","synset_id":"chief.s.01"},{"index":2,"sense_key":"main%3:00:02::","synset_id":"independent.a.03"},{"index":3,"sense_key":"main%5:00:00:intense:00","synset_id":"main.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"main%5:00:00:important:00","call_ids":["semeval2015.d001.s005.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"main%5:00:00:important:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":583,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001286,"input_uncached_usd":0.001166,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9962475419742987}
{"item_id":"semeval2015.d001.s005.t002","gold_sense_keys":["window%1:06:03::"],"candidates":[{"index":1,"sense_key":"window%1:06:00::","synset_id":"window.n.01"},{"index":2,"sense_key":"window%1:06:01::","synset_id":"window.n.02"},{"index":3,"sense_key":"window%1:06:02::","synset_id":"window.n.03"},{"index":4,"sense_key":"window%1:17:00::","synset_id":"window.n.04"},{"index":5,"sense_key":"window%1:28:00::","synset_id":"window.n.05"},{"index":6,"sense_key":"window%1:06:04::","synset_id":"windowpane.n.01"},{"index":7,"sense_key":"window%1:06:05::","synset_id":"window.n.07"},{"index":8,"sense_key":"window%1:06:03::","synset_id":"window.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"window%1:06:03::","call_ids":["semeval2015.d001.s005.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"window%1:06:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":839,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0017980000000000001,"input_uncached_usd":0.001678,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.262144541949965}
{"item_id":"semeval2015.d001.s006.t000","gold_sense_keys":["main%5:00:00:important:00"],"candidates":[{"index":1,"sense_key":"main%5:00:00:important:00","synset_id":"chief.s.01"},{"index":2,"sense_key":"main%3:00:02::","synset_id":"independent.a.03"},{"index":3,"sense_key":"main%5:00:00:intense:00","synset_id":"main.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"main%5:00:00:important:00","call_ids":["semeval2015.d001.s006.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"main%5:00:00:important:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":604,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001328,"input_uncached_usd":0.0012079999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":6.832553083077073}
{"item_id":"semeval2015.d001.s006.t001","gold_sense_keys":["window%1:06:03::"],"candidates":[{"index":1,"sense_key":"window%1:06:00::","synset_id":"window.n.01"},{"index":2,"sense_key":"window%1:06:01::","synset_id":"window.n.02"},{"index":3,"sense_key":"window%1:06:02::","synset_id":"window.n.03"},{"index":4,"sense_key":"window%1:17:00::","synset_id":"window.n.04"},{"index":5,"sense_key":"window%1:28:00::","synset_id":"window.n.05"},{"index":6,"sense_key":"window%1:06:04::","synset_id":"windowpane.n.01"},{"index":7,"sense_key":"window%1:06:05::","synset_id":"window.n.07"},{"index":8,"sense_key":"window%1:06:03::","synset_id":"window.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"window%1:06:03::","call_ids":["semeval2015.d001.s006.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"window%1:06:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":860,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00184,"input_uncached_usd":0.00172,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.168402458075434}
{"item_id":"semeval2015.d001.s007.t000","gold_sense_keys":["main%5:00:00:important:00"],"candidates":[{"index":1,"sense_key":"main%5:00:00:important:00","synset_id":"chief.s.01"},{"index":2,"sense_key":"main%3:00:02::","synset_id":"independent.a.03"},{"index":3,"sense_key":"main%5:00:00:intense:00","synset_id":"main.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"main%5:00:00:important:00","call_ids":["semeval2015.d001.s007.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"main%5:00:00:important:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":585,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0012900000000000001,"input_uncached_usd":0.00117,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1286168330116197}
{"item_id":"semeval2015.d001.s007.t001","gold_sense_keys":["window%1:06:03::"],"candidates":[{"index":1,"sense_key":"window%1:06:00::","synset_id":"window.n.01"},{"index":2,"sense_key":"window%1:06:01::","synset_id":"window.n.02"},{"index":3,"sense_key":"window%1:06:02::","synset_id":"window.n.03"},{"index":4,"sense_key":"window%1:17:00::","synset_id":"window.n.04"},{"index":5,"sense_key":"window%1:28:00::","synset_id":"window.n.05"},{"index":6,"sense_key":"window%1:06:04::","synset_id":"windowpane.n.01"},{"index":7,"sense_key":"window%1:06:05::","synset_id":"window.n.07"},{"index":8,"sense_key":"window%1:06:03::","synset_id":"window.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"window%1:06:03::","call_ids":["semeval2015.d001.s007.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"window%1:06:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":841,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001802,"input_uncached_usd":0.001682,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0605071660829708}
{"item_id":"semeval2015.d001.s007.t002","gold_sense_keys":["consist%2:42:01::"],"candidates":[{"index":1,"sense_key":"consist%2:42:00::","synset_id":"dwell.v.02"},{"index":2,"sense_key":"consist%2:42:04::","synset_id":"consist.v.02"},{"index":3,"sense_key":"consist%2:42:02::","synset_id":"consist.v.03"},{"index":4,"sense_key":"consist%2:42:01::","synset_id":"consist.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"consist%2:42:01::","call_ids":["semeval2015.d001.s007.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"consist%2:42:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":643,"cached_input_tokens":0,"output_tokens":129,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002576,"input_uncached_usd":0.001286,"input_cached_usd":0.0,"output_usd":0.0012900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":11.957116374978796}
{"item_id":"semeval2015.d001.s007.t003","gold_sense_keys":["console%1:06:02::"],"candidates":[{"index":1,"sense_key":"console%1:06:03::","synset_id":"console_table.n.01"},{"index":2,"sense_key":"console%1:06:02::","synset_id":"console.n.02"},{"index":3,"sense_key":"console%1:06:01::","synset_id":"console.n.03"},{"index":4,"sense_key":"console%1:06:00::","synset_id":"cabinet.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"console%1:06:02::","call_ids":["semeval2015.d001.s007.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"console%1:06:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":615,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00135,"input_uncached_usd":0.00123,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.158596916939132}
{"item_id":"semeval2015.d001.s007.t005","gold_sense_keys":["3d%1:09:00::"],"candidates":[{"index":1,"sense_key":"3-d%1:10:00::","synset_id":"three-d.n.01"},{"index":2,"sense_key":"3-d%1:09:00::","synset_id":"three-d.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"3-d%1:09:00::","call_ids":["semeval2015.d001.s007.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"3-d%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":504,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001568,"input_uncached_usd":0.001008,"input_cached_usd":0.0,"output_usd":0.0005600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8606153749860823}
{"item_id":"semeval2015.d001.s008.t000","gold_sense_keys":["below%4:02:01::"],"candidates":[{"index":1,"sense_key":"below%4:02:01::","synset_id":"below.r.01"},{"index":2,"sense_key":"below%4:02:00::","synset_id":"below.r.02"},{"index":3,"sense_key":"below%4:02:02::","synset_id":"below.r.03"},{"index":4,"sense_key":"below%4:02:06::","synset_id":"downstairs.r.01"},{"index":5,"sense_key":"below%4:02:03::","synset_id":"under.r.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"below%4:02:01::","call_ids":["semeval2015.d001.s008.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"below%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":645,"cached_input_tokens":0,"output_tokens":97,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00226,"input_uncached_usd":0.00129,"input_cached_usd":0.0,"output_usd":0.00097,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":14.544102207990363}
{"item_id":"semeval2015.d001.s008.t001","gold_sense_keys":["find%2:39:02::"],"candidates":[{"index":1,"sense_key":"find%2:40:02::","synset_id":"find.v.01"},{"index":2,"sense_key":"find%2:39:02::","synset_id":"detect.v.01"},{"index":3,"sense_key":"find%2:40:00::","synset_id":"find.v.03"},{"index":4,"sense_key":"find%2:32:00::","synset_id":"determine.v.01"},{"index":5,"sense_key":"find%2:31:10::","synset_id":"find.v.05"},{"index":6,"sense_key":"find%2:39:01::","synset_id":"witness.v.02"},{"index":7,"sense_key":"find%2:40:01::","synset_id":"line_up.v.02"},{"index":8,"sense_key":"find%2:36:00::","synset_id":"discover.v.03"},{"index":9,"sense_key":"find%2:31:09::","synset_id":"discover.v.04"},{"index":10,"sense_key":"find%2:40:03::","synset_id":"find.v.10"},{"index":11,"sense_key":"find%2:32:01::","synset_id":"rule.v.04"},{"index":12,"sense_key":"find%2:30:00::","synset_id":"receive.v.02"},{"index":13,"sense_key":"find%2:39:05::","synset_id":"find.v.13"},{"index":14,"sense_key":"find%2:40:15::","synset_id":"recover.v.01"},{"index":15,"sense_key":"find%2:38:10::","synset_id":"find.v.15"},{"index":16,"sense_key":"find%2:30:13::","synset_id":"find_oneself.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"find%2:40:02::","call_ids":["semeval2015.d001.s008.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"find%2:40:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1566,"cached_input_tokens":0,"output_tokens":181,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004942,"input_uncached_usd":0.003132,"input_cached_usd":0.0,"output_usd":0.0018100000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.433951959013939}
{"item_id":"semeval2015.d001.s008.t004","gold_sense_keys":["type%2:32:00::"],"candidates":[{"index":1,"sense_key":"type%2:32:00::","synset_id":"type.v.01"},{"index":2,"sense_key":"type%2:31:00::","synset_id":"type.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"type%2:32:00::","call_ids":["semeval2015.d001.s008.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"type%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":525,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00117,"input_uncached_usd":0.00105,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5075466668931767}
{"item_id":"semeval2015.d001.s008.t005","gold_sense_keys":["function%1:24:00::"],"candidates":[{"index":1,"sense_key":"function%1:24:00::","synset_id":"function.n.01"},{"index":2,"sense_key":"function%1:07:00::","synset_id":"function.n.02"},{"index":3,"sense_key":"function%1:04:00::","synset_id":"function.n.03"},{"index":4,"sense_key":"function%1:24:01::","synset_id":"function.n.04"},{"index":5,"sense_key":"function%1:14:00::","synset_id":"function.n.05"},{"index":6,"sense_key":"function%1:11:00::","synset_id":"affair.n.03"},{"index":7,"sense_key":"function%1:10:00::","synset_id":"routine.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"function%1:24:00::","call_ids":["semeval2015.d001.s008.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"function%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":882,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0023740000000000002,"input_uncached_usd":0.001764,"input_cached_usd":0.0,"output_usd":0.0006100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.16479137504939}
{"item_id":"semeval2015.d001.s008.t006","gold_sense_keys":["do%2:36:01::","do%2:41:01::","do%2:41:04::"],"candidates":[{"index":1,"sense_key":"do%2:41:01::","synset_id":"make.v.01"},{"index":2,"sense_key":"do%2:36:01::","synset_id":"perform.v.01"},{"index":3,"sense_key":"do%2:41:04::","synset_id":"do.v.03"},{"index":4,"sense_key":"do%2:42:00::","synset_id":"do.v.04"},{"index":5,"sense_key":"do%2:36:02::","synset_id":"cause.v.01"},{"index":6,"sense_key":"do%2:41:02::","synset_id":"practice.v.01"},{"index":7,"sense_key":"do%2:42:01::","synset_id":"suffice.v.01"},{"index":8,"sense_key":"do%2:36:00::","synset_id":"do.v.08"},{"index":9,"sense_key":"do%2:29:09::","synset_id":"act.v.02"},{"index":10,"sense_key":"do%2:42:02::","synset_id":"serve.v.09"},{"index":11,"sense_key":"do%2:41:03::","synset_id":"do.v.11"},{"index":12,"sense_key":"do%2:29:04::","synset_id":"dress.v.16"},{"index":13,"sense_key":"do%2:38:00::","synset_id":"do.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"do%2:41:04::","call_ids":["semeval2015.d001.s008.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"do%2:41:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1226,"cached_input_tokens":0,"output_tokens":108,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0035319999999999995,"input_uncached_usd":0.0024519999999999998,"input_cached_usd":0.0,"output_usd":0.00108,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2522420410532504}
{"item_id":"semeval2015.d001.s008.t007","gold_sense_keys":["calculation%1:04:00::","calculation%1:09:00::"],"candidates":[{"index":1,"sense_key":"calculation%1:04:00::","synset_id":"calculation.n.01"},{"index":2,"sense_key":"calculation%1:09:00::","synset_id":"calculation.n.02"},{"index":3,"sense_key":"calculation%1:09:01::","synset_id":"calculation.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"calculation%1:04:00::","call_ids":["semeval2015.d001.s008.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"calculation%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":592,"cached_input_tokens":0,"output_tokens":142,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002604,"input_uncached_usd":0.001184,"input_cached_usd":0.0,"output_usd":0.00142,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.348649041960016}
{"item_id":"semeval2015.d001.s009.t000","gold_sense_keys":["want%2:37:00::"],"candidates":[{"index":1,"sense_key":"want%2:37:00::","synset_id":"desire.v.01"},{"index":2,"sense_key":"want%2:34:00::","synset_id":"want.v.02"},{"index":3,"sense_key":"want%2:35:00::","synset_id":"want.v.03"},{"index":4,"sense_key":"want%2:31:01::","synset_id":"want.v.04"},{"index":5,"sense_key":"want%2:42:01::","synset_id":"want.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"want%2:37:00::","call_ids":["semeval2015.d001.s009.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"want%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":691,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001502,"input_uncached_usd":0.001382,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6713266669539735}
{"item_id":"semeval2015.d001.s009.t001","gold_sense_keys":["understand%2:31:00::","understand%2:31:01::"],"candidates":[{"index":1,"sense_key":"understand%2:31:00::","synset_id":"understand.v.01"},{"index":2,"sense_key":"understand%2:31:01::","synset_id":"understand.v.02"},{"index":3,"sense_key":"understand%2:31:03::","synset_id":"understand.v.03"},{"index":4,"sense_key":"understand%2:31:02::","synset_id":"understand.v.04"},{"index":5,"sense_key":"understand%2:31:04::","synset_id":"sympathize.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"understand%2:31:00::","call_ids":["semeval2015.d001.s009.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"understand%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":722,"cached_input_tokens":0,"output_tokens":90,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002344,"input_uncached_usd":0.001444,"input_cached_usd":0.0,"output_usd":0.0009000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9472749580163509}
{"item_id":"semeval2015.d001.s009.t002","gold_sense_keys":["work%2:35:02::","work%2:41:03::"],"candidates":[{"index":1,"sense_key":"work%2:41:02::","synset_id":"work.v.01"},{"index":2,"sense_key":"work%2:41:00::","synset_id":"work.v.02"},{"index":3,"sense_key":"work%2:41:03::","synset_id":"work.v.03"},{"index":4,"sense_key":"work%2:35:02::","synset_id":"function.v.01"},{"index":5,"sense_key":"work%2:36:00::","synset_id":"work.v.05"},{"index":6,"sense_key":"work%2:29:00::","synset_id":"exercise.v.03"},{"index":7,"sense_key":"work%2:38:00::","synset_id":"make.v.36"},{"index":8,"sense_key":"work%2:41:04::","synset_id":"work.v.08"},{"index":9,"sense_key":"work%2:41:05::","synset_id":"work.v.09"},{"index":10,"sense_key":"work%2:38:02::","synset_id":"work.v.10"},{"index":11,"sense_key":"work%2:36:01::","synset_id":"bring.v.03"},{"index":12,"sense_key":"work%2:41:01::","synset_id":"work.v.12"},{"index":13,"sense_key":"work%2:36:09::","synset_id":"cultivate.v.02"},{"index":14,"sense_key":"work%2:42:10::","synset_id":"work.v.14"},{"index":15,"sense_key":"work%2:41:10::","synset_id":"influence.v.01"},{"index":16,"sense_key":"work%2:41:11::","synset_id":"work.v.16"},{"index":17,"sense_key":"work%2:41:12::","synset_id":"work.v.17"},{"index":18,"sense_key":"work%2:39:10::","synset_id":"work.v.18"},{"index":19,"sense_key":"work%2:37:10::","synset_id":"work.v.19"},{"index":20,"sense_key":"work%2:36:12::","synset_id":"shape.v.02"},{"index":21,"sense_key":"work%2:35:14::","synset_id":"work.v.21"},{"index":22,"sense_key":"work%2:35:00::","synset_id":"knead.v.01"},{"index":23,"sense_key":"work%2:34:10::","synset_id":"exploit.v.01"},{"index":24,"sense_key":"work%2:31:13::","synset_id":"solve.v.01"},{"index":25,"sense_key":"work%2:30:14::","synset_id":"ferment.v.03"},{"index":26,"sense_key":"work%2:30:13::","synset_id":"sour.v.01"},{"index":27,"sense_key":"work%2:30:10::","synset_id":"work.v.27"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"work%2:35:02::","call_ids":["semeval2015.d001.s009.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"work%2:35:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2063,"cached_input_tokens":0,"output_tokens":50,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004626,"input_uncached_usd":0.0041259999999999995,"input_cached_usd":0.0,"output_usd":0.0005,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7723011669004336}
{"item_id":"semeval2015.d001.s009.t003","gold_sense_keys":["best%5:00:00:advisable:00"],"candidates":[{"index":1,"sense_key":"best%3:00:00::","synset_id":"best.a.01"},{"index":2,"sense_key":"best%5:00:00:advisable:00","synset_id":"better.s.01"},{"index":3,"sense_key":"good%3:00:01::","synset_id":"good.a.01"},{"index":4,"sense_key":"full%5:00:00:ample:00","synset_id":"full.s.04"},{"index":5,"sense_key":"good%3:00:02::","synset_id":"good.a.03"},{"index":6,"sense_key":"estimable%5:00:00:reputable:00","synset_id":"estimable.s.01"},{"index":7,"sense_key":"beneficial%5:00:00:advantageous:00","synset_id":"beneficial.s.01"},{"index":8,"sense_key":"good%5:00:00:nice:00","synset_id":"good.s.04"},{"index":9,"sense_key":"good%5:00:00:righteous:00","synset_id":"good.s.05"},{"index":10,"sense_key":"adept%5:00:00:skilled:00","synset_id":"adept.s.01"},{"index":11,"sense_key":"good%5:00:00:complete:00","synset_id":"good.s.07"},{"index":12,"sense_key":"dear%5:00:00:close:02","synset_id":"dear.s.02"},{"index":13,"sense_key":"dependable%5:00:00:sound:00","synset_id":"dependable.s.03"},{"index":14,"sense_key":"good%5:00:00:opportune:00","synset_id":"good.s.10"},{"index":15,"sense_key":"good%5:00:00:fortunate:00","synset_id":"good.s.11"},{"index":16,"sense_key":"effective%5:00:00:operative:00","synset_id":"effective.s.03"},{"index":17,"sense_key":"good%5:00:00:pleasing:00","synset_id":"good.s.13"},{"index":18,"sense_key":"good%5:00:00:intellectual:00","synset_id":"good.s.14"},{"index":19,"sense_key":"good%5:00:00:healthy:00","synset_id":"good.s.15"},{"index":20,"sense_key":"good%5:00:00:healthful:00","synset_id":"good.s.16"},{"index":21,"sense_key":"good%5:00:00:genuine:00","synset_id":"good.s.17"},{"index":22,"sense_key":"good%5:00:00:fresh:01","synset_id":"good.s.18"},{"index":23,"sense_key":"good%5:00:00:discriminating:00","synset_id":"good.s.19"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"best%5:00:00:advisable:00","call_ids":["semeval2015.d001.s009.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"best%5:00:00:advisable:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1892,"cached_input_tokens":0,"output_tokens":97,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004754,"input_uncached_usd":0.003784,"input_cached_usd":0.0,"output_usd":0.00097,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7502986250910908}
{"item_id":"semeval2015.d001.s009.t004","gold_sense_keys":["thing%1:04:00::"],"candidates":[{"index":1,"sense_key":"thing%1:26:00::","synset_id":"thing.n.01"},{"index":2,"sense_key":"thing%1:04:00::","synset_id":"thing.n.02"},{"index":3,"sense_key":"thing%1:09:00::","synset_id":"thing.n.03"},{"index":4,"sense_key":"thing%1:06:00::","synset_id":"thing.n.04"},{"index":5,"sense_key":"thing%1:11:00::","synset_id":"thing.n.05"},{"index":6,"sense_key":"thing%1:09:02::","synset_id":"matter.n.01"},{"index":7,"sense_key":"thing%1:10:00::","synset_id":"thing.n.07"},{"index":8,"sense_key":"thing%1:06:01::","synset_id":"thing.n.08"},{"index":9,"sense_key":"thing%1:07:00::","synset_id":"thing.n.09"},{"index":10,"sense_key":"thing%1:09:01::","synset_id":"thing.n.10"},{"index":11,"sense_key":"thing%1:12:00::","synset_id":"thing.n.11"},{"index":12,"sense_key":"thing%1:03:00::","synset_id":"thing.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"thing%1:04:00::","call_ids":["semeval2015.d001.s009.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"thing%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":981,"cached_input_tokens":0,"output_tokens":110,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0030619999999999996,"input_uncached_usd":0.0019619999999999998,"input_cached_usd":0.0,"output_usd":0.0011,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":14.899461250053719}
{"item_id":"semeval2015.d001.s009.t005","gold_sense_keys":["do%2:36:01::"],"candidates":[{"index":1,"sense_key":"do%2:41:01::","synset_id":"make.v.01"},{"index":2,"sense_key":"do%2:36:01::","synset_id":"perform.v.01"},{"index":3,"sense_key":"do%2:41:04::","synset_id":"do.v.03"},{"index":4,"sense_key":"do%2:42:00::","synset_id":"do.v.04"},{"index":5,"sense_key":"do%2:36:02::","synset_id":"cause.v.01"},{"index":6,"sense_key":"do%2:41:02::","synset_id":"practice.v.01"},{"index":7,"sense_key":"do%2:42:01::","synset_id":"suffice.v.01"},{"index":8,"sense_key":"do%2:36:00::","synset_id":"do.v.08"},{"index":9,"sense_key":"do%2:29:09::","synset_id":"act.v.02"},{"index":10,"sense_key":"do%2:42:02::","synset_id":"serve.v.09"},{"index":11,"sense_key":"do%2:41:03::","synset_id":"do.v.11"},{"index":12,"sense_key":"do%2:29:04::","synset_id":"dress.v.16"},{"index":13,"sense_key":"do%2:38:00::","synset_id":"do.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"do%2:36:01::","call_ids":["semeval2015.d001.s009.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"do%2:36:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1234,"cached_input_tokens":0,"output_tokens":137,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003838,"input_uncached_usd":0.0024679999999999997,"input_cached_usd":0.0,"output_usd":0.0013700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.138773625018075}
{"item_id":"semeval2015.d001.s009.t006","gold_sense_keys":["realize%2:31:01::"],"candidates":[{"index":1,"sense_key":"realize%2:31:01::","synset_id":"recognize.v.02"},{"index":2,"sense_key":"realize%2:31:00::","synset_id":"understand.v.02"},{"index":3,"sense_key":"realize%2:36:00::","synset_id":"realize.v.03"},{"index":4,"sense_key":"realize%2:40:01::","synset_id":"gain.v.08"},{"index":5,"sense_key":"realize%2:40:00::","synset_id":"realize.v.05"},{"index":6,"sense_key":"realize%2:36:07::","synset_id":"realize.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"realize%2:31:01::","call_ids":["semeval2015.d001.s009.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"realize%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":821,"cached_input_tokens":0,"output_tokens":229,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003932000000000001,"input_uncached_usd":0.001642,"input_cached_usd":0.0,"output_usd":0.0022900000000000004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":11.946310083032586}
{"item_id":"semeval2015.d001.s009.t008","gold_sense_keys":["convert%2:30:00::","convert%2:30:02::","convert%2:30:03::"],"candidates":[{"index":1,"sense_key":"convert%2:30:00::","synset_id":"convert.v.01"},{"index":2,"sense_key":"convert%2:30:03::","synset_id":"convert.v.02"},{"index":3,"sense_key":"convert%2:30:01::","synset_id":"convert.v.03"},{"index":4,"sense_key":"convert%2:30:02::","synset_id":"change.v.06"},{"index":5,"sense_key":"convert%2:30:07::","synset_id":"convert.v.05"},{"index":6,"sense_key":"convert%2:33:02::","synset_id":"convert.v.06"},{"index":7,"sense_key":"convert%2:33:01::","synset_id":"convert.v.07"},{"index":8,"sense_key":"convert%2:33:00::","synset_id":"convert.v.08"},{"index":9,"sense_key":"convert%2:32:00::","synset_id":"convert.v.09"},{"index":10,"sense_key":"convert%2:30:09::","synset_id":"commute.v.04"},{"index":11,"sense_key":"convert%2:30:04::","synset_id":"convert.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"convert%2:30:03::","call_ids":["semeval2015.d001.s009.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"convert%2:30:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1043,"cached_input_tokens":0,"output_tokens":137,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003456,"input_uncached_usd":0.0020859999999999997,"input_cached_usd":0.0,"output_usd":0.0013700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.395559166907333}
{"item_id":"semeval2015.d001.s009.t010","gold_sense_keys":["necessary%3:00:00::"],"candidates":[{"index":1,"sense_key":"necessary%3:00:00::","synset_id":"necessary.a.01"},{"index":2,"sense_key":"necessary%5:00:00:inevitable:00","synset_id":"necessary.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"necessary%3:00:00::","call_ids":["semeval2015.d001.s009.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"necessary%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":515,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00115,"input_uncached_usd":0.0010299999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.650142833008431}
{"item_id":"semeval2015.d001.s009.t011","gold_sense_keys":["know%2:31:15::"],"candidates":[{"index":1,"sense_key":"know%2:31:01::","synset_id":"know.v.01"},{"index":2,"sense_key":"know%2:31:03::","synset_id":"know.v.02"},{"index":3,"sense_key":"know%2:31:02::","synset_id":"know.v.03"},{"index":4,"sense_key":"know%2:31:00::","synset_id":"know.v.04"},{"index":5,"sense_key":"know%2:31:04::","synset_id":"know.v.05"},{"index":6,"sense_key":"know%2:31:06::","synset_id":"acknowledge.v.06"},{"index":7,"sense_key":"know%2:31:15::","synset_id":"know.v.07"},{"index":8,"sense_key":"know%2:35:00::","synset_id":"sleep_together.v.01"},{"index":9,"sense_key":"know%2:31:11::","synset_id":"know.v.09"},{"index":10,"sense_key":"know%2:31:12::","synset_id":"know.v.10"},{"index":11,"sense_key":"know%2:31:14::","synset_id":"know.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"know%2:31:15::","call_ids":["semeval2015.d001.s009.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"know%2:31:15::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1145,"cached_input_tokens":0,"output_tokens":101,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0033,"input_uncached_usd":0.00229,"input_cached_usd":0.0,"output_usd":0.00101,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1968475420726463}
{"item_id":"semeval2015.d001.s010.t000","gold_sense_keys":["base%2:31:00::"],"candidates":[{"index":1,"sense_key":"base%2:31:00::","synset_id":"establish.v.08"},{"index":2,"sense_key":"base%2:42:04::","synset_id":"base.v.02"},{"index":3,"sense_key":"base%2:34:00::","synset_id":"free-base.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"base%2:31:00::","call_ids":["semeval2015.d001.s010.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"base%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":566,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001252,"input_uncached_usd":0.001132,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1366036250256002}
{"item_id":"semeval2015.d001.s010.t004","gold_sense_keys":["hard%3:00:06::"],"candidates":[{"index":1,"sense_key":"hard%3:00:06::","synset_id":"difficult.a.01"},{"index":2,"sense_key":"hard%3:00:02::","synset_id":"hard.a.02"},{"index":3,"sense_key":"hard%3:00:01::","synset_id":"hard.a.03"},{"index":4,"sense_key":"hard%5:00:00:strong:00","synset_id":"hard.s.01"},{"index":5,"sense_key":"hard%5:00:01:effortful:00","synset_id":"arduous.s.01"},{"index":6,"sense_key":"hard%3:00:00::","synset_id":"unvoiced.a.01"},{"index":7,"sense_key":"hard%3:00:05::","synset_id":"hard.a.07"},{"index":8,"sense_key":"hard%3:00:03::","synset_id":"hard.a.08"},{"index":9,"sense_key":"hard%5:00:00:indulgent:00","synset_id":"intemperate.s.01"},{"index":10,"sense_key":"hard%5:00:00:alcoholic:00","synset_id":"hard.s.04"},{"index":11,"sense_key":"hard%5:00:00:bad:00","synset_id":"hard.s.05"},{"index":12,"sense_key":"hard%5:00:00:stale:00","synset_id":"hard.s.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"hard%3:00:06::","call_ids":["semeval2015.d001.s010.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"hard%3:00:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1203,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0025259999999999996,"input_uncached_usd":0.0024059999999999997,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.525305249961093}
{"item_id":"semeval2015.d001.s011.t001","gold_sense_keys":["closely%4:02:02::"],"candidates":[{"index":1,"sense_key":"closely%4:02:00::","synset_id":"closely.r.01"},{"index":2,"sense_key":"closely%4:02:02::","synset_id":"close.r.02"},{"index":3,"sense_key":"closely%4:02:01::","synset_id":"closely.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"closely%4:02:02::","call_ids":["semeval2015.d001.s011.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"closely%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":593,"cached_input_tokens":0,"output_tokens":170,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002886,"input_uncached_usd":0.001186,"input_cached_usd":0.0,"output_usd":0.0017000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.222414084011689}
{"item_id":"semeval2015.d001.s011.t002","gold_sense_keys":["look%2:39:01::"],"candidates":[{"index":1,"sense_key":"look%2:39:00::","synset_id":"look.v.01"},{"index":2,"sense_key":"look%2:39:01::","synset_id":"look.v.02"},{"index":3,"sense_key":"look%2:29:00::","synset_id":"look.v.03"},{"index":4,"sense_key":"look%2:39:02::","synset_id":"search.v.02"},{"index":5,"sense_key":"look%2:42:00::","synset_id":"front.v.01"},{"index":6,"sense_key":"look%2:41:13::","synset_id":"attend.v.02"},{"index":7,"sense_key":"look%2:32:00::","synset_id":"look.v.07"},{"index":8,"sense_key":"look%2:31:00::","synset_id":"expect.v.03"},{"index":9,"sense_key":"look%2:42:02::","synset_id":"look.v.09"},{"index":10,"sense_key":"look%2:31:02::","synset_id":"count.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"look%2:39:01::","call_ids":["semeval2015.d001.s011.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"look%2:39:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1036,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002612,"input_uncached_usd":0.002072,"input_cached_usd":0.0,"output_usd":0.00054,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.407374250004068}
{"item_id":"semeval2015.d001.s011.t003","gold_sense_keys":["very%4:02:00::"],"candidates":[{"index":1,"sense_key":"very%4:02:00::","synset_id":"very.r.01"},{"index":2,"sense_key":"very%4:02:01::","synset_id":"very.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"very%4:02:00::","call_ids":["semeval2015.d001.s011.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"very%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":545,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0012100000000000001,"input_uncached_usd":0.00109,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1989879159955308}
{"item_id":"semeval2015.d001.s011.t004","gold_sense_keys":["similar%3:00:00::","similar%3:00:02::","similar%3:00:04::"],"candidates":[{"index":1,"sense_key":"similar%3:00:00::","synset_id":"similar.a.01"},{"index":2,"sense_key":"similar%3:00:04::","synset_id":"alike.a.01"},{"index":3,"sense_key":"similar%3:00:02::","synset_id":"like.a.01"},{"index":4,"sense_key":"similar%5:00:00:synonymous:00","synset_id":"similar.s.01"},{"index":5,"sense_key":"similar%5:00:00:replaceable:00","synset_id":"exchangeable.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"similar%3:00:00::","call_ids":["semeval2015.d001.s011.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"similar%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":760,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00164,"input_uncached_usd":0.0015199999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":4.434911083080806}
{"item_id":"semeval2015.d001.s012.t000","gold_sense_keys":["addition%1:04:01::"],"candidates":[{"index":1,"sense_key":"addition%1:06:00::","synset_id":"addition.n.01"},{"index":2,"sense_key":"addition%1:04:02::","synset_id":"addition.n.02"},{"index":3,"sense_key":"addition%1:23:00::","synset_id":"addition.n.03"},{"index":4,"sense_key":"addition%1:21:00::","synset_id":"accession.n.03"},{"index":5,"sense_key":"addition%1:15:00::","synset_id":"addition.n.05"},{"index":6,"sense_key":"addition%1:04:01::","synset_id":"summation.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"addition%1:04:01::","call_ids":["semeval2015.d001.s012.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"addition%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":840,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0018,"input_uncached_usd":0.0016799999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.136327916989103}
{"item_id":"semeval2015.d001.s012.t001","gold_sense_keys":["subtraction%1:04:01::"],"candidates":[{"index":1,"sense_key":"subtraction%1:04:01::","synset_id":"subtraction.n.01"},{"index":2,"sense_key":"subtraction%1:04:00::","synset_id":"subtraction.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"subtraction%1:04:01::","call_ids":["semeval2015.d001.s012.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"subtraction%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":584,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001288,"input_uncached_usd":0.001168,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1404825830832124}
{"item_id":"semeval2015.d001.s012.t002","gold_sense_keys":["multiplication%1:04:00::"],"candidates":[{"index":1,"sense_key":"multiplication%1:04:01::","synset_id":"generation.n.07"},{"index":2,"sense_key":"multiplication%1:22:00::","synset_id":"multiplication.n.02"},{"index":3,"sense_key":"multiplication%1:04:00::","synset_id":"multiplication.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"multiplication%1:04:00::","call_ids":["semeval2015.d001.s012.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"multiplication%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":642,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001404,"input_uncached_usd":0.001284,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5933713749982417}
{"item_id":"semeval2015.d001.s012.t003","gold_sense_keys":["division%1:04:01::"],"candidates":[{"index":1,"sense_key":"division%1:14:00::","synset_id":"division.n.01"},{"index":2,"sense_key":"division%1:09:00::","synset_id":"part.n.09"},{"index":3,"sense_key":"division%1:04:00::","synset_id":"division.n.03"},{"index":4,"sense_key":"division%1:14:03::","synset_id":"division.n.04"},{"index":5,"sense_key":"division%1:10:00::","synset_id":"division.n.05"},{"index":6,"sense_key":"division%1:14:06::","synset_id":"class.n.05"},{"index":7,"sense_key":"division%1:14:05::","synset_id":"division.n.07"},{"index":8,"sense_key":"division%1:14:04::","synset_id":"division.n.08"},{"index":9,"sense_key":"division%1:14:02::","synset_id":"division.n.09"},{"index":10,"sense_key":"division%1:14:01::","synset_id":"division.n.10"},{"index":11,"sense_key":"division%1:04:01::","synset_id":"division.n.11"},{"index":12,"sense_key":"division%1:04:02::","synset_id":"division.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"division%1:04:01::","call_ids":["semeval2015.d001.s012.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"division%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1031,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002182,"input_uncached_usd":0.002062,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2530257919570431}
{"item_id":"semeval2015.d001.s013.t000","gold_sense_keys":["power%1:10:00::"],"candidates":[{"index":1,"sense_key":"power%1:07:00::","synset_id":"power.n.01"},{"index":2,"sense_key":"power%1:19:00::","synset_id":"power.n.02"},{"index":3,"sense_key":"power%1:09:01::","synset_id":"ability.n.02"},{"index":4,"sense_key":"power%1:26:00::","synset_id":"office.n.04"},{"index":5,"sense_key":"power%1:18:00::","synset_id":"power.n.05"},{"index":6,"sense_key":"power%1:10:00::","synset_id":"exponent.n.03"},{"index":7,"sense_key":"power%1:07:02::","synset_id":"might.n.01"},{"index":8,"sense_key":"power%1:14:00::","synset_id":"world_power.n.01"},{"index":9,"sense_key":"power%1:18:01::","synset_id":"baron.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"power%1:10:00::","call_ids":["semeval2015.d001.s013.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"power%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":939,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0019979999999999998,"input_uncached_usd":0.001878,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8434095000848174}
{"item_id":"semeval2015.d001.s013.t001","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":2,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":3,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":4,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":5,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":6,"sense_key":"use%2:41:03::","synset_id":"use.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%2:34:01::","call_ids":["semeval2015.d001.s013.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":787,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001694,"input_uncached_usd":0.0015739999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8393089160090312}
{"item_id":"semeval2015.d001.s014.t001","gold_sense_keys":["possible%3:00:00::"],"candidates":[{"index":1,"sense_key":"possible%3:00:00::","synset_id":"possible.a.01"},{"index":2,"sense_key":"possible%3:00:04::","synset_id":"potential.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"possible%3:00:00::","call_ids":["semeval2015.d001.s014.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"possible%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":498,"cached_input_tokens":0,"output_tokens":92,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001916,"input_uncached_usd":0.000996,"input_cached_usd":0.0,"output_usd":0.00092,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":14.74393679201603}
{"item_id":"semeval2015.d001.s014.t002","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":2,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":3,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":4,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":5,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":6,"sense_key":"use%2:41:03::","synset_id":"use.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%2:34:01::","call_ids":["semeval2015.d001.s014.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":775,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00167,"input_uncached_usd":0.00155,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":8.79835095803719}
{"item_id":"semeval2015.d001.s015.t000","gold_sense_keys":["power%1:10:00::"],"candidates":[{"index":1,"sense_key":"power%1:07:00::","synset_id":"power.n.01"},{"index":2,"sense_key":"power%1:19:00::","synset_id":"power.n.02"},{"index":3,"sense_key":"power%1:09:01::","synset_id":"ability.n.02"},{"index":4,"sense_key":"power%1:26:00::","synset_id":"office.n.04"},{"index":5,"sense_key":"power%1:18:00::","synset_id":"power.n.05"},{"index":6,"sense_key":"power%1:10:00::","synset_id":"exponent.n.03"},{"index":7,"sense_key":"power%1:07:02::","synset_id":"might.n.01"},{"index":8,"sense_key":"power%1:14:00::","synset_id":"world_power.n.01"},{"index":9,"sense_key":"power%1:18:01::","synset_id":"baron.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"power%1:10:00::","call_ids":["semeval2015.d001.s015.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"power%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":904,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001928,"input_uncached_usd":0.001808,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0951904999092221}
{"item_id":"semeval2015.d001.s015.t001","gold_sense_keys":["way%1:04:01::","way%1:07:01::"],"candidates":[{"index":1,"sense_key":"way%1:07:01::","synset_id":"manner.n.01"},{"index":2,"sense_key":"way%1:04:01::","synset_id":"means.n.01"},{"index":3,"sense_key":"way%1:15:00::","synset_id":"direction.n.01"},{"index":4,"sense_key":"way%1:26:00::","synset_id":"way.n.04"},{"index":5,"sense_key":"way%1:04:00::","synset_id":"way.n.05"},{"index":6,"sense_key":"way%1:06:00::","synset_id":"way.n.06"},{"index":7,"sense_key":"way%1:04:02::","synset_id":"way.n.07"},{"index":8,"sense_key":"way%1:23:00::","synset_id":"room.n.02"},{"index":9,"sense_key":"way%1:07:02::","synset_id":"way.n.09"},{"index":10,"sense_key":"way%1:09:00::","synset_id":"way.n.10"},{"index":11,"sense_key":"way%1:09:01::","synset_id":"way.n.11"},{"index":12,"sense_key":"way%1:21:00::","synset_id":"way.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"way%1:04:01::","call_ids":["semeval2015.d001.s015.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"way%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":983,"cached_input_tokens":0,"output_tokens":75,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0027159999999999997,"input_uncached_usd":0.001966,"input_cached_usd":0.0,"output_usd":0.00075,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9441662089666352}
{"item_id":"semeval2015.d001.s015.t003","gold_sense_keys":["root%1:23:00::"],"candidates":[{"index":1,"sense_key":"root%1:20:00::","synset_id":"root.n.01"},{"index":2,"sense_key":"root%1:15:00::","synset_id":"beginning.n.04"},{"index":3,"sense_key":"root%1:10:00::","synset_id":"root.n.03"},{"index":4,"sense_key":"root%1:23:00::","synset_id":"root.n.04"},{"index":5,"sense_key":"root%1:14:00::","synset_id":"solution.n.04"},{"index":6,"sense_key":"root%1:18:00::","synset_id":"ancestor.n.01"},{"index":7,"sense_key":"root%1:10:01::","synset_id":"etymon.n.01"},{"index":8,"sense_key":"root%1:08:00::","synset_id":"root.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"root%1:23:00::","call_ids":["semeval2015.d001.s015.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"root%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":870,"cached_input_tokens":0,"output_tokens":71,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00245,"input_uncached_usd":0.00174,"input_cached_usd":0.0,"output_usd":0.00071,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9774806250352412}
{"item_id":"semeval2015.d001.s015.t004","gold_sense_keys":["too%4:02:01::"],"candidates":[{"index":1,"sense_key":"too%4:02:00::","synset_id":"excessively.r.01"},{"index":2,"sense_key":"too%4:02:01::","synset_id":"besides.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"too%4:02:01::","call_ids":["semeval2015.d001.s015.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"too%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":478,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001076,"input_uncached_usd":0.0009559999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5725158340064809}
{"item_id":"semeval2015.d001.s016.t000","gold_sense_keys":["way%1:04:01::","way%1:07:01::"],"candidates":[{"index":1,"sense_key":"way%1:07:01::","synset_id":"manner.n.01"},{"index":2,"sense_key":"way%1:04:01::","synset_id":"means.n.01"},{"index":3,"sense_key":"way%1:15:00::","synset_id":"direction.n.01"},{"index":4,"sense_key":"way%1:26:00::","synset_id":"way.n.04"},{"index":5,"sense_key":"way%1:04:00::","synset_id":"way.n.05"},{"index":6,"sense_key":"way%1:06:00::","synset_id":"way.n.06"},{"index":7,"sense_key":"way%1:04:02::","synset_id":"way.n.07"},{"index":8,"sense_key":"way%1:23:00::","synset_id":"room.n.02"},{"index":9,"sense_key":"way%1:07:02::","synset_id":"way.n.09"},{"index":10,"sense_key":"way%1:09:00::","synset_id":"way.n.10"},{"index":11,"sense_key":"way%1:09:01::","synset_id":"way.n.11"},{"index":12,"sense_key":"way%1:21:00::","synset_id":"way.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"way%1:04:01::","call_ids":["semeval2015.d001.s016.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"way%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":972,"cached_input_tokens":0,"output_tokens":112,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003064,"input_uncached_usd":0.001944,"input_cached_usd":0.0,"output_usd":0.0011200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7895512079121545}
{"item_id":"semeval2015.d001.s016.t001","gold_sense_keys":["specify%2:32:02::"],"candidates":[{"index":1,"sense_key":"specify%2:32:03::","synset_id":"stipulate.v.01"},{"index":2,"sense_key":"specify%2:32:02::","synset_id":"specify.v.02"},{"index":3,"sense_key":"specify%2:42:00::","synset_id":"specify.v.03"},{"index":4,"sense_key":"specify%2:32:00::","synset_id":"specify.v.04"},{"index":5,"sense_key":"specify%2:31:02::","synset_id":"pin_down.v.01"},{"index":6,"sense_key":"specify%2:31:00::","synset_id":"intend.v.02"},{"index":7,"sense_key":"specify%2:31:05::","synset_id":"assign.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"specify%2:32:02::","call_ids":["semeval2015.d001.s016.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"specify%2:32:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":839,"cached_input_tokens":0,"output_tokens":118,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0028580000000000003,"input_uncached_usd":0.001678,"input_cached_usd":0.0,"output_usd":0.00118,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":9.67267641692888}
{"item_id":"semeval2015.d001.s016.t003","gold_sense_keys":["variable%1:10:00::"],"candidates":[{"index":1,"sense_key":"variable%1:17:00::","synset_id":"variable.n.01"},{"index":2,"sense_key":"variable%1:09:00::","synset_id":"variable.n.02"},{"index":3,"sense_key":"variable%1:17:01::","synset_id":"variable_star.n.01"},{"index":4,"sense_key":"variable%1:10:00::","synset_id":"variable.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"variable%1:10:00::","call_ids":["semeval2015.d001.s016.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"variable%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":555,"cached_input_tokens":0,"output_tokens":113,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00224,"input_uncached_usd":0.0011099999999999999,"input_cached_usd":0.0,"output_usd":0.0011300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":9.507209458970465}
{"item_id":"semeval2015.d001.s016.t004","gold_sense_keys":["function%1:24:00::"],"candidates":[{"index":1,"sense_key":"function%1:24:00::","synset_id":"function.n.01"},{"index":2,"sense_key":"function%1:07:00::","synset_id":"function.n.02"},{"index":3,"sense_key":"function%1:04:00::","synset_id":"function.n.03"},{"index":4,"sense_key":"function%1:24:01::","synset_id":"function.n.04"},{"index":5,"sense_key":"function%1:14:00::","synset_id":"function.n.05"},{"index":6,"sense_key":"function%1:11:00::","synset_id":"affair.n.03"},{"index":7,"sense_key":"function%1:10:00::","synset_id":"routine.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"function%1:24:00::","call_ids":["semeval2015.d001.s016.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"function%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":819,"cached_input_tokens":0,"output_tokens":114,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002778,"input_uncached_usd":0.001638,"input_cached_usd":0.0,"output_usd":0.0011400000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0529527919134125}
{"item_id":"semeval2015.d001.s017.t000","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":2,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":3,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":4,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":5,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":6,"sense_key":"use%2:41:03::","synset_id":"use.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%2:34:01::","call_ids":["semeval2015.d001.s017.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":741,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0016020000000000001,"input_uncached_usd":0.001482,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6233123750425875}
{"item_id":"semeval2015.d001.s017.t001","gold_sense_keys":["specify%2:32:02::"],"candidates":[{"index":1,"sense_key":"specify%2:32:03::","synset_id":"stipulate.v.01"},{"index":2,"sense_key":"specify%2:32:02::","synset_id":"specify.v.02"},{"index":3,"sense_key":"specify%2:42:00::","synset_id":"specify.v.03"},{"index":4,"sense_key":"specify%2:32:00::","synset_id":"specify.v.04"},{"index":5,"sense_key":"specify%2:31:02::","synset_id":"pin_down.v.01"},{"index":6,"sense_key":"specify%2:31:00::","synset_id":"intend.v.02"},{"index":7,"sense_key":"specify%2:31:05::","synset_id":"assign.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"specify%2:32:02::","call_ids":["semeval2015.d001.s017.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"specify%2:32:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":840,"cached_input_tokens":0,"output_tokens":120,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0028799999999999997,"input_uncached_usd":0.0016799999999999999,"input_cached_usd":0.0,"output_usd":0.0012000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0950977499596775}
{"item_id":"semeval2015.d001.s017.t002","gold_sense_keys":["high%3:00:02::"],"candidates":[{"index":1,"sense_key":"high%3:00:02::","synset_id":"high.a.01"},{"index":2,"sense_key":"high%3:00:01::","synset_id":"high.a.02"},{"index":3,"sense_key":"high%5:00:00:superior:01","synset_id":"eminent.s.01"},{"index":4,"sense_key":"high%3:00:03::","synset_id":"high.a.04"},{"index":5,"sense_key":"high%5:00:00:elated:00","synset_id":"high.s.02"},{"index":6,"sense_key":"high%5:00:00:malodorous:00","synset_id":"gamey.s.02"},{"index":7,"sense_key":"high%5:00:00:intoxicated:00","synset_id":"high.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"high%5:00:00:superior:01","call_ids":["semeval2015.d001.s017.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"high%5:00:00:superior:01","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":764,"cached_input_tokens":0,"output_tokens":103,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002558,"input_uncached_usd":0.0015279999999999998,"input_cached_usd":0.0,"output_usd":0.00103,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8671639580279589}
{"item_id":"semeval2015.d001.s017.t003","gold_sense_keys":["priority%1:26:00::"],"candidates":[{"index":1,"sense_key":"priority%1:26:00::","synset_id":"precedence.n.01"},{"index":2,"sense_key":"priority%1:07:00::","synset_id":"priority.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"priority%1:26:00::","call_ids":["semeval2015.d001.s017.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"priority%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":499,"cached_input_tokens":0,"output_tokens":75,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001748,"input_uncached_usd":0.000998,"input_cached_usd":0.0,"output_usd":0.00075,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.5761874589370564}
{"item_id":"semeval2015.d001.s018.t001","gold_sense_keys":["find%2:32:00::","find%2:39:02::"],"candidates":[{"index":1,"sense_key":"find%2:40:02::","synset_id":"find.v.01"},{"index":2,"sense_key":"find%2:39:02::","synset_id":"detect.v.01"},{"index":3,"sense_key":"find%2:40:00::","synset_id":"find.v.03"},{"index":4,"sense_key":"find%2:32:00::","synset_id":"determine.v.01"},{"index":5,"sense_key":"find%2:31:10::","synset_id":"find.v.05"},{"index":6,"sense_key":"find%2:39:01::","synset_id":"witness.v.02"},{"index":7,"sense_key":"find%2:40:01::","synset_id":"line_up.v.02"},{"index":8,"sense_key":"find%2:36:00::","synset_id":"discover.v.03"},{"index":9,"sense_key":"find%2:31:09::","synset_id":"discover.v.04"},{"index":10,"sense_key":"find%2:40:03::","synset_id":"find.v.10"},{"index":11,"sense_key":"find%2:32:01::","synset_id":"rule.v.04"},{"index":12,"sense_key":"find%2:30:00::","synset_id":"receive.v.02"},{"index":13,"sense_key":"find%2:39:05::","synset_id":"find.v.13"},{"index":14,"sense_key":"find%2:40:15::","synset_id":"recover.v.01"},{"index":15,"sense_key":"find%2:38:10::","synset_id":"find.v.15"},{"index":16,"sense_key":"find%2:30:13::","synset_id":"find_oneself.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"find%2:39:02::","call_ids":["semeval2015.d001.s018.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"find%2:39:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1512,"cached_input_tokens":0,"output_tokens":81,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0038339999999999997,"input_uncached_usd":0.0030239999999999998,"input_cached_usd":0.0,"output_usd":0.0008100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.5939809170085937}
{"item_id":"semeval2015.d001.s018.t002","gold_sense_keys":["function%1:24:00::"],"candidates":[{"index":1,"sense_key":"function%1:24:00::","synset_id":"function.n.01"},{"index":2,"sense_key":"function%1:07:00::","synset_id":"function.n.02"},{"index":3,"sense_key":"function%1:04:00::","synset_id":"function.n.03"},{"index":4,"sense_key":"function%1:24:01::","synset_id":"function.n.04"},{"index":5,"sense_key":"function%1:14:00::","synset_id":"function.n.05"},{"index":6,"sense_key":"function%1:11:00::","synset_id":"affair.n.03"},{"index":7,"sense_key":"function%1:10:00::","synset_id":"routine.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"function%1:10:00::","call_ids":["semeval2015.d001.s018.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"function%1:10:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":826,"cached_input_tokens":0,"output_tokens":119,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002842,"input_uncached_usd":0.0016519999999999998,"input_cached_usd":0.0,"output_usd":0.00119,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7936834580032155}
{"item_id":"semeval2015.d001.s018.t003","gold_sense_keys":["check%2:32:10::"],"candidates":[{"index":1,"sense_key":"check%2:31:00::","synset_id":"check.v.01"},{"index":2,"sense_key":"check%2:41:04::","synset_id":"check.v.02"},{"index":3,"sense_key":"check%2:31:02::","synset_id":"see.v.10"},{"index":4,"sense_key":"check%2:41:08::","synset_id":"control.v.02"},{"index":5,"sense_key":"check%2:38:15::","synset_id":"check.v.05"},{"index":6,"sense_key":"check%2:31:01::","synset_id":"check.v.06"},{"index":7,"sense_key":"check%2:30:02::","synset_id":"check.v.07"},{"index":8,"sense_key":"check%2:42:09::","synset_id":"check.v.08"},{"index":9,"sense_key":"check%2:42:08::","synset_id":"match.v.01"},{"index":10,"sense_key":"check%2:41:14::","synset_id":"check.v.10"},{"index":11,"sense_key":"check%2:41:02::","synset_id":"discipline.v.01"},{"index":12,"sense_key":"check%2:40:13::","synset_id":"check.v.12"},{"index":13,"sense_key":"check%2:40:00::","synset_id":"check.v.13"},{"index":14,"sense_key":"check%2:38:14::","synset_id":"check.v.14"},{"index":15,"sense_key":"check%2:38:12::","synset_id":"check.v.15"},{"index":16,"sense_key":"check%2:36:00::","synset_id":"check.v.16"},{"index":17,"sense_key":"check%2:33:02::","synset_id":"check.v.17"},{"index":18,"sense_key":"check%2:33:06::","synset_id":"check.v.18"},{"index":19,"sense_key":"check%2:33:00::","synset_id":"check.v.19"},{"index":20,"sense_key":"check%2:32:07::","synset_id":"check.v.20"},{"index":21,"sense_key":"check%2:32:10::","synset_id":"determine.v.08"},{"index":22,"sense_key":"check%2:31:12::","synset_id":"check.v.22"},{"index":23,"sense_key":"check%2:30:10::","synset_id":"check.v.23"},{"index":24,"sense_key":"check%2:30:09::","synset_id":"check.v.24"},{"index":25,"sense_key":"check%2:30:03::","synset_id":"crack.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":21,"chosen_sense_key":"check%2:32:10::","call_ids":["semeval2015.d001.s018.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":21,"predicted_sense_key":"check%2:32:10::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1858,"cached_input_tokens":0,"output_tokens":169,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.005405999999999999,"input_uncached_usd":0.0037159999999999997,"input_cached_usd":0.0,"output_usd":0.00169,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2065377079416066}
{"item_id":"semeval2015.d001.s018.t004","gold_sense_keys":["operator%1:24:00::"],"candidates":[{"index":1,"sense_key":"operator%1:24:00::","synset_id":"operator.n.01"},{"index":2,"sense_key":"operator%1:18:00::","synset_id":"operator.n.02"},{"index":3,"sense_key":"operator%1:18:01::","synset_id":"operator.n.03"},{"index":4,"sense_key":"operator%1:18:02::","synset_id":"hustler.n.02"},{"index":5,"sense_key":"operator%1:18:03::","synset_id":"operator.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"operator%1:24:00::","call_ids":["semeval2015.d001.s018.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"operator%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":618,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001356,"input_uncached_usd":0.001236,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":13.46825916704256}
{"item_id":"semeval2015.d001.s019.t000","gold_sense_keys":["treat%2:31:00::"],"candidates":[{"index":1,"sense_key":"treat%2:41:00::","synset_id":"treat.v.01"},{"index":2,"sense_key":"treat%2:30:01::","synset_id":"process.v.01"},{"index":3,"sense_key":"treat%2:29:00::","synset_id":"treat.v.03"},{"index":4,"sense_key":"treat%2:32:00::","synset_id":"cover.v.05"},{"index":5,"sense_key":"treat%2:40:00::","synset_id":"treat.v.05"},{"index":6,"sense_key":"treat%2:34:00::","synset_id":"regale.v.01"},{"index":7,"sense_key":"treat%2:32:03::","synset_id":"treat.v.07"},{"index":8,"sense_key":"treat%2:31:00::","synset_id":"treat.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"treat%2:31:00::","call_ids":["semeval2015.d001.s019.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"treat%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":886,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001892,"input_uncached_usd":0.001772,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9225402500014752}
{"item_id":"semeval2015.d001.s019.t001","gold_sense_keys":["operator%1:24:00::"],"candidates":[{"index":1,"sense_key":"operator%1:24:00::","synset_id":"operator.n.01"},{"index":2,"sense_key":"operator%1:18:00::","synset_id":"operator.n.02"},{"index":3,"sense_key":"operator%1:18:01::","synset_id":"operator.n.03"},{"index":4,"sense_key":"operator%1:18:02::","synset_id":"hustler.n.02"},{"index":5,"sense_key":"operator%1:18:03::","synset_id":"operator.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"operator%1:24:00::","call_ids":["semeval2015.d001.s019.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"operator%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":618,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001356,"input_uncached_usd":0.001236,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8475169580196962}
{"item_id":"semeval2015.d001.s019.t003","gold_sense_keys":["treat%2:31:00::"],"candidates":[{"index":1,"sense_key":"treat%2:41:00::","synset_id":"treat.v.01"},{"index":2,"sense_key":"treat%2:30:01::","synset_id":"process.v.01"},{"index":3,"sense_key":"treat%2:29:00::","synset_id":"treat.v.03"},{"index":4,"sense_key":"treat%2:32:00::","synset_id":"cover.v.05"},{"index":5,"sense_key":"treat%2:40:00::","synset_id":"treat.v.05"},{"index":6,"sense_key":"treat%2:34:00::","synset_id":"regale.v.01"},{"index":7,"sense_key":"treat%2:32:03::","synset_id":"treat.v.07"},{"index":8,"sense_key":"treat%2:31:00::","synset_id":"treat.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"treat%2:31:00::","call_ids":["semeval2015.d001.s019.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"treat%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":886,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002412,"input_uncached_usd":0.001772,"input_cached_usd":0.0,"output_usd":0.00064,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8000370829831809}
{"item_id":"semeval2015.d001.s019.t004","gold_sense_keys":["user%1:18:00::"],"candidates":[{"index":1,"sense_key":"user%1:18:00::","synset_id":"user.n.01"},{"index":2,"sense_key":"user%1:18:01::","synset_id":"exploiter.n.01"},{"index":3,"sense_key":"user%1:18:02::","synset_id":"drug_user.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"user%1:18:00::","call_ids":["semeval2015.d001.s019.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"user%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":493,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001106,"input_uncached_usd":0.000986,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":10.512818209012039}
{"item_id":"semeval2015.d001.s019.t005","gold_sense_keys":["function%1:24:00::"],"candidates":[{"index":1,"sense_key":"function%1:24:00::","synset_id":"function.n.01"},{"index":2,"sense_key":"function%1:07:00::","synset_id":"function.n.02"},{"index":3,"sense_key":"function%1:04:00::","synset_id":"function.n.03"},{"index":4,"sense_key":"function%1:24:01::","synset_id":"function.n.04"},{"index":5,"sense_key":"function%1:14:00::","synset_id":"function.n.05"},{"index":6,"sense_key":"function%1:11:00::","synset_id":"affair.n.03"},{"index":7,"sense_key":"function%1:10:00::","synset_id":"routine.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"function%1:10:00::","call_ids":["semeval2015.d001.s019.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"function%1:10:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":826,"cached_input_tokens":0,"output_tokens":68,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002332,"input_uncached_usd":0.0016519999999999998,"input_cached_usd":0.0,"output_usd":0.00068,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":14.60185979201924}
{"item_id":"semeval2015.d001.s020.t000","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":2,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":3,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":4,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":5,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":6,"sense_key":"use%2:41:03::","synset_id":"use.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%2:34:01::","call_ids":["semeval2015.d001.s020.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":742,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001604,"input_uncached_usd":0.0014839999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":13.904256000067107}
{"item_id":"semeval2015.d001.s020.t001","gold_sense_keys":["define%2:32:01::"],"candidates":[{"index":1,"sense_key":"define%2:42:00::","synset_id":"specify.v.03"},{"index":2,"sense_key":"define%2:32:00::","synset_id":"define.v.02"},{"index":3,"sense_key":"define%2:42:01::","synset_id":"define.v.03"},{"index":4,"sense_key":"define%2:42:03::","synset_id":"define.v.04"},{"index":5,"sense_key":"define%2:32:01::","synset_id":"specify.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"define%2:32:01::","call_ids":["semeval2015.d001.s020.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"define%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":618,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001906,"input_uncached_usd":0.001236,"input_cached_usd":0.0,"output_usd":0.00067,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5576997919706628}
{"item_id":"semeval2015.d001.s020.t002","gold_sense_keys":["value%1:09:00::"],"candidates":[{"index":1,"sense_key":"value%1:09:00::","synset_id":"value.n.01"},{"index":2,"sense_key":"value%1:07:00::","synset_id":"value.n.02"},{"index":3,"sense_key":"value%1:21:00::","synset_id":"value.n.03"},{"index":4,"sense_key":"value%1:07:01::","synset_id":"value.n.04"},{"index":5,"sense_key":"value%1:28:00::","synset_id":"value.n.05"},{"index":6,"sense_key":"value%1:09:01::","synset_id":"value.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"value%1:09:00::","call_ids":["semeval2015.d001.s020.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"value%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":728,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001576,"input_uncached_usd":0.0014559999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.551413874956779}
{"item_id":"semeval2015.d001.s022.t000","gold_sense_keys":["way%1:04:01::","way%1:07:01::"],"candidates":[{"index":1,"sense_key":"way%1:07:01::","synset_id":"manner.n.01"},{"index":2,"sense_key":"way%1:04:01::","synset_id":"means.n.01"},{"index":3,"sense_key":"way%1:15:00::","synset_id":"direction.n.01"},{"index":4,"sense_key":"way%1:26:00::","synset_id":"way.n.04"},{"index":5,"sense_key":"way%1:04:00::","synset_id":"way.n.05"},{"index":6,"sense_key":"way%1:06:00::","synset_id":"way.n.06"},{"index":7,"sense_key":"way%1:04:02::","synset_id":"way.n.07"},{"index":8,"sense_key":"way%1:23:00::","synset_id":"room.n.02"},{"index":9,"sense_key":"way%1:07:02::","synset_id":"way.n.09"},{"index":10,"sense_key":"way%1:09:00::","synset_id":"way.n.10"},{"index":11,"sense_key":"way%1:09:01::","synset_id":"way.n.11"},{"index":12,"sense_key":"way%1:21:00::","synset_id":"way.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"way%1:04:01::","call_ids":["semeval2015.d001.s022.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"way%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1008,"cached_input_tokens":0,"output_tokens":197,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003986,"input_uncached_usd":0.002016,"input_cached_usd":0.0,"output_usd":0.00197,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.0032934169285}
{"item_id":"semeval2015.d001.s022.t001","gold_sense_keys":["define%2:32:01::"],"candidates":[{"index":1,"sense_key":"define%2:42:00::","synset_id":"specify.v.03"},{"index":2,"sense_key":"define%2:32:00::","synset_id":"define.v.02"},{"index":3,"sense_key":"define%2:42:01::","synset_id":"define.v.03"},{"index":4,"sense_key":"define%2:42:03::","synset_id":"define.v.04"},{"index":5,"sense_key":"define%2:32:01::","synset_id":"specify.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"define%2:32:01::","call_ids":["semeval2015.d001.s022.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"define%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":652,"cached_input_tokens":0,"output_tokens":87,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002174,"input_uncached_usd":0.001304,"input_cached_usd":0.0,"output_usd":0.0008700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9264342909445986}
{"item_id":"semeval2015.d001.s022.t002","gold_sense_keys":["conditional%3:00:00::"],"candidates":[{"index":1,"sense_key":"conditional%5:00:00:qualified:02","synset_id":"conditional.s.01"},{"index":2,"sense_key":"conditional%3:00:00::","synset_id":"conditional.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"conditional%3:00:00::","call_ids":["semeval2015.d001.s022.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"conditional%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":486,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001692,"input_uncached_usd":0.000972,"input_cached_usd":0.0,"output_usd":0.00072,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.9809374168980867}
{"item_id":"semeval2015.d001.s022.t003","gold_sense_keys":["operation%1:04:01::"],"candidates":[{"index":1,"sense_key":"operation%1:26:00::","synset_id":"operation.n.01"},{"index":2,"sense_key":"operation%1:04:06::","synset_id":"operation.n.02"},{"index":3,"sense_key":"operation%1:04:03::","synset_id":"operation.n.03"},{"index":4,"sense_key":"operation%1:22:00::","synset_id":"operation.n.04"},{"index":5,"sense_key":"operation%1:04:02::","synset_id":"operation.n.05"},{"index":6,"sense_key":"operation%1:04:00::","synset_id":"operation.n.06"},{"index":7,"sense_key":"operation%1:04:05::","synset_id":"operation.n.07"},{"index":8,"sense_key":"operation%1:22:01::","synset_id":"operation.n.08"},{"index":9,"sense_key":"operation%1:09:00::","synset_id":"process.n.02"},{"index":10,"sense_key":"operation%1:04:01::","synset_id":"mathematical_process.n.01"},{"index":11,"sense_key":"operation%1:04:07::","synset_id":"operation.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"operation%1:22:00::","call_ids":["semeval2015.d001.s022.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"operation%1:22:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1233,"cached_input_tokens":0,"output_tokens":125,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0037159999999999997,"input_uncached_usd":0.002466,"input_cached_usd":0.0,"output_usd":0.00125,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":15.367457290994935}
{"item_id":"semeval2015.d001.s023.t000","gold_sense_keys":["introduce%2:30:03::","introduce%2:35:00::"],"candidates":[{"index":1,"sense_key":"introduce%2:32:00::","synset_id":"introduce.v.01"},{"index":2,"sense_key":"introduce%2:36:00::","synset_id":"introduce.v.02"},{"index":3,"sense_key":"introduce%2:30:03::","synset_id":"insert.v.02"},{"index":4,"sense_key":"introduce%2:38:00::","synset_id":"bring_in.v.01"},{"index":5,"sense_key":"introduce%2:36:01::","synset_id":"introduce.v.05"},{"index":6,"sense_key":"introduce%2:35:00::","synset_id":"insert.v.01"},{"index":7,"sense_key":"introduce%2:32:02::","synset_id":"introduce.v.07"},{"index":8,"sense_key":"introduce%2:31:00::","synset_id":"introduce.v.08"},{"index":9,"sense_key":"introduce%2:32:01::","synset_id":"precede.v.05"},{"index":10,"sense_key":"introduce%2:30:00::","synset_id":"inaugurate.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"introduce%2:35:00::","call_ids":["semeval2015.d001.s023.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"introduce%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1009,"cached_input_tokens":0,"output_tokens":180,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003818,"input_uncached_usd":0.002018,"input_cached_usd":0.0,"output_usd":0.0018000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.4548303749179468}
{"item_id":"semeval2015.d001.s023.t002","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":2,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":3,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":4,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":5,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":6,"sense_key":"use%2:41:03::","synset_id":"use.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%2:34:01::","call_ids":["semeval2015.d001.s023.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":770,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00166,"input_uncached_usd":0.00154,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1849657910643145}
{"item_id":"semeval2015.d001.s023.t005","gold_sense_keys":["true%3:00:00::"],"candidates":[{"index":1,"sense_key":"true%3:00:00::","synset_id":"true.a.01"},{"index":2,"sense_key":"true%5:00:00:accurate:00","synset_id":"true.s.01"},{"index":3,"sense_key":"true%5:00:02:faithful:00","synset_id":"true.s.02"},{"index":4,"sense_key":"true%3:00:04::","synset_id":"truthful.a.01"},{"index":5,"sense_key":"true%5:00:00:typical:00","synset_id":"true.s.03"},{"index":6,"sense_key":"true%5:00:00:trustworthy:00","synset_id":"dependable.s.01"},{"index":7,"sense_key":"true%5:00:00:sincere:00","synset_id":"genuine.s.01"},{"index":8,"sense_key":"true%5:00:00:real:02","synset_id":"true.s.06"},{"index":9,"sense_key":"true%5:00:00:geographic:00","synset_id":"true.s.07"},{"index":10,"sense_key":"true%5:00:00:legitimate:00","synset_id":"true.s.08"},{"index":11,"sense_key":"true%5:00:00:harmonious:00","synset_id":"on-key.s.01"},{"index":12,"sense_key":"true%5:00:00:even:01","synset_id":"true.s.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"true%3:00:00::","call_ids":["semeval2015.d001.s023.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"true%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1148,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0024159999999999997,"input_uncached_usd":0.002296,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1459574999753386}
{"item_id":"semeval2015.d001.s023.t006","gold_sense_keys":["find%2:32:00::","find%2:39:02::"],"candidates":[{"index":1,"sense_key":"find%2:40:02::","synset_id":"find.v.01"},{"index":2,"sense_key":"find%2:39:02::","synset_id":"detect.v.01"},{"index":3,"sense_key":"find%2:40:00::","synset_id":"find.v.03"},{"index":4,"sense_key":"find%2:32:00::","synset_id":"determine.v.01"},{"index":5,"sense_key":"find%2:31:10::","synset_id":"find.v.05"},{"index":6,"sense_key":"find%2:39:01::","synset_id":"witness.v.02"},{"index":7,"sense_key":"find%2:40:01::","synset_id":"line_up.v.02"},{"index":8,"sense_key":"find%2:36:00::","synset_id":"discover.v.03"},{"index":9,"sense_key":"find%2:31:09::","synset_id":"discover.v.04"},{"index":10,"sense_key":"find%2:40:03::","synset_id":"find.v.10"},{"index":11,"sense_key":"find%2:32:01::","synset_id":"rule.v.04"},{"index":12,"sense_key":"find%2:30:00::","synset_id":"receive.v.02"},{"index":13,"sense_key":"find%2:39:05::","synset_id":"find.v.13"},{"index":14,"sense_key":"find%2:40:15::","synset_id":"recover.v.01"},{"index":15,"sense_key":"find%2:38:10::","synset_id":"find.v.15"},{"index":16,"sense_key":"find%2:30:13::","synset_id":"find_oneself.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"find%2:39:02::","call_ids":["semeval2015.d001.s023.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"find%2:39:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1535,"cached_input_tokens":0,"output_tokens":132,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00439,"input_uncached_usd":0.00307,"input_cached_usd":0.0,"output_usd":0.0013200000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0634195839520544}
{"item_id":"semeval2015.d001.s023.t010","gold_sense_keys":["instance%1:11:00::"],"candidates":[{"index":1,"sense_key":"instance%1:11:00::","synset_id":"case.n.01"},{"index":2,"sense_key":"instance%1:09:00::","synset_id":"example.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"instance%1:11:00::","call_ids":["semeval2015.d001.s023.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"instance%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":525,"cached_input_tokens":0,"output_tokens":86,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00191,"input_uncached_usd":0.00105,"input_cached_usd":0.0,"output_usd":0.0008600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.61590941704344}
{"item_id":"semeval2015.d001.s025.t000","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":2,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":3,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":4,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":5,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":6,"sense_key":"use%2:41:03::","synset_id":"use.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%2:34:01::","call_ids":["semeval2015.d001.s025.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":739,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001598,"input_uncached_usd":0.001478,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4932309169089422}
{"item_id":"semeval2015.d001.s025.t001","gold_sense_keys":["define%2:32:01::"],"candidates":[{"index":1,"sense_key":"define%2:42:00::","synset_id":"specify.v.03"},{"index":2,"sense_key":"define%2:32:00::","synset_id":"define.v.02"},{"index":3,"sense_key":"define%2:42:01::","synset_id":"define.v.03"},{"index":4,"sense_key":"define%2:42:03::","synset_id":"define.v.04"},{"index":5,"sense_key":"define%2:32:01::","synset_id":"specify.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"define%2:32:01::","call_ids":["semeval2015.d001.s025.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"define%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":615,"cached_input_tokens":0,"output_tokens":101,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00224,"input_uncached_usd":0.00123,"input_cached_usd":0.0,"output_usd":0.00101,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7630130420438945}
{"item_id":"semeval2015.d001.s026.t001","gold_sense_keys":["useful%3:00:00::"],"candidates":[{"index":1,"sense_key":"useful%3:00:00::","synset_id":"useful.a.01"},{"index":2,"sense_key":"useful%5:00:00:functional:00","synset_id":"utilitarian.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"useful%3:00:00::","call_ids":["semeval2015.d001.s026.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"useful%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":470,"cached_input_tokens":0,"output_tokens":95,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0018900000000000002,"input_uncached_usd":0.00094,"input_cached_usd":0.0,"output_usd":0.0009500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.416645957971923}
{"item_id":"semeval2015.d001.s026.t002","gold_sense_keys":["work%2:41:02::"],"candidates":[{"index":1,"sense_key":"work%2:41:02::","synset_id":"work.v.01"},{"index":2,"sense_key":"work%2:41:00::","synset_id":"work.v.02"},{"index":3,"sense_key":"work%2:41:03::","synset_id":"work.v.03"},{"index":4,"sense_key":"work%2:35:02::","synset_id":"function.v.01"},{"index":5,"sense_key":"work%2:36:00::","synset_id":"work.v.05"},{"index":6,"sense_key":"work%2:29:00::","synset_id":"exercise.v.03"},{"index":7,"sense_key":"work%2:38:00::","synset_id":"make.v.36"},{"index":8,"sense_key":"work%2:41:04::","synset_id":"work.v.08"},{"index":9,"sense_key":"work%2:41:05::","synset_id":"work.v.09"},{"index":10,"sense_key":"work%2:38:02::","synset_id":"work.v.10"},{"index":11,"sense_key":"work%2:36:01::","synset_id":"bring.v.03"},{"index":12,"sense_key":"work%2:41:01::","synset_id":"work.v.12"},{"index":13,"sense_key":"work%2:36:09::","synset_id":"cultivate.v.02"},{"index":14,"sense_key":"work%2:42:10::","synset_id":"work.v.14"},{"index":15,"sense_key":"work%2:41:10::","synset_id":"influence.v.01"},{"index":16,"sense_key":"work%2:41:11::","synset_id":"work.v.16"},{"index":17,"sense_key":"work%2:41:12::","synset_id":"work.v.17"},{"index":18,"sense_key":"work%2:39:10::","synset_id":"work.v.18"},{"index":19,"sense_key":"work%2:37:10::","synset_id":"work.v.19"},{"index":20,"sense_key":"work%2:36:12::","synset_id":"shape.v.02"},{"index":21,"sense_key":"work%2:35:14::","synset_id":"work.v.21"},{"index":22,"sense_key":"work%2:35:00::","synset_id":"knead.v.01"},{"index":23,"sense_key":"work%2:34:10::","synset_id":"exploit.v.01"},{"index":24,"sense_key":"work%2:31:13::","synset_id":"solve.v.01"},{"index":25,"sense_key":"work%2:30:14::","synset_id":"ferment.v.03"},{"index":26,"sense_key":"work%2:30:13::","synset_id":"sour.v.01"},{"index":27,"sense_key":"work%2:30:10::","synset_id":"work.v.27"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"work%2:41:02::","call_ids":["semeval2015.d001.s026.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"work%2:41:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1998,"cached_input_tokens":0,"output_tokens":190,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.005896,"input_uncached_usd":0.0039959999999999996,"input_cached_usd":0.0,"output_usd":0.0019000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.48969437496271}
{"item_id":"semeval2015.d001.s027.t000","gold_sense_keys":["now%4:02:07::"],"candidates":[{"index":1,"sense_key":"now%4:02:05::","synset_id":"now.r.01"},{"index":2,"sense_key":"now%4:02:00::","synset_id":"nowadays.r.01"},{"index":3,"sense_key":"now%4:02:02::","synset_id":"now.r.03"},{"index":4,"sense_key":"now%4:02:04::","synset_id":"now.r.04"},{"index":5,"sense_key":"now%4:02:01::","synset_id":"immediately.r.01"},{"index":6,"sense_key":"now%4:02:07::","synset_id":"now.r.06"},{"index":7,"sense_key":"now%4:02:06::","synset_id":"now.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"now%4:02:07::","call_ids":["semeval2015.d001.s027.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"now%4:02:07::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":828,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001776,"input_uncached_usd":0.001656,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1942299579968676}
{"item_id":"semeval2015.d001.s027.t001","gold_sense_keys":["ask%2:32:04::"],"candidates":[{"index":1,"sense_key":"ask%2:32:00::","synset_id":"ask.v.01"},{"index":2,"sense_key":"ask%2:32:01::","synset_id":"ask.v.02"},{"index":3,"sense_key":"ask%2:32:02::","synset_id":"ask.v.03"},{"index":4,"sense_key":"ask%2:32:05::","synset_id":"ask.v.04"},{"index":5,"sense_key":"ask%2:32:04::","synset_id":"ask.v.05"},{"index":6,"sense_key":"ask%2:42:00::","synset_id":"necessitate.v.01"},{"index":7,"sense_key":"ask%2:32:09::","synset_id":"ask.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"ask%2:32:04::","call_ids":["semeval2015.d001.s027.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"ask%2:32:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":822,"cached_input_tokens":0,"output_tokens":174,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003384,"input_uncached_usd":0.0016439999999999998,"input_cached_usd":0.0,"output_usd":0.0017400000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.136242499924265}
{"item_id":"semeval2015.d001.s027.t002","gold_sense_keys":["user%1:18:00::"],"candidates":[{"index":1,"sense_key":"user%1:18:00::","synset_id":"user.n.01"},{"index":2,"sense_key":"user%1:18:01::","synset_id":"exploiter.n.01"},{"index":3,"sense_key":"user%1:18:02::","synset_id":"drug_user.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"user%1:18:00::","call_ids":["semeval2015.d001.s027.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"user%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":520,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00116,"input_uncached_usd":0.00104,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.005778916995041}
{"item_id":"semeval2015.d001.s028.t000","gold_sense_keys":["easy%3:00:01::"],"candidates":[{"index":1,"sense_key":"easy%3:00:01::","synset_id":"easy.a.01"},{"index":2,"sense_key":"easy%5:00:00:unhurried:00","synset_id":"easy.s.01"},{"index":3,"sense_key":"easy%3:00:02::","synset_id":"easy.a.03"},{"index":4,"sense_key":"easy%5:00:00:pleasing:00","synset_id":"easy.s.02"},{"index":5,"sense_key":"easy%5:00:00:light:08","synset_id":"easy.s.03"},{"index":6,"sense_key":"easy%5:00:00:impressionable:00","synset_id":"easy.s.04"},{"index":7,"sense_key":"easy%5:00:00:rich:00","synset_id":"comfortable.s.03"},{"index":8,"sense_key":"easy%5:00:00:gradual:02","synset_id":"easy.s.06"},{"index":9,"sense_key":"easy%5:00:00:comfortable:00","synset_id":"easy.s.07"},{"index":10,"sense_key":"easy%5:00:00:unchaste:00","synset_id":"easy.s.08"},{"index":11,"sense_key":"easy%5:00:00:available:00","synset_id":"easy.s.09"},{"index":12,"sense_key":"easy%5:00:00:abundant:00","synset_id":"easy.s.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"easy%3:00:01::","call_ids":["semeval2015.d001.s028.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"easy%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1137,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002394,"input_uncached_usd":0.002274,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4918109589489177}
{"item_id":"semeval2015.d001.s028.t002","gold_sense_keys":["function%1:24:00::"],"candidates":[{"index":1,"sense_key":"function%1:24:00::","synset_id":"function.n.01"},{"index":2,"sense_key":"function%1:07:00::","synset_id":"function.n.02"},{"index":3,"sense_key":"function%1:04:00::","synset_id":"function.n.03"},{"index":4,"sense_key":"function%1:24:01::","synset_id":"function.n.04"},{"index":5,"sense_key":"function%1:14:00::","synset_id":"function.n.05"},{"index":6,"sense_key":"function%1:11:00::","synset_id":"affair.n.03"},{"index":7,"sense_key":"function%1:10:00::","synset_id":"routine.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"function%1:24:00::","call_ids":["semeval2015.d001.s028.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"function%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":842,"cached_input_tokens":0,"output_tokens":111,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002794,"input_uncached_usd":0.001684,"input_cached_usd":0.0,"output_usd":0.00111,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.125674624927342}
{"item_id":"semeval2015.d001.s028.t003","gold_sense_keys":["cos%1:24:00::"],"candidates":[{"index":1,"sense_key":"cos%1:24:00::","synset_id":"cosine.n.01"},{"index":2,"sense_key":"cos%1:13:00::","synset_id":"cos.n.02"},{"index":3,"sense_key":"carbon_monoxide%1:27:00::","synset_id":"carbon_monoxide.n.01"},{"index":4,"sense_key":"cobalt%1:27:00::","synset_id":"cobalt.n.01"},{"index":5,"sense_key":"conscientious_objector%1:18:00::","synset_id":"conscientious_objector.n.01"},{"index":6,"sense_key":"colorado%1:15:00::","synset_id":"colorado.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cos%1:24:00::","call_ids":["semeval2015.d001.s028.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cos%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":796,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001712,"input_uncached_usd":0.0015919999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9742687499383464}
{"item_id":"semeval2015.d001.s028.t004","gold_sense_keys":["sin%1:24:00::"],"candidates":[{"index":1,"sense_key":"sin%1:07:00::","synset_id":"sin.n.01"},{"index":2,"sense_key":"sin%1:04:00::","synset_id":"sin.n.02"},{"index":3,"sense_key":"sin%1:24:00::","synset_id":"sine.n.01"},{"index":4,"sense_key":"sin%1:18:00::","synset_id":"sin.n.04"},{"index":5,"sense_key":"sin%1:10:00::","synset_id":"sin.n.05"},{"index":6,"sense_key":"sin%1:04:01::","synset_id":"sin.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"sin%1:24:00::","call_ids":["semeval2015.d001.s028.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"sin%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":677,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001474,"input_uncached_usd":0.001354,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5774122499860823}
{"item_id":"semeval2015.d001.s028.t005","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":2,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":3,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"other%3:00:00::","call_ids":["semeval2015.d001.s028.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":610,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00134,"input_uncached_usd":0.00122,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6483118330361322}
{"item_id":"semeval2015.d001.s028.t008","gold_sense_keys":["sum%1:09:01::"],"candidates":[{"index":1,"sense_key":"sum%1:21:00::","synset_id":"sum.n.01"},{"index":2,"sense_key":"sum%1:09:01::","synset_id":"sum.n.02"},{"index":3,"sense_key":"sum%1:14:00::","synset_id":"sum.n.03"},{"index":4,"sense_key":"sum%1:09:00::","synset_id":"kernel.n.03"},{"index":5,"sense_key":"sum%1:06:00::","synset_id":"sum.n.05"},{"index":6,"sense_key":"sum%1:14:01::","synset_id":"union.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"sum%1:09:01::","call_ids":["semeval2015.d001.s028.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"sum%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":764,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0022979999999999997,"input_uncached_usd":0.0015279999999999998,"input_cached_usd":0.0,"output_usd":0.0007700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.780255958903581}
{"item_id":"semeval2015.d001.s028.t009","gold_sense_keys":["product%1:09:00::"],"candidates":[{"index":1,"sense_key":"product%1:06:01::","synset_id":"merchandise.n.01"},{"index":2,"sense_key":"product%1:06:00::","synset_id":"product.n.02"},{"index":3,"sense_key":"product%1:09:00::","synset_id":"product.n.03"},{"index":4,"sense_key":"product%1:27:00::","synset_id":"product.n.04"},{"index":5,"sense_key":"product%1:19:00::","synset_id":"product.n.05"},{"index":6,"sense_key":"product%1:14:00::","synset_id":"intersection.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"product%1:09:00::","call_ids":["semeval2015.d001.s028.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"product%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":773,"cached_input_tokens":0,"output_tokens":76,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002306,"input_uncached_usd":0.0015459999999999998,"input_cached_usd":0.0,"output_usd":0.00076,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5326402080245316}
{"item_id":"semeval2015.d001.s030.t000","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":2,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":3,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":4,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":5,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":6,"sense_key":"use%2:41:03::","synset_id":"use.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%2:34:01::","call_ids":["semeval2015.d001.s030.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":744,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001608,"input_uncached_usd":0.001488,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":14.979173665982671}
{"item_id":"semeval2015.d001.s030.t001","gold_sense_keys":["plus%1:04:00::"],"candidates":[{"index":1,"sense_key":"plus%1:07:00::","synset_id":"asset.n.01"},{"index":2,"sense_key":"plus%1:04:00::","synset_id":"summation.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"plus%1:04:00::","call_ids":["semeval2015.d001.s030.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"plus%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":481,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001082,"input_uncached_usd":0.000962,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0156338749220595}
{"item_id":"semeval2015.d001.s030.t002","gold_sense_keys":["times%1:04:00::"],"candidates":[{"index":1,"sense_key":"times%1:28:01::","synset_id":"times.n.01"},{"index":2,"sense_key":"times%1:04:00::","synset_id":"multiplication.n.03"},{"index":3,"sense_key":"time%1:11:00::","synset_id":"time.n.01"},{"index":4,"sense_key":"time%1:28:05::","synset_id":"time.n.02"},{"index":5,"sense_key":"time%1:28:00::","synset_id":"time.n.03"},{"index":6,"sense_key":"time%1:28:06::","synset_id":"time.n.04"},{"index":7,"sense_key":"time%1:03:00::","synset_id":"time.n.05"},{"index":8,"sense_key":"time%1:11:01::","synset_id":"time.n.06"},{"index":9,"sense_key":"clock_time%1:28:00::","synset_id":"clock_time.n.01"},{"index":10,"sense_key":"fourth_dimension%1:28:00::","synset_id":"fourth_dimension.n.01"},{"index":11,"sense_key":"meter%1:07:00::","synset_id":"meter.n.04"},{"index":12,"sense_key":"prison_term%1:28:00::","synset_id":"prison_term.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"times%1:04:00::","call_ids":["semeval2015.d001.s030.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"times%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1119,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002828,"input_uncached_usd":0.002238,"input_cached_usd":0.0,"output_usd":0.00059,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0311734169954434}
{"item_id":"semeval2015.d001.s030.t003","gold_sense_keys":["have%2:40:00::","have%2:42:00::"],"candidates":[{"index":1,"sense_key":"have%2:40:00::","synset_id":"have.v.01"},{"index":2,"sense_key":"have%2:42:00::","synset_id":"have.v.02"},{"index":3,"sense_key":"have%2:39:06::","synset_id":"experience.v.03"},{"index":4,"sense_key":"have%2:40:04::","synset_id":"own.v.01"},{"index":5,"sense_key":"have%2:30:00::","synset_id":"get.v.03"},{"index":6,"sense_key":"have%2:34:00::","synset_id":"consume.v.02"},{"index":7,"sense_key":"have%2:41:00::","synset_id":"have.v.07"},{"index":8,"sense_key":"have%2:36:00::","synset_id":"hold.v.03"},{"index":9,"sense_key":"have%2:40:01::","synset_id":"have.v.09"},{"index":10,"sense_key":"have%2:42:12::","synset_id":"have.v.10"},{"index":11,"sense_key":"have%2:30:01::","synset_id":"have.v.11"},{"index":12,"sense_key":"have%2:29:02::","synset_id":"have.v.12"},{"index":13,"sense_key":"have%2:32:00::","synset_id":"induce.v.02"},{"index":14,"sense_key":"have%2:40:05::","synset_id":"accept.v.02"},{"index":15,"sense_key":"have%2:40:03::","synset_id":"receive.v.01"},{"index":16,"sense_key":"have%2:29:05::","synset_id":"suffer.v.02"},{"index":17,"sense_key":"have%2:40:02::","synset_id":"have.v.17"},{"index":18,"sense_key":"have%2:29:00::","synset_id":"give_birth.v.01"},{"index":19,"sense_key":"have%2:35:00::","synset_id":"take.v.35"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"have%2:40:00::","call_ids":["semeval2015.d001.s030.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"have%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1526,"cached_input_tokens":0,"output_tokens":125,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004302,"input_uncached_usd":0.003052,"input_cached_usd":0.0,"output_usd":0.00125,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.26124699995853}
{"item_id":"semeval2015.d001.s030.t004","gold_sense_keys":["operator%1:24:00::"],"candidates":[{"index":1,"sense_key":"operator%1:24:00::","synset_id":"operator.n.01"},{"index":2,"sense_key":"operator%1:18:00::","synset_id":"operator.n.02"},{"index":3,"sense_key":"operator%1:18:01::","synset_id":"operator.n.03"},{"index":4,"sense_key":"operator%1:18:02::","synset_id":"hustler.n.02"},{"index":5,"sense_key":"operator%1:18:03::","synset_id":"operator.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"operator%1:24:00::","call_ids":["semeval2015.d001.s030.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"operator%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":615,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00135,"input_uncached_usd":0.00123,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0968892920063809}
{"item_id":"semeval2015.d001.s031.t000","gold_sense_keys":["useful%3:00:00::"],"candidates":[{"index":1,"sense_key":"useful%3:00:00::","synset_id":"useful.a.01"},{"index":2,"sense_key":"useful%5:00:00:functional:00","synset_id":"utilitarian.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"useful%3:00:00::","call_ids":["semeval2015.d001.s031.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"useful%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":472,"cached_input_tokens":0,"output_tokens":81,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001754,"input_uncached_usd":0.000944,"input_cached_usd":0.0,"output_usd":0.0008100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":14.421608167001978}
{"item_id":"semeval2015.d001.s032.t000","gold_sense_keys":["there%4:02:00::"],"candidates":[{"index":1,"sense_key":"there%4:02:00::","synset_id":"there.r.01"},{"index":2,"sense_key":"there%4:02:02::","synset_id":"there.r.02"},{"index":3,"sense_key":"there%4:02:01::","synset_id":"there.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"there%4:02:00::","call_ids":["semeval2015.d001.s032.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"there%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":533,"cached_input_tokens":0,"output_tokens":57,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0016359999999999999,"input_uncached_usd":0.0010659999999999999,"input_cached_usd":0.0,"output_usd":0.0005700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4261783750262111}
{"item_id":"semeval2015.d001.s032.t001","gold_sense_keys":["have%2:42:12::"],"candidates":[{"index":1,"sense_key":"have%2:40:00::","synset_id":"have.v.01"},{"index":2,"sense_key":"have%2:42:00::","synset_id":"have.v.02"},{"index":3,"sense_key":"have%2:39:06::","synset_id":"experience.v.03"},{"index":4,"sense_key":"have%2:40:04::","synset_id":"own.v.01"},{"index":5,"sense_key":"have%2:30:00::","synset_id":"get.v.03"},{"index":6,"sense_key":"have%2:34:00::","synset_id":"consume.v.02"},{"index":7,"sense_key":"have%2:41:00::","synset_id":"have.v.07"},{"index":8,"sense_key":"have%2:36:00::","synset_id":"hold.v.03"},{"index":9,"sense_key":"have%2:40:01::","synset_id":"have.v.09"},{"index":10,"sense_key":"have%2:42:12::","synset_id":"have.v.10"},{"index":11,"sense_key":"have%2:30:01::","synset_id":"have.v.11"},{"index":12,"sense_key":"have%2:29:02::","synset_id":"have.v.12"},{"index":13,"sense_key":"have%2:32:00::","synset_id":"induce.v.02"},{"index":14,"sense_key":"have%2:40:05::","synset_id":"accept.v.02"},{"index":15,"sense_key":"have%2:40:03::","synset_id":"receive.v.01"},{"index":16,"sense_key":"have%2:29:05::","synset_id":"suffer.v.02"},{"index":17,"sense_key":"have%2:40:02::","synset_id":"have.v.17"},{"index":18,"sense_key":"have%2:29:00::","synset_id":"give_birth.v.01"},{"index":19,"sense_key":"have%2:35:00::","synset_id":"take.v.35"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"have%2:40:00::","call_ids":["semeval2015.d001.s032.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"have%2:40:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1546,"cached_input_tokens":0,"output_tokens":226,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.005352,"input_uncached_usd":0.0030919999999999997,"input_cached_usd":0.0,"output_usd":0.0022600000000000003,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.7845935419900343}
{"item_id":"semeval2015.d001.s032.t002","gold_sense_keys":["list%1:10:00::"],"candidates":[{"index":1,"sense_key":"list%1:10:00::","synset_id":"list.n.01"},{"index":2,"sense_key":"list%1:07:00::","synset_id":"tilt.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"list%1:10:00::","call_ids":["semeval2015.d001.s032.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"list%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":511,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001142,"input_uncached_usd":0.001022,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.197962166974321}
{"item_id":"semeval2015.d001.s032.t003","gold_sense_keys":["variable%1:10:00::"],"candidates":[{"index":1,"sense_key":"variable%1:17:00::","synset_id":"variable.n.01"},{"index":2,"sense_key":"variable%1:09:00::","synset_id":"variable.n.02"},{"index":3,"sense_key":"variable%1:17:01::","synset_id":"variable_star.n.01"},{"index":4,"sense_key":"variable%1:10:00::","synset_id":"variable.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"variable%1:10:00::","call_ids":["semeval2015.d001.s032.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"variable%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":579,"cached_input_tokens":0,"output_tokens":136,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0025180000000000003,"input_uncached_usd":0.001158,"input_cached_usd":0.0,"output_usd":0.00136,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9761436670087278}
{"item_id":"semeval2015.d001.s033.t000","gold_sense_keys":["see%2:39:00::"],"candidates":[{"index":1,"sense_key":"see%2:39:00::","synset_id":"see.v.01"},{"index":2,"sense_key":"see%2:31:01::","synset_id":"understand.v.02"},{"index":3,"sense_key":"see%2:39:02::","synset_id":"witness.v.02"},{"index":4,"sense_key":"see%2:36:00::","synset_id":"visualize.v.01"},{"index":5,"sense_key":"see%2:31:00::","synset_id":"see.v.05"},{"index":6,"sense_key":"see%2:31:02::","synset_id":"learn.v.02"},{"index":7,"sense_key":"see%2:39:03::","synset_id":"watch.v.03"},{"index":8,"sense_key":"see%2:38:00::","synset_id":"meet.v.01"},{"index":9,"sense_key":"see%2:32:00::","synset_id":"determine.v.08"},{"index":10,"sense_key":"see%2:31:03::","synset_id":"see.v.10"},{"index":11,"sense_key":"see%2:41:11::","synset_id":"see.v.11"},{"index":12,"sense_key":"see%2:41:12::","synset_id":"see.v.12"},{"index":13,"sense_key":"see%2:41:10::","synset_id":"visit.v.01"},{"index":14,"sense_key":"see%2:41:14::","synset_id":"attend.v.02"},{"index":15,"sense_key":"see%2:35:09::","synset_id":"see.v.15"},{"index":16,"sense_key":"see%2:41:00::","synset_id":"go_steady.v.01"},{"index":17,"sense_key":"see%2:39:05::","synset_id":"see.v.17"},{"index":18,"sense_key":"see%2:31:08::","synset_id":"see.v.18"},{"index":19,"sense_key":"see%2:39:11::","synset_id":"see.v.19"},{"index":20,"sense_key":"see%2:39:13::","synset_id":"examine.v.02"},{"index":21,"sense_key":"see%2:39:12::","synset_id":"experience.v.01"},{"index":22,"sense_key":"see%2:38:03::","synset_id":"see.v.22"},{"index":23,"sense_key":"see%2:33:00::","synset_id":"see.v.23"},{"index":24,"sense_key":"see%2:31:13::","synset_id":"interpret.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"see%2:39:00::","call_ids":["semeval2015.d001.s033.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"see%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1940,"cached_input_tokens":0,"output_tokens":113,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00501,"input_uncached_usd":0.0038799999999999998,"input_cached_usd":0.0,"output_usd":0.0011300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.18634962500073}
{"item_id":"semeval2015.d001.s033.t001","gold_sense_keys":["let%2:41:00::"],"candidates":[{"index":1,"sense_key":"let%2:41:00::","synset_id":"let.v.01"},{"index":2,"sense_key":"let%2:32:00::","synset_id":"let.v.02"},{"index":3,"sense_key":"let%2:32:02::","synset_id":"permit.v.01"},{"index":4,"sense_key":"let%2:30:01::","synset_id":"get.v.03"},{"index":5,"sense_key":"let%2:30:00::","synset_id":"let.v.05"},{"index":6,"sense_key":"let%2:40:00::","synset_id":"lease.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"let%2:41:00::","call_ids":["semeval2015.d001.s033.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"let%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":754,"cached_input_tokens":0,"output_tokens":99,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0024980000000000002,"input_uncached_usd":0.001508,"input_cached_usd":0.0,"output_usd":0.00099,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.5582551249535754}
{"item_id":"semeval2015.d001.s033.t002","gold_sense_keys":["change%2:30:01::"],"candidates":[{"index":1,"sense_key":"change%2:30:01::","synset_id":"change.v.01"},{"index":2,"sense_key":"change%2:30:00::","synset_id":"change.v.02"},{"index":3,"sense_key":"change%2:30:02::","synset_id":"change.v.03"},{"index":4,"sense_key":"change%2:30:05::","synset_id":"switch.v.03"},{"index":5,"sense_key":"change%2:30:03::","synset_id":"change.v.05"},{"index":6,"sense_key":"change%2:30:04::","synset_id":"change.v.06"},{"index":7,"sense_key":"change%2:40:00::","synset_id":"exchange.v.01"},{"index":8,"sense_key":"change%2:38:00::","synset_id":"transfer.v.06"},{"index":9,"sense_key":"change%2:30:08::","synset_id":"deepen.v.04"},{"index":10,"sense_key":"change%2:30:06::","synset_id":"change.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"change%2:30:01::","call_ids":["semeval2015.d001.s033.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"change%2:30:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1087,"cached_input_tokens":0,"output_tokens":109,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0032639999999999995,"input_uncached_usd":0.0021739999999999997,"input_cached_usd":0.0,"output_usd":0.00109,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7417536249849945}
{"item_id":"semeval2015.d001.s033.t003","gold_sense_keys":["value%1:09:00::"],"candidates":[{"index":1,"sense_key":"value%1:09:00::","synset_id":"value.n.01"},{"index":2,"sense_key":"value%1:07:00::","synset_id":"value.n.02"},{"index":3,"sense_key":"value%1:21:00::","synset_id":"value.n.03"},{"index":4,"sense_key":"value%1:07:01::","synset_id":"value.n.04"},{"index":5,"sense_key":"value%1:28:00::","synset_id":"value.n.05"},{"index":6,"sense_key":"value%1:09:01::","synset_id":"value.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"value%1:09:00::","call_ids":["semeval2015.d001.s033.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"value%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":758,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001636,"input_uncached_usd":0.001516,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1961758329998702}
{"item_id":"semeval2015.d001.s033.t004","gold_sense_keys":["just%4:02:00::"],"candidates":[{"index":1,"sense_key":"just%4:02:00::","synset_id":"merely.r.01"},{"index":2,"sense_key":"just%4:02:05::","synset_id":"precisely.r.01"},{"index":3,"sense_key":"just%4:02:01::","synset_id":"just.r.03"},{"index":4,"sense_key":"just%4:02:03::","synset_id":"just.r.04"},{"index":5,"sense_key":"just%4:02:04::","synset_id":"barely.r.01"},{"index":6,"sense_key":"just%4:02:06::","synset_id":"just.r.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"just%4:02:00::","call_ids":["semeval2015.d001.s033.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"just%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":782,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001684,"input_uncached_usd":0.0015639999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0546886660158634}
{"item_id":"semeval2015.d001.s033.t005","gold_sense_keys":["way%1:04:01::","way%1:07:01::"],"candidates":[{"index":1,"sense_key":"way%1:07:01::","synset_id":"manner.n.01"},{"index":2,"sense_key":"way%1:04:01::","synset_id":"means.n.01"},{"index":3,"sense_key":"way%1:15:00::","synset_id":"direction.n.01"},{"index":4,"sense_key":"way%1:26:00::","synset_id":"way.n.04"},{"index":5,"sense_key":"way%1:04:00::","synset_id":"way.n.05"},{"index":6,"sense_key":"way%1:06:00::","synset_id":"way.n.06"},{"index":7,"sense_key":"way%1:04:02::","synset_id":"way.n.07"},{"index":8,"sense_key":"way%1:23:00::","synset_id":"room.n.02"},{"index":9,"sense_key":"way%1:07:02::","synset_id":"way.n.09"},{"index":10,"sense_key":"way%1:09:00::","synset_id":"way.n.10"},{"index":11,"sense_key":"way%1:09:01::","synset_id":"way.n.11"},{"index":12,"sense_key":"way%1:21:00::","synset_id":"way.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"way%1:04:01::","call_ids":["semeval2015.d001.s033.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"way%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1004,"cached_input_tokens":0,"output_tokens":96,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002968,"input_uncached_usd":0.002008,"input_cached_usd":0.0,"output_usd":0.0009600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.677294332999736}
{"item_id":"semeval2015.d001.s033.t007","gold_sense_keys":["log%1:10:02::"],"candidates":[{"index":1,"sense_key":"log%1:27:01::","synset_id":"log.n.01"},{"index":2,"sense_key":"log%1:10:01::","synset_id":"logarithm.n.01"},{"index":3,"sense_key":"log%1:10:02::","synset_id":"log.n.03"},{"index":4,"sense_key":"log%1:10:00::","synset_id":"log.n.04"},{"index":5,"sense_key":"log%1:06:01::","synset_id":"log.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"log%1:10:01::","call_ids":["semeval2015.d001.s033.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"log%1:10:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":621,"cached_input_tokens":0,"output_tokens":156,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0028020000000000002,"input_uncached_usd":0.001242,"input_cached_usd":0.0,"output_usd":0.0015600000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.7785864159232005}
{"item_id":"semeval2015.d001.s034.t000","gold_sense_keys":["time%1:11:00::"],"candidates":[{"index":1,"sense_key":"time%1:11:00::","synset_id":"time.n.01"},{"index":2,"sense_key":"time%1:28:05::","synset_id":"time.n.02"},{"index":3,"sense_key":"time%1:28:00::","synset_id":"time.n.03"},{"index":4,"sense_key":"time%1:28:06::","synset_id":"time.n.04"},{"index":5,"sense_key":"time%1:03:00::","synset_id":"time.n.05"},{"index":6,"sense_key":"time%1:11:01::","synset_id":"time.n.06"},{"index":7,"sense_key":"time%1:28:03::","synset_id":"clock_time.n.01"},{"index":8,"sense_key":"time%1:28:04::","synset_id":"fourth_dimension.n.01"},{"index":9,"sense_key":"time%1:07:01::","synset_id":"meter.n.04"},{"index":10,"sense_key":"time%1:28:01::","synset_id":"prison_term.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"time%1:11:00::","call_ids":["semeval2015.d001.s034.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"time%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":924,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0024279999999999996,"input_uncached_usd":0.0018479999999999998,"input_cached_usd":0.0,"output_usd":0.00058,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1225781670073047}
{"item_id":"semeval2015.d001.s034.t002","gold_sense_keys":["expression%1:10:03::"],"candidates":[{"index":1,"sense_key":"expression%1:07:00::","synset_id":"expression.n.01"},{"index":2,"sense_key":"expression%1:10:02::","synset_id":"expression.n.02"},{"index":3,"sense_key":"expression%1:10:04::","synset_id":"expression.n.03"},{"index":4,"sense_key":"expression%1:10:00::","synset_id":"saying.n.01"},{"index":5,"sense_key":"expression%1:10:01::","synset_id":"formulation.n.03"},{"index":6,"sense_key":"expression%1:10:03::","synset_id":"formula.n.01"},{"index":7,"sense_key":"expression%1:22:00::","synset_id":"expression.n.07"},{"index":8,"sense_key":"expression%1:10:05::","synset_id":"construction.n.02"},{"index":9,"sense_key":"expression%1:04:00::","synset_id":"expression.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"expression%1:10:03::","call_ids":["semeval2015.d001.s034.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"expression%1:10:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":928,"cached_input_tokens":0,"output_tokens":74,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0025960000000000002,"input_uncached_usd":0.001856,"input_cached_usd":0.0,"output_usd":0.0007400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4745084160240367}
{"item_id":"semeval2015.d001.s034.t003","gold_sense_keys":["variable%1:10:00::"],"candidates":[{"index":1,"sense_key":"variable%1:17:00::","synset_id":"variable.n.01"},{"index":2,"sense_key":"variable%1:09:00::","synset_id":"variable.n.02"},{"index":3,"sense_key":"variable%1:17:01::","synset_id":"variable_star.n.01"},{"index":4,"sense_key":"variable%1:10:00::","synset_id":"variable.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"variable%1:10:00::","call_ids":["semeval2015.d001.s034.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"variable%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":555,"cached_input_tokens":0,"output_tokens":213,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0032400000000000003,"input_uncached_usd":0.0011099999999999999,"input_cached_usd":0.0,"output_usd":0.0021300000000000004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.5907089170068502}
{"item_id":"semeval2015.d001.s034.t004","gold_sense_keys":["value%1:09:00::"],"candidates":[{"index":1,"sense_key":"value%1:09:00::","synset_id":"value.n.01"},{"index":2,"sense_key":"value%1:07:00::","synset_id":"value.n.02"},{"index":3,"sense_key":"value%1:21:00::","synset_id":"value.n.03"},{"index":4,"sense_key":"value%1:07:01::","synset_id":"value.n.04"},{"index":5,"sense_key":"value%1:28:00::","synset_id":"value.n.05"},{"index":6,"sense_key":"value%1:09:01::","synset_id":"value.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"value%1:09:00::","call_ids":["semeval2015.d001.s034.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"value%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":726,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001572,"input_uncached_usd":0.001452,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5556112499907613}
{"item_id":"semeval2015.d001.s034.t005","gold_sense_keys":["change%2:30:01::"],"candidates":[{"index":1,"sense_key":"change%2:30:01::","synset_id":"change.v.01"},{"index":2,"sense_key":"change%2:30:00::","synset_id":"change.v.02"},{"index":3,"sense_key":"change%2:30:02::","synset_id":"change.v.03"},{"index":4,"sense_key":"change%2:30:05::","synset_id":"switch.v.03"},{"index":5,"sense_key":"change%2:30:03::","synset_id":"change.v.05"},{"index":6,"sense_key":"change%2:30:04::","synset_id":"change.v.06"},{"index":7,"sense_key":"change%2:40:00::","synset_id":"exchange.v.01"},{"index":8,"sense_key":"change%2:38:00::","synset_id":"transfer.v.06"},{"index":9,"sense_key":"change%2:30:08::","synset_id":"deepen.v.04"},{"index":10,"sense_key":"change%2:30:06::","synset_id":"change.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"change%2:30:01::","call_ids":["semeval2015.d001.s034.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"change%2:30:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1056,"cached_input_tokens":0,"output_tokens":122,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003332,"input_uncached_usd":0.0021119999999999997,"input_cached_usd":0.0,"output_usd":0.0012200000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.545291916001588}
{"item_id":"semeval2015.d001.s034.t006","gold_sense_keys":["last%3:00:00::","last%5:00:00:past:00"],"candidates":[{"index":1,"sense_key":"last%5:00:00:past:00","synset_id":"last.s.01"},{"index":2,"sense_key":"last%3:00:00::","synset_id":"last.a.02"},{"index":3,"sense_key":"last%5:00:00:closing:00","synset_id":"concluding.s.01"},{"index":4,"sense_key":"last%5:00:00:unlikely:00","synset_id":"last.s.03"},{"index":5,"sense_key":"last%5:00:00:dying:00","synset_id":"last.s.04"},{"index":6,"sense_key":"last%5:00:00:ultimate:00","synset_id":"final.s.02"},{"index":7,"sense_key":"last%5:00:00:high:02","synset_id":"last.s.06"},{"index":8,"sense_key":"last%5:00:00:unalterable:00","synset_id":"final.s.03"},{"index":9,"sense_key":"last%5:00:00:worst:00","synset_id":"last.s.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"last%5:00:00:past:00","call_ids":["semeval2015.d001.s034.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"last%5:00:00:past:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":906,"cached_input_tokens":0,"output_tokens":64,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002452,"input_uncached_usd":0.001812,"input_cached_usd":0.0,"output_usd":0.00064,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4772161670261994}
{"item_id":"semeval2015.d001.s034.t007","gold_sense_keys":["result%1:10:00::","result%1:11:00::"],"candidates":[{"index":1,"sense_key":"result%1:19:00::","synset_id":"consequence.n.01"},{"index":2,"sense_key":"result%1:10:00::","synset_id":"solution.n.02"},{"index":3,"sense_key":"result%1:11:00::","synset_id":"result.n.03"},{"index":4,"sense_key":"result%1:10:01::","synset_id":"resultant_role.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"result%1:10:00::","call_ids":["semeval2015.d001.s034.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"result%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":671,"cached_input_tokens":0,"output_tokens":98,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0023220000000000003,"input_uncached_usd":0.0013419999999999999,"input_cached_usd":0.0,"output_usd":0.0009800000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9766965840244666}
{"item_id":"semeval2015.d001.s035.t000","gold_sense_keys":["new%5:00:00:other:00"],"candidates":[{"index":1,"sense_key":"new%3:00:00::","synset_id":"new.a.01"},{"index":2,"sense_key":"new%5:00:00:original:00","synset_id":"fresh.s.03"},{"index":3,"sense_key":"new%5:00:00:inexperienced:00","synset_id":"raw.s.11"},{"index":4,"sense_key":"new%5:00:00:unprecedented:00","synset_id":"new.s.03"},{"index":5,"sense_key":"new%5:00:00:other:00","synset_id":"new.s.04"},{"index":6,"sense_key":"new%3:00:09::","synset_id":"new.a.06"},{"index":7,"sense_key":"new%5:00:02:original:00","synset_id":"newfangled.s.01"},{"index":8,"sense_key":"new%5:00:02:late:01","synset_id":"new.s.06"},{"index":9,"sense_key":"new%5:00:01:late:01","synset_id":"modern.s.04"},{"index":10,"sense_key":"new%5:00:00:early:02","synset_id":"new.s.08"},{"index":11,"sense_key":"new%5:00:00:unaccustomed:00","synset_id":"new.s.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"new%3:00:00::","call_ids":["semeval2015.d001.s035.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"new%3:00:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1040,"cached_input_tokens":0,"output_tokens":70,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00278,"input_uncached_usd":0.00208,"input_cached_usd":0.0,"output_usd":0.0007000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.090955000021495}
{"item_id":"semeval2015.d001.s035.t001","gold_sense_keys":["window%1:06:03::"],"candidates":[{"index":1,"sense_key":"window%1:06:00::","synset_id":"window.n.01"},{"index":2,"sense_key":"window%1:06:01::","synset_id":"window.n.02"},{"index":3,"sense_key":"window%1:06:02::","synset_id":"window.n.03"},{"index":4,"sense_key":"window%1:17:00::","synset_id":"window.n.04"},{"index":5,"sense_key":"window%1:28:00::","synset_id":"window.n.05"},{"index":6,"sense_key":"window%1:06:04::","synset_id":"windowpane.n.01"},{"index":7,"sense_key":"window%1:06:05::","synset_id":"window.n.07"},{"index":8,"sense_key":"window%1:06:03::","synset_id":"window.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"window%1:06:03::","call_ids":["semeval2015.d001.s035.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"window%1:06:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":800,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00172,"input_uncached_usd":0.0015999999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1099612909602001}
{"item_id":"semeval2015.d001.s036.t000","gold_sense_keys":["log%1:10:02::"],"candidates":[{"index":1,"sense_key":"log%1:27:01::","synset_id":"log.n.01"},{"index":2,"sense_key":"log%1:10:01::","synset_id":"logarithm.n.01"},{"index":3,"sense_key":"log%1:10:02::","synset_id":"log.n.03"},{"index":4,"sense_key":"log%1:10:00::","synset_id":"log.n.04"},{"index":5,"sense_key":"log%1:06:01::","synset_id":"log.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"log%1:10:02::","call_ids":["semeval2015.d001.s036.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"log%1:10:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":577,"cached_input_tokens":0,"output_tokens":102,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0021739999999999997,"input_uncached_usd":0.0011539999999999999,"input_cached_usd":0.0,"output_usd":0.00102,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.229332625051029}
{"item_id":"semeval2015.d001.s037.t000","gold_sense_keys":["load%2:40:12::"],"candidates":[{"index":1,"sense_key":"load%2:35:01::","synset_id":"load.v.01"},{"index":2,"sense_key":"load%2:35:02::","synset_id":"load.v.02"},{"index":3,"sense_key":"load%2:40:12::","synset_id":"load.v.03"},{"index":4,"sense_key":"load%2:35:10::","synset_id":"load.v.04"},{"index":5,"sense_key":"load%2:30:12::","synset_id":"load.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"load%2:40:12::","call_ids":["semeval2015.d001.s037.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"load%2:40:12::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":638,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001396,"input_uncached_usd":0.001276,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9863088330021128}
{"item_id":"semeval2015.d001.s038.t000","gold_sense_keys":["execute%2:41:04::"],"candidates":[{"index":1,"sense_key":"execute%2:41:00::","synset_id":"execute.v.01"},{"index":2,"sense_key":"execute%2:41:01::","synset_id":"execute.v.02"},{"index":3,"sense_key":"execute%2:36:00::","synset_id":"carry_through.v.01"},{"index":4,"sense_key":"execute%2:41:03::","synset_id":"execute.v.04"},{"index":5,"sense_key":"execute%2:41:04::","synset_id":"run.v.19"},{"index":6,"sense_key":"execute%2:36:01::","synset_id":"perform.v.01"},{"index":7,"sense_key":"execute%2:32:00::","synset_id":"execute.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"execute%2:41:04::","call_ids":["semeval2015.d001.s038.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"execute%2:41:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":772,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0016640000000000001,"input_uncached_usd":0.001544,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3817054580431432}
{"item_id":"semeval2015.d001.s038.t001","gold_sense_keys":["instruction%1:10:02::"],"candidates":[{"index":1,"sense_key":"instruction%1:10:04::","synset_id":"direction.n.06"},{"index":2,"sense_key":"instruction%1:04:01::","synset_id":"education.n.01"},{"index":3,"sense_key":"instruction%1:04:00::","synset_id":"teaching.n.01"},{"index":4,"sense_key":"instruction%1:10:02::","synset_id":"instruction.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"instruction%1:10:02::","call_ids":["semeval2015.d001.s038.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"instruction%1:10:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":634,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0013880000000000001,"input_uncached_usd":0.001268,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5100713330321014}
{"item_id":"semeval2015.d001.s038.t002","gold_sense_keys":["file%1:10:00::"],"candidates":[{"index":1,"sense_key":"file%1:10:00::","synset_id":"file.n.01"},{"index":2,"sense_key":"file%1:14:00::","synset_id":"file.n.02"},{"index":3,"sense_key":"file%1:06:01::","synset_id":"file.n.03"},{"index":4,"sense_key":"file%1:06:00::","synset_id":"file.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"file%1:10:00::","call_ids":["semeval2015.d001.s038.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"file%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":557,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001234,"input_uncached_usd":0.001114,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":4.493084374931641}
{"item_id":"semeval2015.d001.s039.t000","gold_sense_keys":["nice%3:00:00::"],"candidates":[{"index":1,"sense_key":"nice%3:00:00::","synset_id":"nice.a.01"},{"index":2,"sense_key":"nice%5:00:00:respectable:00","synset_id":"decent.s.01"},{"index":3,"sense_key":"nice%5:00:02:precise:00","synset_id":"nice.s.02"},{"index":4,"sense_key":"nice%5:00:00:fastidious:00","synset_id":"dainty.s.04"},{"index":5,"sense_key":"nice%5:00:00:polite:00","synset_id":"courteous.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"nice%3:00:00::","call_ids":["semeval2015.d001.s039.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"nice%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":714,"cached_input_tokens":0,"output_tokens":88,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002308,"input_uncached_usd":0.001428,"input_cached_usd":0.0,"output_usd":0.00088,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7918466249248013}
{"item_id":"semeval2015.d001.s039.t001","gold_sense_keys":["want%2:37:00::"],"candidates":[{"index":1,"sense_key":"want%2:37:00::","synset_id":"desire.v.01"},{"index":2,"sense_key":"want%2:34:00::","synset_id":"want.v.02"},{"index":3,"sense_key":"want%2:35:00::","synset_id":"want.v.03"},{"index":4,"sense_key":"want%2:31:01::","synset_id":"want.v.04"},{"index":5,"sense_key":"want%2:42:01::","synset_id":"want.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"want%2:37:00::","call_ids":["semeval2015.d001.s039.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"want%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":595,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00131,"input_uncached_usd":0.0011899999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8659293339587748}
{"item_id":"semeval2015.d001.s039.t002","gold_sense_keys":["define%2:32:01::"],"candidates":[{"index":1,"sense_key":"define%2:42:00::","synset_id":"specify.v.03"},{"index":2,"sense_key":"define%2:32:00::","synset_id":"define.v.02"},{"index":3,"sense_key":"define%2:42:01::","synset_id":"define.v.03"},{"index":4,"sense_key":"define%2:42:03::","synset_id":"define.v.04"},{"index":5,"sense_key":"define%2:32:01::","synset_id":"specify.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"define%2:32:01::","call_ids":["semeval2015.d001.s039.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"define%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":590,"cached_input_tokens":0,"output_tokens":104,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0022199999999999998,"input_uncached_usd":0.0011799999999999998,"input_cached_usd":0.0,"output_usd":0.0010400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7379234170075506}
{"item_id":"semeval2015.d001.s039.t003","gold_sense_keys":["library%1:14:01::"],"candidates":[{"index":1,"sense_key":"library%1:06:01::","synset_id":"library.n.01"},{"index":2,"sense_key":"library%1:14:00::","synset_id":"library.n.02"},{"index":3,"sense_key":"library%1:06:00::","synset_id":"library.n.03"},{"index":4,"sense_key":"library%1:14:01::","synset_id":"library.n.04"},{"index":5,"sense_key":"library%1:06:02::","synset_id":"library.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"library%1:14:01::","call_ids":["semeval2015.d001.s039.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"library%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":555,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00123,"input_uncached_usd":0.0011099999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":4.215719832922332}
{"item_id":"semeval2015.d001.s039.t004","gold_sense_keys":["resume%2:30:00::"],"candidates":[{"index":1,"sense_key":"resume%2:30:00::","synset_id":"resume.v.01"},{"index":2,"sense_key":"resume%2:30:01::","synset_id":"resume.v.02"},{"index":3,"sense_key":"resume%2:41:00::","synset_id":"resume.v.03"},{"index":4,"sense_key":"resume%2:32:00::","synset_id":"sum_up.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"resume%2:30:00::","call_ids":["semeval2015.d001.s039.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"resume%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":539,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001198,"input_uncached_usd":0.001078,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9825868339976296}
{"item_id":"semeval2015.d001.s039.t005","gold_sense_keys":["previous%5:00:00:preceding:00"],"candidates":[{"index":1,"sense_key":"previous%5:00:00:preceding:00","synset_id":"previous.s.01"},{"index":2,"sense_key":"previous%5:00:00:past:00","synset_id":"former.s.02"},{"index":3,"sense_key":"previous%5:00:00:early:00","synset_id":"previous.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"previous%5:00:00:preceding:00","call_ids":["semeval2015.d001.s039.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"previous%5:00:00:preceding:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":517,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001154,"input_uncached_usd":0.001034,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":10.081498375046067}
{"item_id":"semeval2015.d001.s039.t006","gold_sense_keys":["work%1:04:00::"],"candidates":[{"index":1,"sense_key":"work%1:04:00::","synset_id":"work.n.01"},{"index":2,"sense_key":"work%1:06:00::","synset_id":"work.n.02"},{"index":3,"sense_key":"work%1:04:01::","synset_id":"employment.n.02"},{"index":4,"sense_key":"work%1:09:00::","synset_id":"study.n.02"},{"index":5,"sense_key":"work%1:19:00::","synset_id":"work.n.05"},{"index":6,"sense_key":"work%1:06:01::","synset_id":"workplace.n.01"},{"index":7,"sense_key":"work%1:06:02::","synset_id":"oeuvre.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"work%1:04:00::","call_ids":["semeval2015.d001.s039.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"work%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":829,"cached_input_tokens":0,"output_tokens":103,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0026880000000000003,"input_uncached_usd":0.001658,"input_cached_usd":0.0,"output_usd":0.00103,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9393011670326814}
{"item_id":"semeval2015.d001.s040.t000","gold_sense_keys":["save%2:32:04::"],"candidates":[{"index":1,"sense_key":"save%2:41:01::","synset_id":"salvage.v.01"},{"index":2,"sense_key":"save%2:40:00::","synset_id":"save.v.02"},{"index":3,"sense_key":"save%2:41:00::","synset_id":"save.v.03"},{"index":4,"sense_key":"save%2:40:02::","synset_id":"save.v.04"},{"index":5,"sense_key":"save%2:40:01::","synset_id":"save.v.05"},{"index":6,"sense_key":"save%2:30:00::","synset_id":"save.v.06"},{"index":7,"sense_key":"save%2:41:02::","synset_id":"deliver.v.08"},{"index":8,"sense_key":"save%2:41:05::","synset_id":"spare.v.01"},{"index":9,"sense_key":"save%2:40:04::","synset_id":"save.v.09"},{"index":10,"sense_key":"save%2:40:03::","synset_id":"keep_open.v.01"},{"index":11,"sense_key":"save%2:32:04::","synset_id":"write.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"save%2:32:04::","call_ids":["semeval2015.d001.s040.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"save%2:32:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":910,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00194,"input_uncached_usd":0.00182,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":8.984041249961592}
{"item_id":"semeval2015.d001.s041.t000","gold_sense_keys":["save%2:32:04::"],"candidates":[{"index":1,"sense_key":"save%2:41:01::","synset_id":"salvage.v.01"},{"index":2,"sense_key":"save%2:40:00::","synset_id":"save.v.02"},{"index":3,"sense_key":"save%2:41:00::","synset_id":"save.v.03"},{"index":4,"sense_key":"save%2:40:02::","synset_id":"save.v.04"},{"index":5,"sense_key":"save%2:40:01::","synset_id":"save.v.05"},{"index":6,"sense_key":"save%2:30:00::","synset_id":"save.v.06"},{"index":7,"sense_key":"save%2:41:02::","synset_id":"deliver.v.08"},{"index":8,"sense_key":"save%2:41:05::","synset_id":"spare.v.01"},{"index":9,"sense_key":"save%2:40:04::","synset_id":"save.v.09"},{"index":10,"sense_key":"save%2:40:03::","synset_id":"keep_open.v.01"},{"index":11,"sense_key":"save%2:32:04::","synset_id":"write.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"save%2:32:04::","call_ids":["semeval2015.d001.s041.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"save%2:32:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":927,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001974,"input_uncached_usd":0.001854,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6093810410238802}
{"item_id":"semeval2015.d001.s041.t001","gold_sense_keys":["instruction%1:10:02::"],"candidates":[{"index":1,"sense_key":"instruction%1:10:04::","synset_id":"direction.n.06"},{"index":2,"sense_key":"instruction%1:04:01::","synset_id":"education.n.01"},{"index":3,"sense_key":"instruction%1:04:00::","synset_id":"teaching.n.01"},{"index":4,"sense_key":"instruction%1:10:02::","synset_id":"instruction.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"instruction%1:10:02::","call_ids":["semeval2015.d001.s041.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"instruction%1:10:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":625,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00192,"input_uncached_usd":0.00125,"input_cached_usd":0.0,"output_usd":0.00067,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6312174999620765}
{"item_id":"semeval2015.d001.s041.t002","gold_sense_keys":["type%2:32:00::"],"candidates":[{"index":1,"sense_key":"type%2:32:00::","synset_id":"type.v.01"},{"index":2,"sense_key":"type%2:31:00::","synset_id":"type.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"type%2:32:00::","call_ids":["semeval2015.d001.s041.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"type%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":450,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00102,"input_uncached_usd":0.0009,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.848739124950953}
{"item_id":"semeval2015.d001.s041.t003","gold_sense_keys":["begin%2:42:00::","begin%2:42:04::"],"candidates":[{"index":1,"sense_key":"begin%2:30:00::","synset_id":"get_down.v.07"},{"index":2,"sense_key":"begin%2:42:00::","synset_id":"begin.v.02"},{"index":3,"sense_key":"begin%2:30:01::","synset_id":"begin.v.03"},{"index":4,"sense_key":"begin%2:32:03::","synset_id":"begin.v.04"},{"index":5,"sense_key":"begin%2:42:02::","synset_id":"begin.v.05"},{"index":6,"sense_key":"begin%2:42:04::","synset_id":"begin.v.06"},{"index":7,"sense_key":"begin%2:42:03::","synset_id":"begin.v.07"},{"index":8,"sense_key":"begin%2:41:04::","synset_id":"begin.v.08"},{"index":9,"sense_key":"begin%2:41:00::","synset_id":"begin.v.09"},{"index":10,"sense_key":"begin%2:32:04::","synset_id":"begin.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"begin%2:42:04::","call_ids":["semeval2015.d001.s041.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"begin%2:42:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1053,"cached_input_tokens":0,"output_tokens":111,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003216,"input_uncached_usd":0.002106,"input_cached_usd":0.0,"output_usd":0.00111,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0187981659546494}
{"item_id":"semeval2015.d001.s041.t004","gold_sense_keys":["able%3:00:00::"],"candidates":[{"index":1,"sense_key":"able%3:00:00::","synset_id":"able.a.01"},{"index":2,"sense_key":"able%5:00:00:competent:00","synset_id":"able.s.01"},{"index":3,"sense_key":"able%5:00:00:capable:00","synset_id":"able.s.02"},{"index":4,"sense_key":"able%5:00:00:fit:01","synset_id":"able.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"able%3:00:00::","call_ids":["semeval2015.d001.s041.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"able%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":593,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0019160000000000002,"input_uncached_usd":0.001186,"input_cached_usd":0.0,"output_usd":0.0007300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4825343339471146}
{"item_id":"semeval2015.d001.s042.t000","gold_sense_keys":["generate%2:36:00::"],"candidates":[{"index":1,"sense_key":"generate%2:36:00::","synset_id":"generate.v.01"},{"index":2,"sense_key":"generate%2:36:02::","synset_id":"render.v.04"},{"index":3,"sense_key":"generate%2:36:01::","synset_id":"generate.v.03"},{"index":4,"sense_key":"generate%2:29:00::","synset_id":"beget.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"generate%2:36:00::","call_ids":["semeval2015.d001.s042.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"generate%2:36:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":626,"cached_input_tokens":0,"output_tokens":101,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002262,"input_uncached_usd":0.0012519999999999999,"input_cached_usd":0.0,"output_usd":0.00101,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1573028749553487}
{"item_id":"semeval2015.d001.s042.t002","gold_sense_keys":["easy%3:00:01::"],"candidates":[{"index":1,"sense_key":"easy%3:00:01::","synset_id":"easy.a.01"},{"index":2,"sense_key":"easy%5:00:00:unhurried:00","synset_id":"easy.s.01"},{"index":3,"sense_key":"easy%3:00:02::","synset_id":"easy.a.03"},{"index":4,"sense_key":"easy%5:00:00:pleasing:00","synset_id":"easy.s.02"},{"index":5,"sense_key":"easy%5:00:00:light:08","synset_id":"easy.s.03"},{"index":6,"sense_key":"easy%5:00:00:impressionable:00","synset_id":"easy.s.04"},{"index":7,"sense_key":"easy%5:00:00:rich:00","synset_id":"comfortable.s.03"},{"index":8,"sense_key":"easy%5:00:00:gradual:02","synset_id":"easy.s.06"},{"index":9,"sense_key":"easy%5:00:00:comfortable:00","synset_id":"easy.s.07"},{"index":10,"sense_key":"easy%5:00:00:unchaste:00","synset_id":"easy.s.08"},{"index":11,"sense_key":"easy%5:00:00:available:00","synset_id":"easy.s.09"},{"index":12,"sense_key":"easy%5:00:00:abundant:00","synset_id":"easy.s.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"easy%3:00:01::","call_ids":["semeval2015.d001.s042.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"easy%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1104,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0023279999999999998,"input_uncached_usd":0.002208,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4423609169898555}
{"item_id":"semeval2015.d001.s042.t003","gold_sense_keys":["fix%2:30:01::"],"candidates":[{"index":1,"sense_key":"fix%2:30:01::","synset_id":"repair.v.01"},{"index":2,"sense_key":"fix%2:35:00::","synset_id":"fasten.v.01"},{"index":3,"sense_key":"fix%2:32:03::","synset_id":"specify.v.02"},{"index":4,"sense_key":"fix%2:36:00::","synset_id":"cook.v.02"},{"index":5,"sense_key":"fix%2:33:00::","synset_id":"pay_back.v.02"},{"index":6,"sense_key":"fix%2:36:01::","synset_id":"fix.v.06"},{"index":7,"sense_key":"fix%2:30:02::","synset_id":"fix.v.07"},{"index":8,"sense_key":"fix%2:30:00::","synset_id":"fixate.v.03"},{"index":9,"sense_key":"fix%2:29:00::","synset_id":"sterilize.v.02"},{"index":10,"sense_key":"fix%2:41:04::","synset_id":"fix.v.10"},{"index":11,"sense_key":"fix%2:35:01::","synset_id":"situate.v.02"},{"index":12,"sense_key":"fix%2:30:03::","synset_id":"fix.v.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"fix%2:30:01::","call_ids":["semeval2015.d001.s042.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"fix%2:30:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1152,"cached_input_tokens":0,"output_tokens":53,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002834,"input_uncached_usd":0.002304,"input_cached_usd":0.0,"output_usd":0.0005300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":10.96447025006637}
{"item_id":"semeval2015.d001.s042.t004","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":2,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":3,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":4,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":5,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":6,"sense_key":"use%2:41:03::","synset_id":"use.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%2:34:01::","call_ids":["semeval2015.d001.s042.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":729,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001578,"input_uncached_usd":0.0014579999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.918066457961686}
{"item_id":"semeval2015.d001.s042.t005","gold_sense_keys":["text_editor%1:10:00::"],"candidates":[{"index":1,"sense_key":"text_editor%1:18:00::","synset_id":"copy_editor.n.01"},{"index":2,"sense_key":"text_editor%1:10:00::","synset_id":"text_editor.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"text_editor%1:10:00::","call_ids":["semeval2015.d001.s042.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"text_editor%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":441,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001002,"input_uncached_usd":0.000882,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.197629375034012}
{"item_id":"semeval2015.d001.s043.t000","gold_sense_keys":["save%2:32:04::"],"candidates":[{"index":1,"sense_key":"save%2:41:01::","synset_id":"salvage.v.01"},{"index":2,"sense_key":"save%2:40:00::","synset_id":"save.v.02"},{"index":3,"sense_key":"save%2:41:00::","synset_id":"save.v.03"},{"index":4,"sense_key":"save%2:40:02::","synset_id":"save.v.04"},{"index":5,"sense_key":"save%2:40:01::","synset_id":"save.v.05"},{"index":6,"sense_key":"save%2:30:00::","synset_id":"save.v.06"},{"index":7,"sense_key":"save%2:41:02::","synset_id":"deliver.v.08"},{"index":8,"sense_key":"save%2:41:05::","synset_id":"spare.v.01"},{"index":9,"sense_key":"save%2:40:04::","synset_id":"save.v.09"},{"index":10,"sense_key":"save%2:40:03::","synset_id":"keep_open.v.01"},{"index":11,"sense_key":"save%2:32:04::","synset_id":"write.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"save%2:32:04::","call_ids":["semeval2015.d001.s043.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"save%2:32:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":942,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0020039999999999997,"input_uncached_usd":0.0018839999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.031207749969326}
{"item_id":"semeval2015.d001.s043.t001","gold_sense_keys":["log%1:10:02::"],"candidates":[{"index":1,"sense_key":"log%1:27:01::","synset_id":"log.n.01"},{"index":2,"sense_key":"log%1:10:01::","synset_id":"logarithm.n.01"},{"index":3,"sense_key":"log%1:10:02::","synset_id":"log.n.03"},{"index":4,"sense_key":"log%1:10:00::","synset_id":"log.n.04"},{"index":5,"sense_key":"log%1:06:01::","synset_id":"log.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"log%1:10:02::","call_ids":["semeval2015.d001.s043.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"log%1:10:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":592,"cached_input_tokens":0,"output_tokens":103,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002214,"input_uncached_usd":0.001184,"input_cached_usd":0.0,"output_usd":0.00103,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.3957284579519182}
{"item_id":"semeval2015.d001.s044.t000","gold_sense_keys":["save%2:32:04::"],"candidates":[{"index":1,"sense_key":"save%2:41:01::","synset_id":"salvage.v.01"},{"index":2,"sense_key":"save%2:40:00::","synset_id":"save.v.02"},{"index":3,"sense_key":"save%2:41:00::","synset_id":"save.v.03"},{"index":4,"sense_key":"save%2:40:02::","synset_id":"save.v.04"},{"index":5,"sense_key":"save%2:40:01::","synset_id":"save.v.05"},{"index":6,"sense_key":"save%2:30:00::","synset_id":"save.v.06"},{"index":7,"sense_key":"save%2:41:02::","synset_id":"deliver.v.08"},{"index":8,"sense_key":"save%2:41:05::","synset_id":"spare.v.01"},{"index":9,"sense_key":"save%2:40:04::","synset_id":"save.v.09"},{"index":10,"sense_key":"save%2:40:03::","synset_id":"keep_open.v.01"},{"index":11,"sense_key":"save%2:32:04::","synset_id":"write.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"save%2:32:04::","call_ids":["semeval2015.d001.s044.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"save%2:32:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":938,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001996,"input_uncached_usd":0.0018759999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1543681250186637}
{"item_id":"semeval2015.d001.s044.t001","gold_sense_keys":["log%1:10:02::"],"candidates":[{"index":1,"sense_key":"log%1:27:01::","synset_id":"log.n.01"},{"index":2,"sense_key":"log%1:10:01::","synset_id":"logarithm.n.01"},{"index":3,"sense_key":"log%1:10:02::","synset_id":"log.n.03"},{"index":4,"sense_key":"log%1:10:00::","synset_id":"log.n.04"},{"index":5,"sense_key":"log%1:06:01::","synset_id":"log.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"log%1:10:02::","call_ids":["semeval2015.d001.s044.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"log%1:10:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":586,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001892,"input_uncached_usd":0.0011719999999999999,"input_cached_usd":0.0,"output_usd":0.00072,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0685040839016438}
{"item_id":"semeval2015.d001.s044.t002","gold_sense_keys":["result%1:10:00::","result%1:11:00::"],"candidates":[{"index":1,"sense_key":"result%1:19:00::","synset_id":"consequence.n.01"},{"index":2,"sense_key":"result%1:10:00::","synset_id":"solution.n.02"},{"index":3,"sense_key":"result%1:11:00::","synset_id":"result.n.03"},{"index":4,"sense_key":"result%1:10:01::","synset_id":"resultant_role.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"result%1:11:00::","call_ids":["semeval2015.d001.s044.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"result%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":668,"cached_input_tokens":0,"output_tokens":131,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002646,"input_uncached_usd":0.001336,"input_cached_usd":0.0,"output_usd":0.0013100000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.234209999907762}
{"item_id":"semeval2015.d001.s044.t004","gold_sense_keys":["able%3:00:00::"],"candidates":[{"index":1,"sense_key":"able%3:00:00::","synset_id":"able.a.01"},{"index":2,"sense_key":"able%5:00:00:competent:00","synset_id":"able.s.01"},{"index":3,"sense_key":"able%5:00:00:capable:00","synset_id":"able.s.02"},{"index":4,"sense_key":"able%5:00:00:fit:01","synset_id":"able.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"able%3:00:00::","call_ids":["semeval2015.d001.s044.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"able%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":604,"cached_input_tokens":0,"output_tokens":118,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002388,"input_uncached_usd":0.0012079999999999999,"input_cached_usd":0.0,"output_usd":0.00118,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":11.40777520800475}
{"item_id":"semeval2015.d001.s044.t005","gold_sense_keys":["print%2:36:00::"],"candidates":[{"index":1,"sense_key":"print%2:36:02::","synset_id":"print.v.01"},{"index":2,"sense_key":"print%2:36:03::","synset_id":"print.v.02"},{"index":3,"sense_key":"print%2:36:01::","synset_id":"print.v.03"},{"index":4,"sense_key":"print%2:36:00::","synset_id":"print.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"print%2:36:00::","call_ids":["semeval2015.d001.s044.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"print%2:36:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":512,"cached_input_tokens":0,"output_tokens":145,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002474,"input_uncached_usd":0.001024,"input_cached_usd":0.0,"output_usd":0.0014500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2846272920724005}
{"item_id":"semeval2015.d001.s044.t006","gold_sense_keys":["publish%2:32:00::"],"candidates":[{"index":1,"sense_key":"publish%2:36:01::","synset_id":"print.v.01"},{"index":2,"sense_key":"publish%2:32:00::","synset_id":"publish.v.02"},{"index":3,"sense_key":"publish%2:36:00::","synset_id":"publish.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"publish%2:32:00::","call_ids":["semeval2015.d001.s044.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"publish%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":551,"cached_input_tokens":0,"output_tokens":138,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002482,"input_uncached_usd":0.001102,"input_cached_usd":0.0,"output_usd":0.0013800000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.3363342500524595}
{"item_id":"semeval2015.d001.s045.t000","gold_sense_keys":["quit%2:42:04::"],"candidates":[{"index":1,"sense_key":"quit%2:42:04::","synset_id":"discontinue.v.01"},{"index":2,"sense_key":"quit%2:41:00::","synset_id":"leave_office.v.01"},{"index":3,"sense_key":"quit%2:38:00::","synset_id":"depart.v.04"},{"index":4,"sense_key":"quit%2:40:00::","synset_id":"foreswear.v.02"},{"index":5,"sense_key":"quit%2:33:05::","synset_id":"drop_out.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"quit%2:42:04::","call_ids":["semeval2015.d001.s045.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"quit%2:42:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":689,"cached_input_tokens":0,"output_tokens":150,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002878,"input_uncached_usd":0.0013779999999999999,"input_cached_usd":0.0,"output_usd":0.0015,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.6191780830267817}
{"item_id":"semeval2015.d001.s046.t000","gold_sense_keys":["program%1:10:02::"],"candidates":[{"index":1,"sense_key":"program%1:09:00::","synset_id":"plan.n.01"},{"index":2,"sense_key":"program%1:09:01::","synset_id":"program.n.02"},{"index":3,"sense_key":"program%1:10:01::","synset_id":"broadcast.n.02"},{"index":4,"sense_key":"program%1:10:04::","synset_id":"platform.n.02"},{"index":5,"sense_key":"program%1:10:00::","synset_id":"program.n.05"},{"index":6,"sense_key":"program%1:10:05::","synset_id":"course_of_study.n.01"},{"index":7,"sense_key":"program%1:10:02::","synset_id":"program.n.07"},{"index":8,"sense_key":"program%1:04:00::","synset_id":"program.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"program%1:10:02::","call_ids":["semeval2015.d001.s046.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"program%1:10:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":933,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001986,"input_uncached_usd":0.0018659999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6933039999566972}
{"item_id":"semeval2015.d001.s047.t000","gold_sense_keys":["add%2:30:00::"],"candidates":[{"index":1,"sense_key":"add%2:30:00::","synset_id":"add.v.01"},{"index":2,"sense_key":"add%2:32:01::","synset_id":"add.v.02"},{"index":3,"sense_key":"add%2:40:00::","synset_id":"lend.v.01"},{"index":4,"sense_key":"add%2:31:00::","synset_id":"add.v.04"},{"index":5,"sense_key":"add%2:32:00::","synset_id":"total.v.02"},{"index":6,"sense_key":"add%2:42:00::","synset_id":"add.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"add%2:30:00::","call_ids":["semeval2015.d001.s047.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"add%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":749,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001618,"input_uncached_usd":0.001498,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6498119999887422}
{"item_id":"semeval2015.d001.s047.t001","gold_sense_keys":["new%5:00:00:other:00"],"candidates":[{"index":1,"sense_key":"new%3:00:00::","synset_id":"new.a.01"},{"index":2,"sense_key":"new%5:00:00:original:00","synset_id":"fresh.s.03"},{"index":3,"sense_key":"new%5:00:00:inexperienced:00","synset_id":"raw.s.11"},{"index":4,"sense_key":"new%5:00:00:unprecedented:00","synset_id":"new.s.03"},{"index":5,"sense_key":"new%5:00:00:other:00","synset_id":"new.s.04"},{"index":6,"sense_key":"new%3:00:09::","synset_id":"new.a.06"},{"index":7,"sense_key":"new%5:00:02:original:00","synset_id":"newfangled.s.01"},{"index":8,"sense_key":"new%5:00:02:late:01","synset_id":"new.s.06"},{"index":9,"sense_key":"new%5:00:01:late:01","synset_id":"modern.s.04"},{"index":10,"sense_key":"new%5:00:00:early:02","synset_id":"new.s.08"},{"index":11,"sense_key":"new%5:00:00:unaccustomed:00","synset_id":"new.s.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"new%3:00:00::","call_ids":["semeval2015.d001.s047.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"new%3:00:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1063,"cached_input_tokens":0,"output_tokens":74,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002866,"input_uncached_usd":0.002126,"input_cached_usd":0.0,"output_usd":0.0007400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.569338082917966}
{"item_id":"semeval2015.d001.s047.t003","gold_sense_keys":["do%2:36:01::","do%2:41:04::"],"candidates":[{"index":1,"sense_key":"do%2:41:01::","synset_id":"make.v.01"},{"index":2,"sense_key":"do%2:36:01::","synset_id":"perform.v.01"},{"index":3,"sense_key":"do%2:41:04::","synset_id":"do.v.03"},{"index":4,"sense_key":"do%2:42:00::","synset_id":"do.v.04"},{"index":5,"sense_key":"do%2:36:02::","synset_id":"cause.v.01"},{"index":6,"sense_key":"do%2:41:02::","synset_id":"practice.v.01"},{"index":7,"sense_key":"do%2:42:01::","synset_id":"suffice.v.01"},{"index":8,"sense_key":"do%2:36:00::","synset_id":"do.v.08"},{"index":9,"sense_key":"do%2:29:09::","synset_id":"act.v.02"},{"index":10,"sense_key":"do%2:42:02::","synset_id":"serve.v.09"},{"index":11,"sense_key":"do%2:41:03::","synset_id":"do.v.11"},{"index":12,"sense_key":"do%2:29:04::","synset_id":"dress.v.16"},{"index":13,"sense_key":"do%2:38:00::","synset_id":"do.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"do%2:36:01::","call_ids":["semeval2015.d001.s047.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"do%2:36:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1183,"cached_input_tokens":0,"output_tokens":105,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003416,"input_uncached_usd":0.002366,"input_cached_usd":0.0,"output_usd":0.0010500000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7761174580082297}
{"item_id":"semeval2015.d001.s047.t004","gold_sense_keys":["go%2:38:00::"],"candidates":[{"index":1,"sense_key":"go%2:38:00::","synset_id":"travel.v.01"},{"index":2,"sense_key":"go%2:41:00::","synset_id":"go.v.02"},{"index":3,"sense_key":"go%2:38:01::","synset_id":"go.v.03"},{"index":4,"sense_key":"go%2:30:04::","synset_id":"become.v.01"},{"index":5,"sense_key":"go%2:42:06::","synset_id":"go.v.05"},{"index":6,"sense_key":"go%2:42:02::","synset_id":"run.v.05"},{"index":7,"sense_key":"go%2:42:00::","synset_id":"run.v.03"},{"index":8,"sense_key":"go%2:30:01::","synset_id":"proceed.v.04"},{"index":9,"sense_key":"go%2:30:08::","synset_id":"go.v.09"},{"index":10,"sense_key":"go%2:42:07::","synset_id":"go.v.10"},{"index":11,"sense_key":"go%2:39:00::","synset_id":"sound.v.02"},{"index":12,"sense_key":"go%2:35:00::","synset_id":"function.v.01"},{"index":13,"sense_key":"go%2:34:00::","synset_id":"run_low.v.01"},{"index":14,"sense_key":"go%2:30:02::","synset_id":"move.v.13"},{"index":15,"sense_key":"go%2:42:14::","synset_id":"survive.v.01"},{"index":16,"sense_key":"go%2:42:03::","synset_id":"go.v.16"},{"index":17,"sense_key":"go%2:30:00::","synset_id":"die.v.01"},{"index":18,"sense_key":"go%2:42:04::","synset_id":"belong.v.03"},{"index":19,"sense_key":"go%2:42:10::","synset_id":"go.v.19"},{"index":20,"sense_key":"go%2:38:09::","synset_id":"start.v.09"},{"index":21,"sense_key":"go%2:33:00::","synset_id":"move.v.15"},{"index":22,"sense_key":"go%2:42:13::","synset_id":"go.v.22"},{"index":23,"sense_key":"go%2:42:12::","synset_id":"go.v.23"},{"index":24,"sense_key":"go%2:42:11::","synset_id":"blend.v.02"},{"index":25,"sense_key":"go%2:42:08::","synset_id":"go.v.25"},{"index":26,"sense_key":"go%2:42:05::","synset_id":"fit.v.02"},{"index":27,"sense_key":"go%2:35:13::","synset_id":"rifle.v.02"},{"index":28,"sense_key":"go%2:34:09::","synset_id":"go.v.28"},{"index":29,"sense_key":"go%2:31:12::","synset_id":"plump.v.04"},{"index":30,"sense_key":"go%2:30:03::","synset_id":"fail.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"go%2:38:00::","call_ids":["semeval2015.d001.s047.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"go%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2275,"cached_input_tokens":0,"output_tokens":105,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.005600000000000001,"input_uncached_usd":0.00455,"input_cached_usd":0.0,"output_usd":0.0010500000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.822077083052136}
{"item_id":"semeval2015.d001.s047.t006","gold_sense_keys":["add%2:30:00::"],"candidates":[{"index":1,"sense_key":"add%2:30:00::","synset_id":"add.v.01"},{"index":2,"sense_key":"add%2:32:01::","synset_id":"add.v.02"},{"index":3,"sense_key":"add%2:40:00::","synset_id":"lend.v.01"},{"index":4,"sense_key":"add%2:31:00::","synset_id":"add.v.04"},{"index":5,"sense_key":"add%2:32:00::","synset_id":"total.v.02"},{"index":6,"sense_key":"add%2:42:00::","synset_id":"add.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"add%2:30:00::","call_ids":["semeval2015.d001.s047.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"add%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":750,"cached_input_tokens":0,"output_tokens":81,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00231,"input_uncached_usd":0.0015,"input_cached_usd":0.0,"output_usd":0.0008100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6632391659077257}
{"item_id":"semeval2015.d001.s047.t007","gold_sense_keys":["add%2:30:00::"],"candidates":[{"index":1,"sense_key":"add%2:30:00::","synset_id":"add.v.01"},{"index":2,"sense_key":"add%2:32:01::","synset_id":"add.v.02"},{"index":3,"sense_key":"add%2:40:00::","synset_id":"lend.v.01"},{"index":4,"sense_key":"add%2:31:00::","synset_id":"add.v.04"},{"index":5,"sense_key":"add%2:32:00::","synset_id":"total.v.02"},{"index":6,"sense_key":"add%2:42:00::","synset_id":"add.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"add%2:30:00::","call_ids":["semeval2015.d001.s047.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"add%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":749,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001618,"input_uncached_usd":0.001498,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6398612499469891}
{"item_id":"semeval2015.d001.s047.t008","gold_sense_keys":["new%5:00:00:other:00"],"candidates":[{"index":1,"sense_key":"new%3:00:00::","synset_id":"new.a.01"},{"index":2,"sense_key":"new%5:00:00:original:00","synset_id":"fresh.s.03"},{"index":3,"sense_key":"new%5:00:00:inexperienced:00","synset_id":"raw.s.11"},{"index":4,"sense_key":"new%5:00:00:unprecedented:00","synset_id":"new.s.03"},{"index":5,"sense_key":"new%5:00:00:other:00","synset_id":"new.s.04"},{"index":6,"sense_key":"new%3:00:09::","synset_id":"new.a.06"},{"index":7,"sense_key":"new%5:00:02:original:00","synset_id":"newfangled.s.01"},{"index":8,"sense_key":"new%5:00:02:late:01","synset_id":"new.s.06"},{"index":9,"sense_key":"new%5:00:01:late:01","synset_id":"modern.s.04"},{"index":10,"sense_key":"new%5:00:00:early:02","synset_id":"new.s.08"},{"index":11,"sense_key":"new%5:00:00:unaccustomed:00","synset_id":"new.s.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"new%3:00:00::","call_ids":["semeval2015.d001.s047.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"new%3:00:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1063,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002796,"input_uncached_usd":0.002126,"input_cached_usd":0.0,"output_usd":0.00067,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8782112919725478}
{"item_id":"semeval2015.d001.s047.t009","gold_sense_keys":["function%1:24:00::"],"candidates":[{"index":1,"sense_key":"function%1:24:00::","synset_id":"function.n.01"},{"index":2,"sense_key":"function%1:07:00::","synset_id":"function.n.02"},{"index":3,"sense_key":"function%1:04:00::","synset_id":"function.n.03"},{"index":4,"sense_key":"function%1:24:01::","synset_id":"function.n.04"},{"index":5,"sense_key":"function%1:14:00::","synset_id":"function.n.05"},{"index":6,"sense_key":"function%1:11:00::","synset_id":"affair.n.03"},{"index":7,"sense_key":"function%1:10:00::","synset_id":"routine.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"function%1:24:00::","call_ids":["semeval2015.d001.s047.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"function%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":838,"cached_input_tokens":0,"output_tokens":74,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002416,"input_uncached_usd":0.001676,"input_cached_usd":0.0,"output_usd":0.0007400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.514149541966617}
{"item_id":"semeval2015.d001.s048.t000","gold_sense_keys":["then%4:02:00::"],"candidates":[{"index":1,"sense_key":"then%4:02:00::","synset_id":"then.r.01"},{"index":2,"sense_key":"then%4:02:02::","synset_id":"then.r.02"},{"index":3,"sense_key":"then%4:02:01::","synset_id":"then.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"then%4:02:00::","call_ids":["semeval2015.d001.s048.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"then%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":536,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0011920000000000001,"input_uncached_usd":0.001072,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":8.214969250024296}
{"item_id":"semeval2015.d001.s048.t001","gold_sense_keys":["focus%1:09:00::"],"candidates":[{"index":1,"sense_key":"focus%1:09:00::","synset_id":"focus.n.01"},{"index":2,"sense_key":"focus%1:07:01::","synset_id":"focus.n.02"},{"index":3,"sense_key":"focus%1:07:03::","synset_id":"focus.n.03"},{"index":4,"sense_key":"focus%1:26:00::","synset_id":"focus.n.04"},{"index":5,"sense_key":"focus%1:26:01::","synset_id":"stress.n.03"},{"index":6,"sense_key":"focus%1:19:00::","synset_id":"focus.n.06"},{"index":7,"sense_key":"focus%1:15:00::","synset_id":"focus.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"focus%1:09:00::","call_ids":["semeval2015.d001.s048.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"focus%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":785,"cached_input_tokens":0,"output_tokens":78,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00235,"input_uncached_usd":0.00157,"input_cached_usd":0.0,"output_usd":0.0007800000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.733652999973856}
{"item_id":"semeval2015.d001.s048.t003","gold_sense_keys":["text%1:10:00::"],"candidates":[{"index":1,"sense_key":"text%1:10:00::","synset_id":"text.n.01"},{"index":2,"sense_key":"text%1:10:03::","synset_id":"text.n.02"},{"index":3,"sense_key":"text%1:10:02::","synset_id":"textbook.n.01"},{"index":4,"sense_key":"text%1:10:01::","synset_id":"text.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"text%1:10:00::","call_ids":["semeval2015.d001.s048.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"text%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":660,"cached_input_tokens":0,"output_tokens":69,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00201,"input_uncached_usd":0.00132,"input_cached_usd":0.0,"output_usd":0.0006900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.674658542033285}
{"item_id":"semeval2015.d001.s048.t005","gold_sense_keys":["type%2:32:00::"],"candidates":[{"index":1,"sense_key":"type%2:32:00::","synset_id":"type.v.01"},{"index":2,"sense_key":"type%2:31:00::","synset_id":"type.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"type%2:32:00::","call_ids":["semeval2015.d001.s048.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"type%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":481,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001082,"input_uncached_usd":0.000962,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1028070830507204}
{"item_id":"semeval2015.d001.s048.t006","gold_sense_keys":["function%1:24:00::"],"candidates":[{"index":1,"sense_key":"function%1:24:00::","synset_id":"function.n.01"},{"index":2,"sense_key":"function%1:07:00::","synset_id":"function.n.02"},{"index":3,"sense_key":"function%1:04:00::","synset_id":"function.n.03"},{"index":4,"sense_key":"function%1:24:01::","synset_id":"function.n.04"},{"index":5,"sense_key":"function%1:14:00::","synset_id":"function.n.05"},{"index":6,"sense_key":"function%1:11:00::","synset_id":"affair.n.03"},{"index":7,"sense_key":"function%1:10:00::","synset_id":"routine.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"function%1:24:00::","call_ids":["semeval2015.d001.s048.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"function%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":837,"cached_input_tokens":0,"output_tokens":78,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002454,"input_uncached_usd":0.001674,"input_cached_usd":0.0,"output_usd":0.0007800000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9944890839979053}
{"item_id":"semeval2015.d001.s049.t000","gold_sense_keys":["want%2:37:00::"],"candidates":[{"index":1,"sense_key":"want%2:37:00::","synset_id":"desire.v.01"},{"index":2,"sense_key":"want%2:34:00::","synset_id":"want.v.02"},{"index":3,"sense_key":"want%2:35:00::","synset_id":"want.v.03"},{"index":4,"sense_key":"want%2:31:01::","synset_id":"want.v.04"},{"index":5,"sense_key":"want%2:42:01::","synset_id":"want.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"want%2:37:00::","call_ids":["semeval2015.d001.s049.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"want%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":652,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0014240000000000001,"input_uncached_usd":0.001304,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6828614580444992}
{"item_id":"semeval2015.d001.s049.t001","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":2,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":3,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":4,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":5,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":6,"sense_key":"use%2:41:03::","synset_id":"use.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%2:34:01::","call_ids":["semeval2015.d001.s049.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":771,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001662,"input_uncached_usd":0.001542,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2107633750420064}
{"item_id":"semeval2015.d001.s049.t002","gold_sense_keys":["typical%3:00:00::","typical%5:00:00:normal:01"],"candidates":[{"index":1,"sense_key":"typical%3:00:00::","synset_id":"typical.a.01"},{"index":2,"sense_key":"typical%5:00:00:characteristic:00","synset_id":"distinctive.s.01"},{"index":3,"sense_key":"typical%5:00:00:normal:01","synset_id":"typical.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"typical%5:00:00:normal:01","call_ids":["semeval2015.d001.s049.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"typical%5:00:00:normal:01","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":601,"cached_input_tokens":0,"output_tokens":126,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0024619999999999998,"input_uncached_usd":0.001202,"input_cached_usd":0.0,"output_usd":0.00126,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8857156250160187}
{"item_id":"semeval2015.d001.s049.t004","gold_sense_keys":["necessary%3:00:00::"],"candidates":[{"index":1,"sense_key":"necessary%3:00:00::","synset_id":"necessary.a.01"},{"index":2,"sense_key":"necessary%5:00:00:inevitable:00","synset_id":"necessary.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"necessary%3:00:00::","call_ids":["semeval2015.d001.s049.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"necessary%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":476,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001072,"input_uncached_usd":0.0009519999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.262374416925013}
{"item_id":"semeval2015.d001.s049.t005","gold_sense_keys":["specify%2:32:02::"],"candidates":[{"index":1,"sense_key":"specify%2:32:03::","synset_id":"stipulate.v.01"},{"index":2,"sense_key":"specify%2:32:02::","synset_id":"specify.v.02"},{"index":3,"sense_key":"specify%2:42:00::","synset_id":"specify.v.03"},{"index":4,"sense_key":"specify%2:32:00::","synset_id":"specify.v.04"},{"index":5,"sense_key":"specify%2:31:02::","synset_id":"pin_down.v.01"},{"index":6,"sense_key":"specify%2:31:00::","synset_id":"intend.v.02"},{"index":7,"sense_key":"specify%2:31:05::","synset_id":"assign.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"specify%2:32:00::","call_ids":["semeval2015.d001.s049.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"specify%2:32:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":870,"cached_input_tokens":0,"output_tokens":138,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0031200000000000004,"input_uncached_usd":0.00174,"input_cached_usd":0.0,"output_usd":0.0013800000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0057351670693606}
{"item_id":"semeval2015.d001.s050.t001","gold_sense_keys":["function%1:24:00::"],"candidates":[{"index":1,"sense_key":"function%1:24:00::","synset_id":"function.n.01"},{"index":2,"sense_key":"function%1:07:00::","synset_id":"function.n.02"},{"index":3,"sense_key":"function%1:04:00::","synset_id":"function.n.03"},{"index":4,"sense_key":"function%1:24:01::","synset_id":"function.n.04"},{"index":5,"sense_key":"function%1:14:00::","synset_id":"function.n.05"},{"index":6,"sense_key":"function%1:11:00::","synset_id":"affair.n.03"},{"index":7,"sense_key":"function%1:10:00::","synset_id":"routine.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"function%1:24:00::","call_ids":["semeval2015.d001.s050.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"function%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":839,"cached_input_tokens":0,"output_tokens":69,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0023680000000000003,"input_uncached_usd":0.001678,"input_cached_usd":0.0,"output_usd":0.0006900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.071889542043209}
{"item_id":"semeval2015.d001.s050.t003","gold_sense_keys":["display%2:39:00::"],"candidates":[{"index":1,"sense_key":"display%2:39:00::","synset_id":"expose.v.03"},{"index":2,"sense_key":"display%2:41:00::","synset_id":"display.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"display%2:39:00::","call_ids":["semeval2015.d001.s050.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"display%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":505,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0011300000000000001,"input_uncached_usd":0.00101,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2849438750417903}
{"item_id":"semeval2015.d001.s050.t005","gold_sense_keys":["main%5:00:00:important:00"],"candidates":[{"index":1,"sense_key":"main%5:00:00:important:00","synset_id":"chief.s.01"},{"index":2,"sense_key":"main%3:00:02::","synset_id":"independent.a.03"},{"index":3,"sense_key":"main%5:00:00:intense:00","synset_id":"main.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"main%5:00:00:important:00","call_ids":["semeval2015.d001.s050.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"main%5:00:00:important:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":567,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0012540000000000001,"input_uncached_usd":0.001134,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8286760830087587}
{"item_id":"semeval2015.d001.s050.t006","gold_sense_keys":["window%1:06:03::"],"candidates":[{"index":1,"sense_key":"window%1:06:00::","synset_id":"window.n.01"},{"index":2,"sense_key":"window%1:06:01::","synset_id":"window.n.02"},{"index":3,"sense_key":"window%1:06:02::","synset_id":"window.n.03"},{"index":4,"sense_key":"window%1:17:00::","synset_id":"window.n.04"},{"index":5,"sense_key":"window%1:28:00::","synset_id":"window.n.05"},{"index":6,"sense_key":"window%1:06:04::","synset_id":"windowpane.n.01"},{"index":7,"sense_key":"window%1:06:05::","synset_id":"window.n.07"},{"index":8,"sense_key":"window%1:06:03::","synset_id":"window.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"window%1:06:03::","call_ids":["semeval2015.d001.s050.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"window%1:06:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":823,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001766,"input_uncached_usd":0.001646,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2155344580532983}
{"item_id":"semeval2015.d001.s051.t000","gold_sense_keys":["set%2:35:00::"],"candidates":[{"index":1,"sense_key":"set%2:35:00::","synset_id":"put.v.01"},{"index":2,"sense_key":"set%2:31:04::","synset_id":"determine.v.03"},{"index":3,"sense_key":"set%2:32:00::","synset_id":"specify.v.02"},{"index":4,"sense_key":"set%2:32:01::","synset_id":"set.v.04"},{"index":5,"sense_key":"set%2:30:01::","synset_id":"set.v.05"},{"index":6,"sense_key":"set%2:35:02::","synset_id":"set.v.06"},{"index":7,"sense_key":"set%2:30:09::","synset_id":"fix.v.12"},{"index":8,"sense_key":"set%2:30:02::","synset_id":"set.v.08"},{"index":9,"sense_key":"set%2:36:01::","synset_id":"set.v.09"},{"index":10,"sense_key":"set%2:38:00::","synset_id":"set.v.10"},{"index":11,"sense_key":"set%2:36:00::","synset_id":"arrange.v.06"},{"index":12,"sense_key":"set%2:35:09::","synset_id":"plant.v.01"},{"index":13,"sense_key":"set%2:36:12::","synset_id":"set.v.13"},{"index":14,"sense_key":"set%2:30:07::","synset_id":"jell.v.01"},{"index":15,"sense_key":"set%2:36:11::","synset_id":"typeset.v.01"},{"index":16,"sense_key":"set%2:35:15::","synset_id":"set.v.16"},{"index":17,"sense_key":"set%2:35:05::","synset_id":"set.v.17"},{"index":18,"sense_key":"set%2:35:04::","synset_id":"set.v.18"},{"index":19,"sense_key":"set%2:33:00::","synset_id":"sic.v.01"},{"index":20,"sense_key":"set%2:31:13::","synset_id":"place.v.11"},{"index":21,"sense_key":"set%2:30:06::","synset_id":"rig.v.04"},{"index":22,"sense_key":"set%2:30:12::","synset_id":"set_up.v.04"},{"index":23,"sense_key":"set%2:30:00::","synset_id":"adjust.v.01"},{"index":24,"sense_key":"set%2:29:06::","synset_id":"fructify.v.03"},{"index":25,"sense_key":"set%2:29:05::","synset_id":"dress.v.16"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"set%2:35:00::","call_ids":["semeval2015.d001.s051.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"set%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1760,"cached_input_tokens":0,"output_tokens":106,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00458,"input_uncached_usd":0.0035199999999999997,"input_cached_usd":0.0,"output_usd":0.0010600000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6838093340629712}
{"item_id":"semeval2015.d001.s051.t001","gold_sense_keys":["several%5:00:00:some:00"],"candidates":[{"index":1,"sense_key":"several%5:00:00:some:00","synset_id":"several.s.01"},{"index":2,"sense_key":"several%5:00:00:individual:00","synset_id":"respective.s.01"},{"index":3,"sense_key":"several%5:00:00:different:00","synset_id":"several.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"several%5:00:00:some:00","call_ids":["semeval2015.d001.s051.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"several%5:00:00:some:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":545,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0012100000000000001,"input_uncached_usd":0.00109,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1953308340162039}
{"item_id":"semeval2015.d001.s051.t003","gold_sense_keys":["same%3:00:02::"],"candidates":[{"index":1,"sense_key":"same%3:00:02::","synset_id":"same.a.01"},{"index":2,"sense_key":"same%3:00:00::","synset_id":"same.a.02"},{"index":3,"sense_key":"same%3:00:04::","synset_id":"like.a.02"},{"index":4,"sense_key":"same%5:00:00:unchanged:00","synset_id":"same.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"same%3:00:02::","call_ids":["semeval2015.d001.s051.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"same%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":600,"cached_input_tokens":0,"output_tokens":87,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00207,"input_uncached_usd":0.0012,"input_cached_usd":0.0,"output_usd":0.0008700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.779058332904242}
{"item_id":"semeval2015.d001.s052.t000","gold_sense_keys":["just%4:02:00::"],"candidates":[{"index":1,"sense_key":"just%4:02:00::","synset_id":"merely.r.01"},{"index":2,"sense_key":"just%4:02:05::","synset_id":"precisely.r.01"},{"index":3,"sense_key":"just%4:02:01::","synset_id":"just.r.03"},{"index":4,"sense_key":"just%4:02:03::","synset_id":"just.r.04"},{"index":5,"sense_key":"just%4:02:04::","synset_id":"barely.r.01"},{"index":6,"sense_key":"just%4:02:06::","synset_id":"just.r.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"just%4:02:00::","call_ids":["semeval2015.d001.s052.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"just%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":778,"cached_input_tokens":0,"output_tokens":130,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002856,"input_uncached_usd":0.0015559999999999999,"input_cached_usd":0.0,"output_usd":0.0013000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0540057501057163}
{"item_id":"semeval2015.d001.s052.t001","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":2,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":3,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":4,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":5,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":6,"sense_key":"use%2:41:03::","synset_id":"use.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%2:34:01::","call_ids":["semeval2015.d001.s052.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":767,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001654,"input_uncached_usd":0.001534,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":6.793133583036251}
{"item_id":"semeval2015.d001.s052.t002","gold_sense_keys":["add%2:30:00::"],"candidates":[{"index":1,"sense_key":"add%2:30:00::","synset_id":"add.v.01"},{"index":2,"sense_key":"add%2:32:01::","synset_id":"add.v.02"},{"index":3,"sense_key":"add%2:40:00::","synset_id":"lend.v.01"},{"index":4,"sense_key":"add%2:31:00::","synset_id":"add.v.04"},{"index":5,"sense_key":"add%2:32:00::","synset_id":"total.v.02"},{"index":6,"sense_key":"add%2:42:00::","synset_id":"add.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"add%2:30:00::","call_ids":["semeval2015.d001.s052.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"add%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":758,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0020959999999999998,"input_uncached_usd":0.001516,"input_cached_usd":0.0,"output_usd":0.00058,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0383795408997685}
{"item_id":"semeval2015.d001.s052.t003","gold_sense_keys":["list%1:10:00::"],"candidates":[{"index":1,"sense_key":"list%1:10:00::","synset_id":"list.n.01"},{"index":2,"sense_key":"list%1:07:00::","synset_id":"tilt.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"list%1:10:00::","call_ids":["semeval2015.d001.s052.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"list%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":514,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0011480000000000001,"input_uncached_usd":0.001028,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":13.827188041992486}
{"item_id":"semeval2015.d001.s052.t004","gold_sense_keys":["mode%1:26:00::"],"candidates":[{"index":1,"sense_key":"mode%1:07:00::","synset_id":"manner.n.01"},{"index":2,"sense_key":"mode%1:26:00::","synset_id":"mode.n.02"},{"index":3,"sense_key":"mode%1:24:00::","synset_id":"modality.n.01"},{"index":4,"sense_key":"mode%1:24:02::","synset_id":"mood.n.03"},{"index":5,"sense_key":"mode%1:10:00::","synset_id":"mode.n.05"},{"index":6,"sense_key":"mode%1:09:00::","synset_id":"mode.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"mode%1:26:00::","call_ids":["semeval2015.d001.s052.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"mode%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":708,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001536,"input_uncached_usd":0.001416,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0850664590252563}
{"item_id":"semeval2015.d001.s053.t000","gold_sense_keys":["set%2:32:00::"],"candidates":[{"index":1,"sense_key":"set%2:35:00::","synset_id":"put.v.01"},{"index":2,"sense_key":"set%2:31:04::","synset_id":"determine.v.03"},{"index":3,"sense_key":"set%2:32:00::","synset_id":"specify.v.02"},{"index":4,"sense_key":"set%2:32:01::","synset_id":"set.v.04"},{"index":5,"sense_key":"set%2:30:01::","synset_id":"set.v.05"},{"index":6,"sense_key":"set%2:35:02::","synset_id":"set.v.06"},{"index":7,"sense_key":"set%2:30:09::","synset_id":"fix.v.12"},{"index":8,"sense_key":"set%2:30:02::","synset_id":"set.v.08"},{"index":9,"sense_key":"set%2:36:01::","synset_id":"set.v.09"},{"index":10,"sense_key":"set%2:38:00::","synset_id":"set.v.10"},{"index":11,"sense_key":"set%2:36:00::","synset_id":"arrange.v.06"},{"index":12,"sense_key":"set%2:35:09::","synset_id":"plant.v.01"},{"index":13,"sense_key":"set%2:36:12::","synset_id":"set.v.13"},{"index":14,"sense_key":"set%2:30:07::","synset_id":"jell.v.01"},{"index":15,"sense_key":"set%2:36:11::","synset_id":"typeset.v.01"},{"index":16,"sense_key":"set%2:35:15::","synset_id":"set.v.16"},{"index":17,"sense_key":"set%2:35:05::","synset_id":"set.v.17"},{"index":18,"sense_key":"set%2:35:04::","synset_id":"set.v.18"},{"index":19,"sense_key":"set%2:33:00::","synset_id":"sic.v.01"},{"index":20,"sense_key":"set%2:31:13::","synset_id":"place.v.11"},{"index":21,"sense_key":"set%2:30:06::","synset_id":"rig.v.04"},{"index":22,"sense_key":"set%2:30:12::","synset_id":"set_up.v.04"},{"index":23,"sense_key":"set%2:30:00::","synset_id":"adjust.v.01"},{"index":24,"sense_key":"set%2:29:06::","synset_id":"fructify.v.03"},{"index":25,"sense_key":"set%2:29:05::","synset_id":"dress.v.16"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"set%2:32:00::","call_ids":["semeval2015.d001.s053.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"set%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1714,"cached_input_tokens":0,"output_tokens":133,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004758,"input_uncached_usd":0.003428,"input_cached_usd":0.0,"output_usd":0.00133,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.040445666993037}
{"item_id":"semeval2015.d001.s053.t002","gold_sense_keys":["color%1:07:00::","color%1:07:01::"],"candidates":[{"index":1,"sense_key":"color%1:07:00::","synset_id":"color.n.01"},{"index":2,"sense_key":"color%1:07:03::","synset_id":"color.n.02"},{"index":3,"sense_key":"color%1:07:02::","synset_id":"color.n.03"},{"index":4,"sense_key":"color%1:14:00::","synset_id":"color.n.04"},{"index":5,"sense_key":"color%1:07:04::","synset_id":"semblance.n.01"},{"index":6,"sense_key":"color%1:27:00::","synset_id":"coloring_material.n.01"},{"index":7,"sense_key":"color%1:09:01::","synset_id":"color.n.07"},{"index":8,"sense_key":"color%1:07:01::","synset_id":"color.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"color%1:07:00::","call_ids":["semeval2015.d001.s053.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"color%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":914,"cached_input_tokens":0,"output_tokens":122,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0030480000000000004,"input_uncached_usd":0.001828,"input_cached_usd":0.0,"output_usd":0.0012200000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.471569334040396}
{"item_id":"semeval2015.d002.s000.t000","gold_sense_keys":["foundation%1:14:00::"],"candidates":[{"index":1,"sense_key":"foundation%1:24:00::","synset_id":"foundation.n.01"},{"index":2,"sense_key":"foundation%1:14:00::","synset_id":"foundation.n.02"},{"index":3,"sense_key":"foundation%1:06:00::","synset_id":"foundation.n.03"},{"index":4,"sense_key":"foundation%1:09:01::","synset_id":"foundation.n.04"},{"index":5,"sense_key":"foundation%1:09:00::","synset_id":"basis.n.02"},{"index":6,"sense_key":"foundation%1:06:01::","synset_id":"foundation_garment.n.01"},{"index":7,"sense_key":"foundation%1:04:00::","synset_id":"initiation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"foundation%1:14:00::","call_ids":["semeval2015.d002.s000.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"foundation%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":807,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001734,"input_uncached_usd":0.001614,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8693007909459993}
{"item_id":"semeval2015.d002.s000.t002","gold_sense_keys":["involve%2:42:04::"],"candidates":[{"index":1,"sense_key":"involve%2:42:01::","synset_id":"involve.v.01"},{"index":2,"sense_key":"involve%2:42:04::","synset_id":"involve.v.02"},{"index":3,"sense_key":"involve%2:42:00::","synset_id":"imply.v.05"},{"index":4,"sense_key":"involve%2:42:07::","synset_id":"necessitate.v.01"},{"index":5,"sense_key":"involve%2:42:02::","synset_id":"involve.v.05"},{"index":6,"sense_key":"involve%2:31:00::","synset_id":"involve.v.06"},{"index":7,"sense_key":"involve%2:30:00::","synset_id":"involve.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"involve%2:42:04::","call_ids":["semeval2015.d002.s000.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"involve%2:42:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":690,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0015,"input_uncached_usd":0.00138,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.560841333004646}
{"item_id":"semeval2015.d002.s000.t003","gold_sense_keys":["organise%2:36:00::","organise%2:41:00::"],"candidates":[{"index":1,"sense_key":"organise%2:30:00::","synset_id":"organize.v.04"},{"index":2,"sense_key":"organise%2:41:01::","synset_id":"form.v.01"},{"index":3,"sense_key":"organise%2:41:02::","synset_id":"unionize.v.02"},{"index":4,"sense_key":"organise%2:41:00::","synset_id":"organize.v.02"},{"index":5,"sense_key":"organise%2:36:00::","synset_id":"organize.v.05"},{"index":6,"sense_key":"organise%2:31:00::","synset_id":"mastermind.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"organise%2:36:00::","call_ids":["semeval2015.d002.s000.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"organise%2:36:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":684,"cached_input_tokens":0,"output_tokens":99,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002358,"input_uncached_usd":0.0013679999999999999,"input_cached_usd":0.0,"output_usd":0.00099,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.013431458035484}
{"item_id":"semeval2015.d002.s000.t004","gold_sense_keys":["conference%1:14:00::"],"candidates":[{"index":1,"sense_key":"conference%1:14:00::","synset_id":"conference.n.01"},{"index":2,"sense_key":"conference%1:14:01::","synset_id":"league.n.01"},{"index":3,"sense_key":"conference%1:10:00::","synset_id":"conference.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"conference%1:14:00::","call_ids":["semeval2015.d002.s000.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"conference%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":466,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001052,"input_uncached_usd":0.000932,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5022825839696452}
{"item_id":"semeval2015.d002.s000.t005","gold_sense_keys":["deal%2:32:08::"],"candidates":[{"index":1,"sense_key":"deal%2:32:08::","synset_id":"cover.v.05"},{"index":2,"sense_key":"deal%2:31:10::","synset_id":"consider.v.03"},{"index":3,"sense_key":"deal%2:41:09::","synset_id":"deal.v.03"},{"index":4,"sense_key":"deal%2:41:00::","synset_id":"cope.v.01"},{"index":5,"sense_key":"deal%2:40:01::","synset_id":"distribute.v.01"},{"index":6,"sense_key":"deal%2:40:00::","synset_id":"deal.v.06"},{"index":7,"sense_key":"deal%2:41:13::","synset_id":"manage.v.02"},{"index":8,"sense_key":"deal%2:41:04::","synset_id":"deal.v.08"},{"index":9,"sense_key":"deal%2:40:02::","synset_id":"deal.v.09"},{"index":10,"sense_key":"deal%2:41:02::","synset_id":"conduct.v.01"},{"index":11,"sense_key":"deal%2:40:07::","synset_id":"share.v.04"},{"index":12,"sense_key":"deal%2:40:06::","synset_id":"deal.v.12"},{"index":13,"sense_key":"deal%2:40:11::","synset_id":"deal.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"deal%2:32:08::","call_ids":["semeval2015.d002.s000.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"deal%2:32:08::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1120,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0023599999999999997,"input_uncached_usd":0.00224,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.494659333024174}
{"item_id":"semeval2015.d002.s000.t006","gold_sense_keys":["critical%5:00:00:crucial:00"],"candidates":[{"index":1,"sense_key":"critical%3:00:01::","synset_id":"critical.a.01"},{"index":2,"sense_key":"critical%3:00:04::","synset_id":"critical.a.02"},{"index":3,"sense_key":"critical%3:00:02::","synset_id":"critical.a.03"},{"index":4,"sense_key":"critical%5:00:00:indispensable:00","synset_id":"critical.s.01"},{"index":5,"sense_key":"critical%5:00:00:crucial:00","synset_id":"critical.s.02"},{"index":6,"sense_key":"critical%3:00:03::","synset_id":"critical.a.06"},{"index":7,"sense_key":"critical%3:01:00::","synset_id":"critical.a.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"critical%5:00:00:crucial:00","call_ids":["semeval2015.d002.s000.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"critical%5:00:00:crucial:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":767,"cached_input_tokens":0,"output_tokens":205,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0035840000000000004,"input_uncached_usd":0.001534,"input_cached_usd":0.0,"output_usd":0.00205,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.9213222078979015}
{"item_id":"semeval2015.d002.s000.t007","gold_sense_keys":["challenge%1:26:00::"],"candidates":[{"index":1,"sense_key":"challenge%1:26:00::","synset_id":"challenge.n.01"},{"index":2,"sense_key":"challenge%1:10:00::","synset_id":"challenge.n.02"},{"index":3,"sense_key":"challenge%1:10:01::","synset_id":"challenge.n.03"},{"index":4,"sense_key":"challenge%1:10:02::","synset_id":"challenge.n.04"},{"index":5,"sense_key":"challenge%1:10:03::","synset_id":"challenge.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"challenge%1:26:00::","call_ids":["semeval2015.d002.s000.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"challenge%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":545,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0012100000000000001,"input_uncached_usd":0.00109,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0087041249498725}
{"item_id":"semeval2015.d002.s001.t000","gold_sense_keys":["participant%1:18:00::"],"candidates":[{"index":1,"sense_key":"participant%1:18:00::","synset_id":"participant.n.01"},{"index":2,"sense_key":"participant%1:18:01::","synset_id":"player.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"participant%1:18:00::","call_ids":["semeval2015.d002.s001.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"participant%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":427,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0009739999999999999,"input_uncached_usd":0.0008539999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0521386669715866}
{"item_id":"semeval2015.d002.s001.t001","gold_sense_keys":["main%5:00:00:important:00"],"candidates":[{"index":1,"sense_key":"main%5:00:00:important:00","synset_id":"chief.s.01"},{"index":2,"sense_key":"main%3:00:02::","synset_id":"independent.a.03"},{"index":3,"sense_key":"main%5:00:00:intense:00","synset_id":"main.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"main%5:00:00:important:00","call_ids":["semeval2015.d002.s001.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"main%5:00:00:important:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":544,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001208,"input_uncached_usd":0.001088,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6332696659956127}
{"item_id":"semeval2015.d002.s001.t003","gold_sense_keys":["attend%2:42:00::"],"candidates":[{"index":1,"sense_key":"attend%2:42:00::","synset_id":"attend.v.01"},{"index":2,"sense_key":"attend%2:41:12::","synset_id":"attend.v.02"},{"index":3,"sense_key":"attend%2:42:01::","synset_id":"attend.v.03"},{"index":4,"sense_key":"attend%2:41:00::","synset_id":"serve.v.10"},{"index":5,"sense_key":"attend%2:39:00::","synset_id":"attend.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"attend%2:42:00::","call_ids":["semeval2015.d002.s001.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"attend%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":697,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001514,"input_uncached_usd":0.0013939999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1038547500502318}
{"item_id":"semeval2015.d002.s001.t004","gold_sense_keys":["debate%2:32:00::"],"candidates":[{"index":1,"sense_key":"debate%2:31:00::","synset_id":"debate.v.01"},{"index":2,"sense_key":"debate%2:32:02::","synset_id":"consider.v.05"},{"index":3,"sense_key":"debate%2:32:00::","synset_id":"debate.v.03"},{"index":4,"sense_key":"debate%2:32:01::","synset_id":"argue.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"debate%2:32:00::","call_ids":["semeval2015.d002.s001.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"debate%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":565,"cached_input_tokens":0,"output_tokens":114,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0022700000000000003,"input_uncached_usd":0.00113,"input_cached_usd":0.0,"output_usd":0.0011400000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.243022375041619}
{"item_id":"semeval2015.d002.s001.t005","gold_sense_keys":["issue%1:09:01::"],"candidates":[{"index":1,"sense_key":"issue%1:09:01::","synset_id":"issue.n.01"},{"index":2,"sense_key":"issue%1:10:00::","synset_id":"issue.n.02"},{"index":3,"sense_key":"issue%1:09:00::","synset_id":"topic.n.02"},{"index":4,"sense_key":"issue%1:04:01::","synset_id":"issue.n.04"},{"index":5,"sense_key":"issue%1:21:01::","synset_id":"issue.n.05"},{"index":6,"sense_key":"issue%1:21:00::","synset_id":"return.n.06"},{"index":7,"sense_key":"issue%1:19:00::","synset_id":"consequence.n.01"},{"index":8,"sense_key":"issue%1:18:00::","synset_id":"offspring.n.01"},{"index":9,"sense_key":"issue%1:11:00::","synset_id":"emergence.n.02"},{"index":10,"sense_key":"issue%1:06:00::","synset_id":"exit.n.01"},{"index":11,"sense_key":"issue%1:04:00::","synset_id":"issue.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"issue%1:09:01::","call_ids":["semeval2015.d002.s001.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"issue%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1129,"cached_input_tokens":0,"output_tokens":190,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004158,"input_uncached_usd":0.002258,"input_cached_usd":0.0,"output_usd":0.0019000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":4.434038124978542}
{"item_id":"semeval2015.d002.s001.t008","gold_sense_keys":["actor%1:18:01::"],"candidates":[{"index":1,"sense_key":"actor%1:18:00::","synset_id":"actor.n.01"},{"index":2,"sense_key":"actor%1:18:01::","synset_id":"actor.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"actor%1:18:01::","call_ids":["semeval2015.d002.s001.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"actor%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":467,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001054,"input_uncached_usd":0.0009339999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.117779582971707}
{"item_id":"semeval2015.d002.s001.t011","gold_sense_keys":["authority%1:18:01::"],"candidates":[{"index":1,"sense_key":"authority%1:07:00::","synset_id":"authority.n.01"},{"index":2,"sense_key":"authority%1:18:01::","synset_id":"authority.n.02"},{"index":3,"sense_key":"authority%1:18:00::","synset_id":"authority.n.03"},{"index":4,"sense_key":"authority%1:09:00::","synset_id":"assurance.n.01"},{"index":5,"sense_key":"authority%1:14:00::","synset_id":"agency.n.01"},{"index":6,"sense_key":"authority%1:07:02::","synset_id":"authority.n.06"},{"index":7,"sense_key":"authority%1:10:00::","synset_id":"authority.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"authority%1:14:00::","call_ids":["semeval2015.d002.s001.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"authority%1:14:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":833,"cached_input_tokens":0,"output_tokens":249,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004156,"input_uncached_usd":0.001666,"input_cached_usd":0.0,"output_usd":0.00249,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":16.671180457924493}
{"item_id":"semeval2015.d002.s001.t012","gold_sense_keys":["social%3:01:00::"],"candidates":[{"index":1,"sense_key":"social%3:01:00::","synset_id":"social.a.01"},{"index":2,"sense_key":"social%3:00:00::","synset_id":"social.a.02"},{"index":3,"sense_key":"social%3:01:02::","synset_id":"social.a.03"},{"index":4,"sense_key":"social%5:00:00:sociable:00","synset_id":"social.s.01"},{"index":5,"sense_key":"social%5:00:00:gregarious:00","synset_id":"social.s.02"},{"index":6,"sense_key":"social%5:00:00:friendly:01","synset_id":"social.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"social%3:01:00::","call_ids":["semeval2015.d002.s001.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"social%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":687,"cached_input_tokens":0,"output_tokens":69,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002064,"input_uncached_usd":0.001374,"input_cached_usd":0.0,"output_usd":0.0006900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4387009580386803}
{"item_id":"semeval2015.d002.s001.t013","gold_sense_keys":["voluntary%3:00:01::"],"candidates":[{"index":1,"sense_key":"voluntary%3:00:01::","synset_id":"voluntary.a.01"},{"index":2,"sense_key":"voluntary%3:00:02::","synset_id":"voluntary.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"voluntary%3:00:01::","call_ids":["semeval2015.d002.s001.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"voluntary%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":471,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001062,"input_uncached_usd":0.0009419999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5953137920005247}
{"item_id":"semeval2015.d002.s001.t014","gold_sense_keys":["community%1:14:00::"],"candidates":[{"index":1,"sense_key":"community%1:14:00::","synset_id":"community.n.01"},{"index":2,"sense_key":"community%1:21:00::","synset_id":"community.n.02"},{"index":3,"sense_key":"community%1:14:04::","synset_id":"community.n.03"},{"index":4,"sense_key":"community%1:26:00::","synset_id":"community.n.04"},{"index":5,"sense_key":"community%1:15:00::","synset_id":"residential_district.n.01"},{"index":6,"sense_key":"community%1:14:02::","synset_id":"community.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"community%1:14:00::","call_ids":["semeval2015.d002.s001.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"community%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":654,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001428,"input_uncached_usd":0.001308,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.110715416027233}
{"item_id":"semeval2015.d002.s001.t015","gold_sense_keys":["organisation%1:14:00::","organisation%1:14:01::"],"candidates":[{"index":1,"sense_key":"organisation%1:14:01::","synset_id":"administration.n.02"},{"index":2,"sense_key":"organisation%1:14:00::","synset_id":"organization.n.01"},{"index":3,"sense_key":"organisation%1:09:00::","synset_id":"arrangement.n.03"},{"index":4,"sense_key":"organisation%1:07:00::","synset_id":"organization.n.05"},{"index":5,"sense_key":"organisation%1:04:00::","synset_id":"organization.n.04"},{"index":6,"sense_key":"organisation%1:04:01::","synset_id":"organization.n.06"},{"index":7,"sense_key":"organisation%1:04:02::","synset_id":"constitution.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"organisation%1:14:00::","call_ids":["semeval2015.d002.s001.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"organisation%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":895,"cached_input_tokens":0,"output_tokens":87,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00266,"input_uncached_usd":0.00179,"input_cached_usd":0.0,"output_usd":0.0008700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9565568750258535}
{"item_id":"semeval2015.d002.s001.t017","gold_sense_keys":["service%1:04:08::"],"candidates":[{"index":1,"sense_key":"service%1:04:08::","synset_id":"service.n.01"},{"index":2,"sense_key":"service%1:04:00::","synset_id":"service.n.02"},{"index":3,"sense_key":"service%1:04:01::","synset_id":"service.n.03"},{"index":4,"sense_key":"service%1:14:05::","synset_id":"service.n.04"},{"index":5,"sense_key":"service%1:04:07::","synset_id":"service.n.05"},{"index":6,"sense_key":"service%1:14:00::","synset_id":"military_service.n.01"},{"index":7,"sense_key":"service%1:18:00::","synset_id":"service.n.07"},{"index":8,"sense_key":"service%1:07:00::","synset_id":"avail.n.01"},{"index":9,"sense_key":"service%1:06:00::","synset_id":"service.n.09"},{"index":10,"sense_key":"service%1:04:03::","synset_id":"servicing.n.01"},{"index":11,"sense_key":"service%1:04:09::","synset_id":"service.n.11"},{"index":12,"sense_key":"service%1:04:02::","synset_id":"serve.n.01"},{"index":13,"sense_key":"service%1:04:04::","synset_id":"service.n.13"},{"index":14,"sense_key":"service%1:04:05::","synset_id":"overhaul.n.01"},{"index":15,"sense_key":"service%1:04:06::","synset_id":"service.n.15"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"service%1:04:08::","call_ids":["semeval2015.d002.s001.t017__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"service%1:04:08::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1149,"cached_input_tokens":0,"output_tokens":118,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0034779999999999998,"input_uncached_usd":0.0022979999999999997,"input_cached_usd":0.0,"output_usd":0.00118,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":11.402811749954708}
{"item_id":"semeval2015.d002.s001.t018","gold_sense_keys":["user%1:18:00::"],"candidates":[{"index":1,"sense_key":"user%1:18:00::","synset_id":"user.n.01"},{"index":2,"sense_key":"user%1:18:01::","synset_id":"exploiter.n.01"},{"index":3,"sense_key":"user%1:18:02::","synset_id":"drug_user.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"user%1:18:00::","call_ids":["semeval2015.d002.s001.t018__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"user%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":483,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001086,"input_uncached_usd":0.000966,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0769375420641154}
{"item_id":"semeval2015.d002.s002.t002","gold_sense_keys":["case%1:11:00::"],"candidates":[{"index":1,"sense_key":"case%1:11:00::","synset_id":"case.n.01"},{"index":2,"sense_key":"case%1:26:00::","synset_id":"event.n.02"},{"index":3,"sense_key":"case%1:04:00::","synset_id":"lawsuit.n.01"},{"index":4,"sense_key":"case%1:09:00::","synset_id":"case.n.04"},{"index":5,"sense_key":"case%1:06:00::","synset_id":"case.n.05"},{"index":6,"sense_key":"case%1:18:03::","synset_id":"case.n.06"},{"index":7,"sense_key":"case%1:18:02::","synset_id":"subject.n.06"},{"index":8,"sense_key":"case%1:10:02::","synset_id":"case.n.08"},{"index":9,"sense_key":"case%1:10:01::","synset_id":"case.n.09"},{"index":10,"sense_key":"case%1:23:00::","synset_id":"case.n.10"},{"index":11,"sense_key":"case%1:10:00::","synset_id":"case.n.11"},{"index":12,"sense_key":"case%1:26:02::","synset_id":"case.n.12"},{"index":13,"sense_key":"case%1:18:04::","synset_id":"character.n.05"},{"index":14,"sense_key":"case%1:10:03::","synset_id":"font.n.01"},{"index":15,"sense_key":"case%1:08:00::","synset_id":"sheath.n.02"},{"index":16,"sense_key":"case%1:06:06::","synset_id":"shell.n.08"},{"index":17,"sense_key":"case%1:06:07::","synset_id":"casing.n.03"},{"index":18,"sense_key":"case%1:06:03::","synset_id":"case.n.18"},{"index":19,"sense_key":"case%1:06:02::","synset_id":"case.n.19"},{"index":20,"sense_key":"case%1:06:01::","synset_id":"case.n.20"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"case%1:11:00::","call_ids":["semeval2015.d002.s002.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"case%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1625,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00385,"input_uncached_usd":0.00325,"input_cached_usd":0.0,"output_usd":0.0006000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7648609169991687}
{"item_id":"semeval2015.d002.s002.t003","gold_sense_keys":["report%2:32:04::"],"candidates":[{"index":1,"sense_key":"report%2:32:00::","synset_id":"report.v.01"},{"index":2,"sense_key":"report%2:32:04::","synset_id":"report.v.02"},{"index":3,"sense_key":"report%2:32:03::","synset_id":"report.v.03"},{"index":4,"sense_key":"report%2:32:01::","synset_id":"report.v.04"},{"index":5,"sense_key":"report%2:32:13::","synset_id":"report.v.05"},{"index":6,"sense_key":"report%2:32:02::","synset_id":"report.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"report%2:32:04::","call_ids":["semeval2015.d002.s002.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"report%2:32:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":715,"cached_input_tokens":0,"output_tokens":93,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00236,"input_uncached_usd":0.0014299999999999998,"input_cached_usd":0.0,"output_usd":0.00093,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9436939170118421}
{"item_id":"semeval2015.d002.s002.t004","gold_sense_keys":["attempt%1:04:00::"],"candidates":[{"index":1,"sense_key":"attempt%1:04:00::","synset_id":"attempt.n.01"},{"index":2,"sense_key":"attempt%1:04:02::","synset_id":"attack.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"attempt%1:04:00::","call_ids":["semeval2015.d002.s002.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"attempt%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":525,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00117,"input_uncached_usd":0.00105,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5370907919714227}
{"item_id":"semeval2015.d002.s002.t005","gold_sense_keys":["involve%2:42:04::"],"candidates":[{"index":1,"sense_key":"involve%2:42:01::","synset_id":"involve.v.01"},{"index":2,"sense_key":"involve%2:42:04::","synset_id":"involve.v.02"},{"index":3,"sense_key":"involve%2:42:00::","synset_id":"imply.v.05"},{"index":4,"sense_key":"involve%2:42:07::","synset_id":"necessitate.v.01"},{"index":5,"sense_key":"involve%2:42:02::","synset_id":"involve.v.05"},{"index":6,"sense_key":"involve%2:31:00::","synset_id":"involve.v.06"},{"index":7,"sense_key":"involve%2:30:00::","synset_id":"involve.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"involve%2:42:04::","call_ids":["semeval2015.d002.s002.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"involve%2:42:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":748,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002086,"input_uncached_usd":0.001496,"input_cached_usd":0.0,"output_usd":0.00059,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":14.5857708749827}
{"item_id":"semeval2015.d002.s002.t006","gold_sense_keys":["user%1:18:00::"],"candidates":[{"index":1,"sense_key":"user%1:18:00::","synset_id":"user.n.01"},{"index":2,"sense_key":"user%1:18:01::","synset_id":"exploiter.n.01"},{"index":3,"sense_key":"user%1:18:02::","synset_id":"drug_user.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"user%1:18:00::","call_ids":["semeval2015.d002.s002.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"user%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":503,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001126,"input_uncached_usd":0.001006,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1091074999421835}
{"item_id":"semeval2015.d002.s002.t007","gold_sense_keys":["service%1:04:08::"],"candidates":[{"index":1,"sense_key":"service%1:04:08::","synset_id":"service.n.01"},{"index":2,"sense_key":"service%1:04:00::","synset_id":"service.n.02"},{"index":3,"sense_key":"service%1:04:01::","synset_id":"service.n.03"},{"index":4,"sense_key":"service%1:14:05::","synset_id":"service.n.04"},{"index":5,"sense_key":"service%1:04:07::","synset_id":"service.n.05"},{"index":6,"sense_key":"service%1:14:00::","synset_id":"military_service.n.01"},{"index":7,"sense_key":"service%1:18:00::","synset_id":"service.n.07"},{"index":8,"sense_key":"service%1:07:00::","synset_id":"avail.n.01"},{"index":9,"sense_key":"service%1:06:00::","synset_id":"service.n.09"},{"index":10,"sense_key":"service%1:04:03::","synset_id":"servicing.n.01"},{"index":11,"sense_key":"service%1:04:09::","synset_id":"service.n.11"},{"index":12,"sense_key":"service%1:04:02::","synset_id":"serve.n.01"},{"index":13,"sense_key":"service%1:04:04::","synset_id":"service.n.13"},{"index":14,"sense_key":"service%1:04:05::","synset_id":"overhaul.n.01"},{"index":15,"sense_key":"service%1:04:06::","synset_id":"service.n.15"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"service%1:04:08::","call_ids":["semeval2015.d002.s002.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"service%1:04:08::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1169,"cached_input_tokens":0,"output_tokens":122,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003558,"input_uncached_usd":0.002338,"input_cached_usd":0.0,"output_usd":0.0012200000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7545591670786962}
{"item_id":"semeval2015.d002.s002.t008","gold_sense_keys":["planning%1:04:00::","planning%1:04:02::"],"candidates":[{"index":1,"sense_key":"planning%1:04:00::","synset_id":"planning.n.01"},{"index":2,"sense_key":"planning%1:04:02::","synset_id":"planning.n.02"},{"index":3,"sense_key":"planning%1:09:00::","synset_id":"planning.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"planning%1:04:00::","call_ids":["semeval2015.d002.s002.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"planning%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":549,"cached_input_tokens":0,"output_tokens":218,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003278,"input_uncached_usd":0.001098,"input_cached_usd":0.0,"output_usd":0.00218,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":14.990495416917838}
{"item_id":"semeval2015.d002.s002.t009","gold_sense_keys":["involvement%1:04:00::"],"candidates":[{"index":1,"sense_key":"involvement%1:04:00::","synset_id":"engagement.n.07"},{"index":2,"sense_key":"involvement%1:24:00::","synset_id":"involvement.n.02"},{"index":3,"sense_key":"involvement%1:09:00::","synset_id":"interest.n.01"},{"index":4,"sense_key":"involvement%1:26:00::","synset_id":"affair.n.02"},{"index":5,"sense_key":"involvement%1:26:01::","synset_id":"participation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"involvement%1:04:00::","call_ids":["semeval2015.d002.s002.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"involvement%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":687,"cached_input_tokens":0,"output_tokens":136,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0027340000000000003,"input_uncached_usd":0.001374,"input_cached_usd":0.0,"output_usd":0.00136,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.4879668329376727}
{"item_id":"semeval2015.d002.s002.t010","gold_sense_keys":["service%1:04:08::"],"candidates":[{"index":1,"sense_key":"service%1:04:08::","synset_id":"service.n.01"},{"index":2,"sense_key":"service%1:04:00::","synset_id":"service.n.02"},{"index":3,"sense_key":"service%1:04:01::","synset_id":"service.n.03"},{"index":4,"sense_key":"service%1:14:05::","synset_id":"service.n.04"},{"index":5,"sense_key":"service%1:04:07::","synset_id":"service.n.05"},{"index":6,"sense_key":"service%1:14:00::","synset_id":"military_service.n.01"},{"index":7,"sense_key":"service%1:18:00::","synset_id":"service.n.07"},{"index":8,"sense_key":"service%1:07:00::","synset_id":"avail.n.01"},{"index":9,"sense_key":"service%1:06:00::","synset_id":"service.n.09"},{"index":10,"sense_key":"service%1:04:03::","synset_id":"servicing.n.01"},{"index":11,"sense_key":"service%1:04:09::","synset_id":"service.n.11"},{"index":12,"sense_key":"service%1:04:02::","synset_id":"serve.n.01"},{"index":13,"sense_key":"service%1:04:04::","synset_id":"service.n.13"},{"index":14,"sense_key":"service%1:04:05::","synset_id":"overhaul.n.01"},{"index":15,"sense_key":"service%1:04:06::","synset_id":"service.n.15"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"service%1:04:08::","call_ids":["semeval2015.d002.s002.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"service%1:04:08::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1169,"cached_input_tokens":0,"output_tokens":96,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0032979999999999997,"input_uncached_usd":0.002338,"input_cached_usd":0.0,"output_usd":0.0009600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8628201249521226}
{"item_id":"semeval2015.d002.s002.t011","gold_sense_keys":["provision%1:04:01::"],"candidates":[{"index":1,"sense_key":"provision%1:10:00::","synset_id":"provision.n.01"},{"index":2,"sense_key":"provision%1:04:01::","synset_id":"provision.n.02"},{"index":3,"sense_key":"provision%1:09:00::","synset_id":"planning.n.03"},{"index":4,"sense_key":"provision%1:21:00::","synset_id":"provision.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"provision%1:04:01::","call_ids":["semeval2015.d002.s002.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"provision%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":589,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0012980000000000001,"input_uncached_usd":0.001178,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.491442249971442}
{"item_id":"semeval2015.d002.s002.t012","gold_sense_keys":["find%2:32:00::","find%2:39:02::"],"candidates":[{"index":1,"sense_key":"find%2:40:02::","synset_id":"find.v.01"},{"index":2,"sense_key":"find%2:39:02::","synset_id":"detect.v.01"},{"index":3,"sense_key":"find%2:40:00::","synset_id":"find.v.03"},{"index":4,"sense_key":"find%2:32:00::","synset_id":"determine.v.01"},{"index":5,"sense_key":"find%2:31:10::","synset_id":"find.v.05"},{"index":6,"sense_key":"find%2:39:01::","synset_id":"witness.v.02"},{"index":7,"sense_key":"find%2:40:01::","synset_id":"line_up.v.02"},{"index":8,"sense_key":"find%2:36:00::","synset_id":"discover.v.03"},{"index":9,"sense_key":"find%2:31:09::","synset_id":"discover.v.04"},{"index":10,"sense_key":"find%2:40:03::","synset_id":"find.v.10"},{"index":11,"sense_key":"find%2:32:01::","synset_id":"rule.v.04"},{"index":12,"sense_key":"find%2:30:00::","synset_id":"receive.v.02"},{"index":13,"sense_key":"find%2:39:05::","synset_id":"find.v.13"},{"index":14,"sense_key":"find%2:40:15::","synset_id":"recover.v.01"},{"index":15,"sense_key":"find%2:38:10::","synset_id":"find.v.15"},{"index":16,"sense_key":"find%2:30:13::","synset_id":"find_oneself.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"find%2:32:00::","call_ids":["semeval2015.d002.s002.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"find%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1521,"cached_input_tokens":0,"output_tokens":206,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.005102000000000001,"input_uncached_usd":0.003042,"input_cached_usd":0.0,"output_usd":0.00206,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.40208283404354}
{"item_id":"semeval2015.d002.s002.t013","gold_sense_keys":["more%4:02:00::"],"candidates":[{"index":1,"sense_key":"more%4:02:00::","synset_id":"more.r.01"},{"index":2,"sense_key":"more%4:02:01::","synset_id":"more.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"more%4:02:00::","call_ids":["semeval2015.d002.s002.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"more%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":487,"cached_input_tokens":0,"output_tokens":44,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001414,"input_uncached_usd":0.0009739999999999999,"input_cached_usd":0.0,"output_usd":0.00044,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3567640410037711}
{"item_id":"semeval2015.d002.s002.t014","gold_sense_keys":["common%3:00:01::","common%5:00:00:familiar:02"],"candidates":[{"index":1,"sense_key":"common%3:00:02::","synset_id":"common.a.01"},{"index":2,"sense_key":"common%3:00:01::","synset_id":"common.a.02"},{"index":3,"sense_key":"common%5:00:00:shared:00","synset_id":"common.s.01"},{"index":4,"sense_key":"common%5:00:00:familiar:02","synset_id":"common.s.02"},{"index":5,"sense_key":"common%5:00:00:informal:02","synset_id":"common.s.03"},{"index":6,"sense_key":"common%5:00:00:lowborn:00","synset_id":"common.s.04"},{"index":7,"sense_key":"common%5:00:00:inferior:02","synset_id":"coarse.s.02"},{"index":8,"sense_key":"common%5:00:00:unrefined:01","synset_id":"coarse.s.01"},{"index":9,"sense_key":"common%5:00:00:ordinary:00","synset_id":"common.s.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"common%5:00:00:familiar:02","call_ids":["semeval2015.d002.s002.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"common%5:00:00:familiar:02","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1000,"cached_input_tokens":0,"output_tokens":133,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00333,"input_uncached_usd":0.002,"input_cached_usd":0.0,"output_usd":0.00133,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6341170830419287}
{"item_id":"semeval2015.d002.s003.t002","gold_sense_keys":["variety%1:07:00::","variety%1:14:01::"],"candidates":[{"index":1,"sense_key":"variety%1:14:01::","synset_id":"assortment.n.01"},{"index":2,"sense_key":"variety%1:07:00::","synset_id":"diverseness.n.01"},{"index":3,"sense_key":"variety%1:14:00::","synset_id":"variety.n.03"},{"index":4,"sense_key":"variety%1:10:00::","synset_id":"variety_show.n.01"},{"index":5,"sense_key":"variety%1:09:01::","synset_id":"kind.n.01"},{"index":6,"sense_key":"variety%1:07:01::","synset_id":"variety.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"variety%1:07:00::","call_ids":["semeval2015.d002.s003.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"variety%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":885,"cached_input_tokens":0,"output_tokens":106,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00283,"input_uncached_usd":0.0017699999999999999,"input_cached_usd":0.0,"output_usd":0.0010600000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":10.406144833075814}
{"item_id":"semeval2015.d002.s003.t003","gold_sense_keys":["different%3:00:02::","different%5:00:01:other:00"],"candidates":[{"index":1,"sense_key":"different%3:00:00::","synset_id":"different.a.01"},{"index":2,"sense_key":"different%5:00:00:other:00","synset_id":"different.s.01"},{"index":3,"sense_key":"different%5:00:00:unusual:00","synset_id":"different.s.02"},{"index":4,"sense_key":"different%3:00:02::","synset_id":"unlike.a.01"},{"index":5,"sense_key":"different%5:00:01:other:00","synset_id":"different.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"different%3:00:00::","call_ids":["semeval2015.d002.s003.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"different%3:00:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":718,"cached_input_tokens":0,"output_tokens":122,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0026560000000000004,"input_uncached_usd":0.001436,"input_cached_usd":0.0,"output_usd":0.0012200000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":5.270009667030536}
{"item_id":"semeval2015.d002.s003.t005","gold_sense_keys":["play%2:41:12::"],"candidates":[{"index":1,"sense_key":"play%2:33:00::","synset_id":"play.v.01"},{"index":2,"sense_key":"play%2:41:12::","synset_id":"play.v.02"},{"index":3,"sense_key":"play%2:36:12::","synset_id":"play.v.03"},{"index":4,"sense_key":"play%2:36:02::","synset_id":"act.v.03"},{"index":5,"sense_key":"play%2:41:03::","synset_id":"play.v.05"},{"index":6,"sense_key":"play%2:36:00::","synset_id":"play.v.06"},{"index":7,"sense_key":"play%2:36:01::","synset_id":"play.v.07"},{"index":8,"sense_key":"play%2:29:01::","synset_id":"act.v.05"},{"index":9,"sense_key":"play%2:38:00::","synset_id":"play.v.09"},{"index":10,"sense_key":"play%2:33:01::","synset_id":"play.v.10"},{"index":11,"sense_key":"play%2:41:00::","synset_id":"play.v.11"},{"index":12,"sense_key":"play%2:36:06::","synset_id":"play.v.12"},{"index":13,"sense_key":"play%2:36:11::","synset_id":"play.v.13"},{"index":14,"sense_key":"play%2:36:03::","synset_id":"play.v.14"},{"index":15,"sense_key":"play%2:33:03::","synset_id":"play.v.15"},{"index":16,"sense_key":"play%2:29:00::","synset_id":"play.v.16"},{"index":17,"sense_key":"play%2:41:02::","synset_id":"play.v.17"},{"index":18,"sense_key":"play%2:36:05::","synset_id":"play.v.18"},{"index":19,"sense_key":"play%2:35:00::","synset_id":"toy.v.02"},{"index":20,"sense_key":"play%2:34:13::","synset_id":"play.v.20"},{"index":21,"sense_key":"play%2:31:01::","synset_id":"dally.v.04"},{"index":22,"sense_key":"play%2:42:00::","synset_id":"play.v.22"},{"index":23,"sense_key":"play%2:41:13::","synset_id":"dally.v.01"},{"index":24,"sense_key":"play%2:38:03::","synset_id":"play.v.24"},{"index":25,"sense_key":"play%2:36:10::","synset_id":"act.v.10"},{"index":26,"sense_key":"play%2:36:04::","synset_id":"play.v.26"},{"index":27,"sense_key":"play%2:36:13::","synset_id":"bring.v.03"},{"index":28,"sense_key":"play%2:35:07::","synset_id":"play.v.28"},{"index":29,"sense_key":"play%2:33:07::","synset_id":"play.v.29"},{"index":30,"sense_key":"play%2:33:08::","synset_id":"bet.v.02"},{"index":31,"sense_key":"play%2:33:09::","synset_id":"play.v.31"},{"index":32,"sense_key":"play%2:33:13::","synset_id":"play.v.32"},{"index":33,"sense_key":"play%2:33:12::","synset_id":"play.v.33"},{"index":34,"sense_key":"play%2:33:02::","synset_id":"meet.v.10"},{"index":35,"sense_key":"play%2:29:04::","synset_id":"play.v.35"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"play%2:41:12::","call_ids":["semeval2015.d002.s003.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"play%2:41:12::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2562,"cached_input_tokens":0,"output_tokens":112,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.006244,"input_uncached_usd":0.005124,"input_cached_usd":0.0,"output_usd":0.0011200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":5.967538290889934}
{"item_id":"semeval2015.d002.s003.t006","gold_sense_keys":["public%5:00:00:common:02"],"candidates":[{"index":1,"sense_key":"public%3:00:00::","synset_id":"public.a.01"},{"index":2,"sense_key":"public%5:00:00:common:02","synset_id":"public.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"public%3:00:00::","call_ids":["semeval2015.d002.s003.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"public%3:00:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":529,"cached_input_tokens":0,"output_tokens":96,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002018,"input_uncached_usd":0.001058,"input_cached_usd":0.0,"output_usd":0.0009600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":4.9473977910820395}
{"item_id":"semeval2015.d002.s003.t007","gold_sense_keys":["welfare%1:04:00::"],"candidates":[{"index":1,"sense_key":"welfare%1:04:00::","synset_id":"social_welfare.n.01"},{"index":2,"sense_key":"welfare%1:07:00::","synset_id":"benefit.n.02"},{"index":3,"sense_key":"welfare%1:26:00::","synset_id":"wellbeing.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"welfare%1:04:00::","call_ids":["semeval2015.d002.s003.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"welfare%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":649,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001418,"input_uncached_usd":0.001298,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1642734160413966}
{"item_id":"semeval2015.d002.s004.t000","gold_sense_keys":["foundation%1:14:00::"],"candidates":[{"index":1,"sense_key":"foundation%1:24:00::","synset_id":"foundation.n.01"},{"index":2,"sense_key":"foundation%1:14:00::","synset_id":"foundation.n.02"},{"index":3,"sense_key":"foundation%1:06:00::","synset_id":"foundation.n.03"},{"index":4,"sense_key":"foundation%1:09:01::","synset_id":"foundation.n.04"},{"index":5,"sense_key":"foundation%1:09:00::","synset_id":"basis.n.02"},{"index":6,"sense_key":"foundation%1:06:01::","synset_id":"foundation_garment.n.01"},{"index":7,"sense_key":"foundation%1:04:00::","synset_id":"initiation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"foundation%1:14:00::","call_ids":["semeval2015.d002.s004.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"foundation%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":977,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002074,"input_uncached_usd":0.001954,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.513979500043206}
{"item_id":"semeval2015.d002.s004.t002","gold_sense_keys":["commission%1:14:00::","commission%1:14:01::"],"candidates":[{"index":1,"sense_key":"commission%1:14:00::","synset_id":"committee.n.01"},{"index":2,"sense_key":"commission%1:21:00::","synset_id":"commission.n.02"},{"index":3,"sense_key":"commission%1:04:02::","synset_id":"commission.n.03"},{"index":4,"sense_key":"commission%1:26:00::","synset_id":"commission.n.04"},{"index":5,"sense_key":"commission%1:14:01::","synset_id":"deputation.n.01"},{"index":6,"sense_key":"commission%1:10:01::","synset_id":"commission.n.06"},{"index":7,"sense_key":"commission%1:10:00::","synset_id":"commission.n.07"},{"index":8,"sense_key":"commission%1:04:01::","synset_id":"perpetration.n.01"},{"index":9,"sense_key":"commission%1:04:00::","synset_id":"mission.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"commission%1:14:00::","call_ids":["semeval2015.d002.s004.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"commission%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1020,"cached_input_tokens":0,"output_tokens":121,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00325,"input_uncached_usd":0.0020399999999999997,"input_cached_usd":0.0,"output_usd":0.0012100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8053739999886602}
{"item_id":"semeval2015.d002.s004.t004","gold_sense_keys":["ministry%1:14:00::"],"candidates":[{"index":1,"sense_key":"ministry%1:14:01::","synset_id":"ministry.n.01"},{"index":2,"sense_key":"ministry%1:06:00::","synset_id":"ministry.n.02"},{"index":3,"sense_key":"ministry%1:14:00::","synset_id":"ministry.n.03"},{"index":4,"sense_key":"ministry%1:04:00::","synset_id":"ministry.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"ministry%1:14:00::","call_ids":["semeval2015.d002.s004.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"ministry%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":642,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001404,"input_uncached_usd":0.001284,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":4.198793916963041}
{"item_id":"semeval2015.d002.s004.t005","gold_sense_keys":["social%3:01:00::"],"candidates":[{"index":1,"sense_key":"social%3:01:00::","synset_id":"social.a.01"},{"index":2,"sense_key":"social%3:00:00::","synset_id":"social.a.02"},{"index":3,"sense_key":"social%3:01:02::","synset_id":"social.a.03"},{"index":4,"sense_key":"social%5:00:00:sociable:00","synset_id":"social.s.01"},{"index":5,"sense_key":"social%5:00:00:gregarious:00","synset_id":"social.s.02"},{"index":6,"sense_key":"social%5:00:00:friendly:01","synset_id":"social.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"social%3:01:00::","call_ids":["semeval2015.d002.s004.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"social%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":820,"cached_input_tokens":0,"output_tokens":52,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00216,"input_uncached_usd":0.00164,"input_cached_usd":0.0,"output_usd":0.0005200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.266199708916247}
{"item_id":"semeval2015.d002.s004.t006","gold_sense_keys":["affairs%1:04:00::"],"candidates":[{"index":1,"sense_key":"affairs%1:09:00::","synset_id":"personal_business.n.01"},{"index":2,"sense_key":"affairs%1:04:00::","synset_id":"affairs.n.02"},{"index":3,"sense_key":"matter%1:09:01::","synset_id":"matter.n.01"},{"index":4,"sense_key":"affair%1:26:00::","synset_id":"affair.n.02"},{"index":5,"sense_key":"affair%1:11:00::","synset_id":"affair.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"affairs%1:04:00::","call_ids":["semeval2015.d002.s004.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"affairs%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":800,"cached_input_tokens":0,"output_tokens":56,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00216,"input_uncached_usd":0.0015999999999999999,"input_cached_usd":0.0,"output_usd":0.0005600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4165672499220818}
{"item_id":"semeval2015.d002.s004.t007","gold_sense_keys":["regional%5:00:00:territorial:00"],"candidates":[{"index":1,"sense_key":"regional%3:01:00::","synset_id":"regional.a.01"},{"index":2,"sense_key":"regional%5:00:00:territorial:00","synset_id":"regional.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"regional%5:00:00:territorial:00","call_ids":["semeval2015.d002.s004.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"regional%5:00:00:territorial:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":567,"cached_input_tokens":0,"output_tokens":102,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002154,"input_uncached_usd":0.001134,"input_cached_usd":0.0,"output_usd":0.00102,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.503350584069267}
{"item_id":"semeval2015.d002.s004.t008","gold_sense_keys":["government%1:14:00::"],"candidates":[{"index":1,"sense_key":"government%1:14:00::","synset_id":"government.n.01"},{"index":2,"sense_key":"government%1:04:00::","synset_id":"government.n.02"},{"index":3,"sense_key":"government%1:09:00::","synset_id":"government.n.03"},{"index":4,"sense_key":"government%1:09:01::","synset_id":"politics.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"government%1:14:00::","call_ids":["semeval2015.d002.s004.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"government%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":723,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0015660000000000001,"input_uncached_usd":0.001446,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.46894562500529}
{"item_id":"semeval2015.d002.s004.t010","gold_sense_keys":["conference%1:14:00::"],"candidates":[{"index":1,"sense_key":"conference%1:14:00::","synset_id":"conference.n.01"},{"index":2,"sense_key":"conference%1:14:01::","synset_id":"league.n.01"},{"index":3,"sense_key":"conference%1:10:00::","synset_id":"conference.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"conference%1:14:00::","call_ids":["semeval2015.d002.s004.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"conference%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":636,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001392,"input_uncached_usd":0.001272,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1351237919880077}
{"item_id":"semeval2015.d002.s004.t011","gold_sense_keys":["social%3:01:00::"],"candidates":[{"index":1,"sense_key":"social%3:01:00::","synset_id":"social.a.01"},{"index":2,"sense_key":"social%3:00:00::","synset_id":"social.a.02"},{"index":3,"sense_key":"social%3:01:02::","synset_id":"social.a.03"},{"index":4,"sense_key":"social%5:00:00:sociable:00","synset_id":"social.s.01"},{"index":5,"sense_key":"social%5:00:00:gregarious:00","synset_id":"social.s.02"},{"index":6,"sense_key":"social%5:00:00:friendly:01","synset_id":"social.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"social%3:01:00::","call_ids":["semeval2015.d002.s004.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"social%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":819,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001758,"input_uncached_usd":0.001638,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3811655840836465}
{"item_id":"semeval2015.d002.s004.t012","gold_sense_keys":["exclusion%1:26:00::"],"candidates":[{"index":1,"sense_key":"exclusion%1:26:00::","synset_id":"exclusion.n.01"},{"index":2,"sense_key":"exclusion%1:26:01::","synset_id":"excommunication.n.01"},{"index":3,"sense_key":"exclusion%1:09:00::","synset_id":"exception.n.01"},{"index":4,"sense_key":"exclusion%1:04:01::","synset_id":"ejection.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"exclusion%1:26:00::","call_ids":["semeval2015.d002.s004.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"exclusion%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":708,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001536,"input_uncached_usd":0.001416,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":9.500927916029468}
{"item_id":"semeval2015.d002.s004.t013","gold_sense_keys":["major%3:00:02::","major%3:00:06::"],"candidates":[{"index":1,"sense_key":"major%3:00:06::","synset_id":"major.a.01"},{"index":2,"sense_key":"major%3:00:02::","synset_id":"major.a.02"},{"index":3,"sense_key":"major%3:00:01::","synset_id":"major.a.03"},{"index":4,"sense_key":"major%3:00:05::","synset_id":"major.a.04"},{"index":5,"sense_key":"major%3:00:04::","synset_id":"major.a.05"},{"index":6,"sense_key":"major%3:00:07::","synset_id":"major.a.06"},{"index":7,"sense_key":"major%3:00:03::","synset_id":"major.a.07"},{"index":8,"sense_key":"major%5:00:00:senior:00","synset_id":"major.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"major%3:00:02::","call_ids":["semeval2015.d002.s004.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"major%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":835,"cached_input_tokens":0,"output_tokens":154,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00321,"input_uncached_usd":0.0016699999999999998,"input_cached_usd":0.0,"output_usd":0.0015400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.483786416007206}
{"item_id":"semeval2015.d002.s004.t014","gold_sense_keys":["challenge%1:26:00::"],"candidates":[{"index":1,"sense_key":"challenge%1:26:00::","synset_id":"challenge.n.01"},{"index":2,"sense_key":"challenge%1:10:00::","synset_id":"challenge.n.02"},{"index":3,"sense_key":"challenge%1:10:01::","synset_id":"challenge.n.03"},{"index":4,"sense_key":"challenge%1:10:02::","synset_id":"challenge.n.04"},{"index":5,"sense_key":"challenge%1:10:03::","synset_id":"challenge.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"challenge%1:26:00::","call_ids":["semeval2015.d002.s004.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"challenge%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":715,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00155,"input_uncached_usd":0.0014299999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0765693329740316}
{"item_id":"semeval2015.d002.s004.t015","gold_sense_keys":["public%5:00:00:common:02"],"candidates":[{"index":1,"sense_key":"public%3:00:00::","synset_id":"public.a.01"},{"index":2,"sense_key":"public%5:00:00:common:02","synset_id":"public.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"public%3:00:00::","call_ids":["semeval2015.d002.s004.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"public%3:00:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":577,"cached_input_tokens":0,"output_tokens":94,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002094,"input_uncached_usd":0.0011539999999999999,"input_cached_usd":0.0,"output_usd":0.0009400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.881747292005457}
{"item_id":"semeval2015.d002.s004.t016","gold_sense_keys":["welfare%1:04:00::"],"candidates":[{"index":1,"sense_key":"welfare%1:04:00::","synset_id":"social_welfare.n.01"},{"index":2,"sense_key":"welfare%1:07:00::","synset_id":"benefit.n.02"},{"index":3,"sense_key":"welfare%1:26:00::","synset_id":"wellbeing.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"welfare%1:04:00::","call_ids":["semeval2015.d002.s004.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"welfare%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":697,"cached_input_tokens":0,"output_tokens":45,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001844,"input_uncached_usd":0.0013939999999999998,"input_cached_usd":0.0,"output_usd":0.00045000000000000004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.205539415939711}
{"item_id":"semeval2015.d002.s005.t000","gold_sense_keys":["foundation%1:14:00::"],"candidates":[{"index":1,"sense_key":"foundation%1:24:00::","synset_id":"foundation.n.01"},{"index":2,"sense_key":"foundation%1:14:00::","synset_id":"foundation.n.02"},{"index":3,"sense_key":"foundation%1:06:00::","synset_id":"foundation.n.03"},{"index":4,"sense_key":"foundation%1:09:01::","synset_id":"foundation.n.04"},{"index":5,"sense_key":"foundation%1:09:00::","synset_id":"basis.n.02"},{"index":6,"sense_key":"foundation%1:06:01::","synset_id":"foundation_garment.n.01"},{"index":7,"sense_key":"foundation%1:04:00::","synset_id":"initiation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"foundation%1:14:00::","call_ids":["semeval2015.d002.s005.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"foundation%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1021,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002162,"input_uncached_usd":0.002042,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0896851249272004}
{"item_id":"semeval2015.d002.s005.t002","gold_sense_keys":["publish%2:32:00::","publish%2:36:01::"],"candidates":[{"index":1,"sense_key":"publish%2:36:01::","synset_id":"print.v.01"},{"index":2,"sense_key":"publish%2:32:00::","synset_id":"publish.v.02"},{"index":3,"sense_key":"publish%2:36:00::","synset_id":"publish.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"publish%2:32:00::","call_ids":["semeval2015.d002.s005.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"publish%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":726,"cached_input_tokens":0,"output_tokens":97,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002422,"input_uncached_usd":0.001452,"input_cached_usd":0.0,"output_usd":0.00097,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6922098329523578}
{"item_id":"semeval2015.d002.s005.t003","gold_sense_keys":["comparative%3:01:00::"],"candidates":[{"index":1,"sense_key":"comparative%3:01:00::","synset_id":"comparative.a.01"},{"index":2,"sense_key":"comparative%3:00:00::","synset_id":"relative.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"comparative%3:01:00::","call_ids":["semeval2015.d002.s005.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"comparative%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":632,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001384,"input_uncached_usd":0.001264,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.7875799169996753}
{"item_id":"semeval2015.d002.s005.t004","gold_sense_keys":["study%1:04:00::","study%1:10:00::"],"candidates":[{"index":1,"sense_key":"study%1:04:00::","synset_id":"survey.n.01"},{"index":2,"sense_key":"study%1:09:03::","synset_id":"study.n.02"},{"index":3,"sense_key":"study%1:10:00::","synset_id":"report.n.01"},{"index":4,"sense_key":"study%1:09:04::","synset_id":"study.n.04"},{"index":5,"sense_key":"study%1:06:00::","synset_id":"study.n.05"},{"index":6,"sense_key":"study%1:09:02::","synset_id":"discipline.n.01"},{"index":7,"sense_key":"study%1:06:01::","synset_id":"sketch.n.01"},{"index":8,"sense_key":"study%1:09:01::","synset_id":"cogitation.n.02"},{"index":9,"sense_key":"study%1:18:00::","synset_id":"study.n.09"},{"index":10,"sense_key":"study%1:10:01::","synset_id":"study.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"study%1:10:00::","call_ids":["semeval2015.d002.s005.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"study%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1114,"cached_input_tokens":0,"output_tokens":106,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003288,"input_uncached_usd":0.002228,"input_cached_usd":0.0,"output_usd":0.0010600000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8402199170086533}
{"item_id":"semeval2015.d002.s005.t005","gold_sense_keys":["part%1:09:00::","part%1:24:00::"],"candidates":[{"index":1,"sense_key":"part%1:24:00::","synset_id":"part.n.01"},{"index":2,"sense_key":"part%1:06:00::","synset_id":"part.n.02"},{"index":3,"sense_key":"part%1:17:00::","synset_id":"part.n.03"},{"index":4,"sense_key":"part%1:09:02::","synset_id":"part.n.04"},{"index":5,"sense_key":"part%1:15:00::","synset_id":"region.n.01"},{"index":6,"sense_key":"part%1:04:01::","synset_id":"function.n.03"},{"index":7,"sense_key":"part%1:09:01::","synset_id":"character.n.04"},{"index":8,"sense_key":"part%1:21:00::","synset_id":"share.n.01"},{"index":9,"sense_key":"part%1:09:00::","synset_id":"part.n.09"},{"index":10,"sense_key":"part%1:08:01::","synset_id":"part.n.10"},{"index":11,"sense_key":"part%1:10:00::","synset_id":"part.n.11"},{"index":12,"sense_key":"part%1:04:03::","synset_id":"contribution.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"part%1:24:00::","call_ids":["semeval2015.d002.s005.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"part%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1337,"cached_input_tokens":0,"output_tokens":126,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003934,"input_uncached_usd":0.0026739999999999997,"input_cached_usd":0.0,"output_usd":0.00126,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.575761542073451}
{"item_id":"semeval2015.d002.s005.t007","gold_sense_keys":["assess%2:31:00::"],"candidates":[{"index":1,"sense_key":"assess%2:31:00::","synset_id":"measure.v.04"},{"index":2,"sense_key":"assess%2:40:01::","synset_id":"assess.v.02"},{"index":3,"sense_key":"assess%2:40:00::","synset_id":"tax.v.02"},{"index":4,"sense_key":"assess%2:31:01::","synset_id":"assess.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"assess%2:31:00::","call_ids":["semeval2015.d002.s005.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"assess%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":775,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00167,"input_uncached_usd":0.00155,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9875794579274952}
{"item_id":"semeval2015.d002.s005.t008","gold_sense_keys":["extent%1:26:00::"],"candidates":[{"index":1,"sense_key":"extent%1:26:00::","synset_id":"extent.n.01"},{"index":2,"sense_key":"extent%1:07:00::","synset_id":"extent.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"extent%1:26:00::","call_ids":["semeval2015.d002.s005.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"extent%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":638,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001396,"input_uncached_usd":0.001276,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":14.706438333028927}
{"item_id":"semeval2015.d002.s005.t009","gold_sense_keys":["direct%5:00:00:immediate:00"],"candidates":[{"index":1,"sense_key":"direct%3:00:00::","synset_id":"direct.a.01"},{"index":2,"sense_key":"direct%5:00:00:immediate:00","synset_id":"direct.s.01"},{"index":3,"sense_key":"direct%3:00:02::","synset_id":"direct.a.03"},{"index":4,"sense_key":"direct%3:00:04::","synset_id":"lineal.a.01"},{"index":5,"sense_key":"direct%3:00:05::","synset_id":"direct.a.05"},{"index":6,"sense_key":"direct%3:00:03::","synset_id":"direct.a.06"},{"index":7,"sense_key":"direct%3:00:01::","synset_id":"direct.a.07"},{"index":8,"sense_key":"direct%5:00:00:primary:00","synset_id":"direct.s.02"},{"index":9,"sense_key":"direct%5:00:00:exact:00","synset_id":"direct.s.03"},{"index":10,"sense_key":"direct%5:00:00:absolute:00","synset_id":"direct.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"direct%5:00:00:immediate:00","call_ids":["semeval2015.d002.s005.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"direct%5:00:00:immediate:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1131,"cached_input_tokens":0,"output_tokens":67,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0029319999999999997,"input_uncached_usd":0.0022619999999999997,"input_cached_usd":0.0,"output_usd":0.00067,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8960723340278491}
{"item_id":"semeval2015.d002.s005.t010","gold_sense_keys":["participation%1:04:00::"],"candidates":[{"index":1,"sense_key":"participation%1:04:00::","synset_id":"engagement.n.07"},{"index":2,"sense_key":"participation%1:26:00::","synset_id":"participation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"participation%1:04:00::","call_ids":["semeval2015.d002.s005.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"participation%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":656,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001962,"input_uncached_usd":0.001312,"input_cached_usd":0.0,"output_usd":0.0006500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8149035830283538}
{"item_id":"semeval2015.d002.s005.t012","gold_sense_keys":["work%1:04:01::"],"candidates":[{"index":1,"sense_key":"work%1:04:00::","synset_id":"work.n.01"},{"index":2,"sense_key":"work%1:06:00::","synset_id":"work.n.02"},{"index":3,"sense_key":"work%1:04:01::","synset_id":"employment.n.02"},{"index":4,"sense_key":"work%1:09:00::","synset_id":"study.n.02"},{"index":5,"sense_key":"work%1:19:00::","synset_id":"work.n.05"},{"index":6,"sense_key":"work%1:06:01::","synset_id":"workplace.n.01"},{"index":7,"sense_key":"work%1:06:02::","synset_id":"oeuvre.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"work%1:04:00::","call_ids":["semeval2015.d002.s005.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"work%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1028,"cached_input_tokens":0,"output_tokens":136,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003416,"input_uncached_usd":0.002056,"input_cached_usd":0.0,"output_usd":0.00136,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.99663733295165}
{"item_id":"semeval2015.d002.s005.t013","gold_sense_keys":["increase%2:30:02::"],"candidates":[{"index":1,"sense_key":"increase%2:30:00::","synset_id":"increase.v.01"},{"index":2,"sense_key":"increase%2:30:02::","synset_id":"increase.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"increase%2:30:02::","call_ids":["semeval2015.d002.s005.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"increase%2:30:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":627,"cached_input_tokens":0,"output_tokens":74,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001994,"input_uncached_usd":0.001254,"input_cached_usd":0.0,"output_usd":0.0007400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5829182910965756}
{"item_id":"semeval2015.d002.s006.t000","gold_sense_keys":["study%1:10:00::"],"candidates":[{"index":1,"sense_key":"study%1:04:00::","synset_id":"survey.n.01"},{"index":2,"sense_key":"study%1:09:03::","synset_id":"study.n.02"},{"index":3,"sense_key":"study%1:10:00::","synset_id":"report.n.01"},{"index":4,"sense_key":"study%1:09:04::","synset_id":"study.n.04"},{"index":5,"sense_key":"study%1:06:00::","synset_id":"study.n.05"},{"index":6,"sense_key":"study%1:09:02::","synset_id":"discipline.n.01"},{"index":7,"sense_key":"study%1:06:01::","synset_id":"sketch.n.01"},{"index":8,"sense_key":"study%1:09:01::","synset_id":"cogitation.n.02"},{"index":9,"sense_key":"study%1:18:00::","synset_id":"study.n.09"},{"index":10,"sense_key":"study%1:10:01::","synset_id":"study.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"study%1:10:00::","call_ids":["semeval2015.d002.s006.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"study%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1109,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002338,"input_uncached_usd":0.002218,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":13.732370291952975}
{"item_id":"semeval2015.d002.s006.t001","gold_sense_keys":["base%2:31:00::"],"candidates":[{"index":1,"sense_key":"base%2:31:00::","synset_id":"establish.v.08"},{"index":2,"sense_key":"base%2:42:04::","synset_id":"base.v.02"},{"index":3,"sense_key":"base%2:34:00::","synset_id":"free-base.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"base%2:31:00::","call_ids":["semeval2015.d002.s006.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"base%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":671,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001462,"input_uncached_usd":0.0013419999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1454597919946536}
{"item_id":"semeval2015.d002.s006.t002","gold_sense_keys":["personal%3:00:00::","personal%5:00:00:subjective:00"],"candidates":[{"index":1,"sense_key":"personal%3:00:00::","synset_id":"personal.a.01"},{"index":2,"sense_key":"personal%5:00:00:subjective:00","synset_id":"personal.s.01"},{"index":3,"sense_key":"personal%3:01:01::","synset_id":"personal.a.03"},{"index":4,"sense_key":"personal%5:00:00:physical:00","synset_id":"personal.s.02"},{"index":5,"sense_key":"personal%3:01:00::","synset_id":"personal.a.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"personal%5:00:00:subjective:00","call_ids":["semeval2015.d002.s006.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"personal%5:00:00:subjective:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":743,"cached_input_tokens":0,"output_tokens":163,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003116,"input_uncached_usd":0.001486,"input_cached_usd":0.0,"output_usd":0.0016300000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.5352291249437258}
{"item_id":"semeval2015.d002.s006.t003","gold_sense_keys":["interview%1:10:01::"],"candidates":[{"index":1,"sense_key":"interview%1:10:01::","synset_id":"interview.n.01"},{"index":2,"sense_key":"interview%1:10:00::","synset_id":"consultation.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"interview%1:10:01::","call_ids":["semeval2015.d002.s006.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"interview%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":670,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0014600000000000001,"input_uncached_usd":0.00134,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1047475000377744}
{"item_id":"semeval2015.d002.s006.t004","gold_sense_keys":["senior%3:00:00::"],"candidates":[{"index":1,"sense_key":"senior%3:00:00::","synset_id":"senior.a.01"},{"index":2,"sense_key":"senior%5:00:00:last:00","synset_id":"senior.s.01"},{"index":3,"sense_key":"senior%5:00:00:old:02","synset_id":"aged.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"senior%3:00:00::","call_ids":["semeval2015.d002.s006.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"senior%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":723,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0015660000000000001,"input_uncached_usd":0.001446,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1377127090236172}
{"item_id":"semeval2015.d002.s006.t005","gold_sense_keys":["official%1:18:01::"],"candidates":[{"index":1,"sense_key":"official%1:18:01::","synset_id":"official.n.01"},{"index":2,"sense_key":"official%1:18:00::","synset_id":"official.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"official%1:18:01::","call_ids":["semeval2015.d002.s006.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"official%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":621,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0013620000000000001,"input_uncached_usd":0.001242,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0735246250405908}
{"item_id":"semeval2015.d002.s006.t006","gold_sense_keys":["social%3:01:00::"],"candidates":[{"index":1,"sense_key":"social%3:01:00::","synset_id":"social.a.01"},{"index":2,"sense_key":"social%3:00:00::","synset_id":"social.a.02"},{"index":3,"sense_key":"social%3:01:02::","synset_id":"social.a.03"},{"index":4,"sense_key":"social%5:00:00:sociable:00","synset_id":"social.s.01"},{"index":5,"sense_key":"social%5:00:00:gregarious:00","synset_id":"social.s.02"},{"index":6,"sense_key":"social%5:00:00:friendly:01","synset_id":"social.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"social%3:01:00::","call_ids":["semeval2015.d002.s006.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"social%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":858,"cached_input_tokens":0,"output_tokens":85,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0025659999999999997,"input_uncached_usd":0.0017159999999999999,"input_cached_usd":0.0,"output_usd":0.0008500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1489915419369936}
{"item_id":"semeval2015.d002.s006.t007","gold_sense_keys":["representative%1:18:00::","representative%1:18:02::"],"candidates":[{"index":1,"sense_key":"representative%1:18:00::","synset_id":"representative.n.01"},{"index":2,"sense_key":"representative%1:18:02::","synset_id":"spokesperson.n.01"},{"index":3,"sense_key":"representative%1:18:01::","synset_id":"congressman.n.01"},{"index":4,"sense_key":"representative%1:09:00::","synset_id":"example.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"representative%1:18:00::","call_ids":["semeval2015.d002.s006.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"representative%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":761,"cached_input_tokens":0,"output_tokens":120,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002722,"input_uncached_usd":0.001522,"input_cached_usd":0.0,"output_usd":0.0012000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.7556098750792444}
{"item_id":"semeval2015.d002.s006.t008","gold_sense_keys":["national%3:00:01::","national%3:01:00::"],"candidates":[{"index":1,"sense_key":"national%3:01:00::","synset_id":"national.a.01"},{"index":2,"sense_key":"national%3:00:00::","synset_id":"national.a.02"},{"index":3,"sense_key":"national%3:00:01::","synset_id":"national.a.03"},{"index":4,"sense_key":"national%5:00:00:public:00","synset_id":"national.s.01"},{"index":5,"sense_key":"national%5:00:00:domestic:00","synset_id":"home.s.01"},{"index":6,"sense_key":"national%3:01:02::","synset_id":"national.a.06"},{"index":7,"sense_key":"national%3:01:01::","synset_id":"national.a.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"national%3:00:01::","call_ids":["semeval2015.d002.s006.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"national%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":884,"cached_input_tokens":0,"output_tokens":96,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002728,"input_uncached_usd":0.0017679999999999998,"input_cached_usd":0.0,"output_usd":0.0009600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8031515830662102}
{"item_id":"semeval2015.d002.s006.t009","gold_sense_keys":["level%1:26:01::"],"candidates":[{"index":1,"sense_key":"level%1:07:00::","synset_id":"degree.n.01"},{"index":2,"sense_key":"level%1:26:01::","synset_id":"grade.n.02"},{"index":3,"sense_key":"level%1:26:00::","synset_id":"degree.n.02"},{"index":4,"sense_key":"level%1:07:01::","synset_id":"level.n.04"},{"index":5,"sense_key":"level%1:06:00::","synset_id":"level.n.05"},{"index":6,"sense_key":"level%1:06:03::","synset_id":"horizontal_surface.n.01"},{"index":7,"sense_key":"level%1:09:00::","synset_id":"level.n.07"},{"index":8,"sense_key":"level%1:06:01::","synset_id":"floor.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"level%1:26:01::","call_ids":["semeval2015.d002.s006.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"level%1:26:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1044,"cached_input_tokens":0,"output_tokens":169,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003778,"input_uncached_usd":0.002088,"input_cached_usd":0.0,"output_usd":0.00169,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":15.275609542033635}
{"item_id":"semeval2015.d002.s006.t010","gold_sense_keys":["organisation%1:14:00::","organisation%1:14:01::"],"candidates":[{"index":1,"sense_key":"organisation%1:14:01::","synset_id":"administration.n.02"},{"index":2,"sense_key":"organisation%1:14:00::","synset_id":"organization.n.01"},{"index":3,"sense_key":"organisation%1:09:00::","synset_id":"arrangement.n.03"},{"index":4,"sense_key":"organisation%1:07:00::","synset_id":"organization.n.05"},{"index":5,"sense_key":"organisation%1:04:00::","synset_id":"organization.n.04"},{"index":6,"sense_key":"organisation%1:04:01::","synset_id":"organization.n.06"},{"index":7,"sense_key":"organisation%1:04:02::","synset_id":"constitution.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"organisation%1:14:00::","call_ids":["semeval2015.d002.s006.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"organisation%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1066,"cached_input_tokens":0,"output_tokens":94,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003072,"input_uncached_usd":0.0021319999999999998,"input_cached_usd":0.0,"output_usd":0.0009400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.706521457992494}
{"item_id":"semeval2015.d002.s006.t013","gold_sense_keys":["country%1:15:00::"],"candidates":[{"index":1,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":2,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":3,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":4,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":5,"sense_key":"country%1:15:02::","synset_id":"area.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"country%1:15:00::","call_ids":["semeval2015.d002.s006.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"country%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":874,"cached_input_tokens":0,"output_tokens":117,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002918,"input_uncached_usd":0.001748,"input_cached_usd":0.0,"output_usd":0.00117,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7660487910034135}
{"item_id":"semeval2015.d002.s007.t000","gold_sense_keys":["important%3:00:00::","important%3:00:02::","important%3:00:04::"],"candidates":[{"index":1,"sense_key":"important%3:00:00::","synset_id":"important.a.01"},{"index":2,"sense_key":"important%3:00:04::","synset_id":"significant.a.01"},{"index":3,"sense_key":"important%3:00:02::","synset_id":"crucial.a.01"},{"index":4,"sense_key":"important%5:00:00:influential:00","synset_id":"authoritative.s.01"},{"index":5,"sense_key":"important%5:00:00:immodest:02","synset_id":"important.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"important%3:00:00::","call_ids":["semeval2015.d002.s007.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"important%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":911,"cached_input_tokens":0,"output_tokens":114,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0029620000000000002,"input_uncached_usd":0.0018219999999999998,"input_cached_usd":0.0,"output_usd":0.0011400000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":15.033341958071105}
{"item_id":"semeval2015.d002.s007.t001","gold_sense_keys":["industrial%3:01:00::"],"candidates":[{"index":1,"sense_key":"industrial%3:01:00::","synset_id":"industrial.a.01"},{"index":2,"sense_key":"industrial%3:00:00::","synset_id":"industrial.a.02"},{"index":3,"sense_key":"industrial%5:00:00:blue-collar:00","synset_id":"industrial.s.01"},{"index":4,"sense_key":"industrial%5:00:00:heavy-duty:00","synset_id":"industrial.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"industrial%3:01:00::","call_ids":["semeval2015.d002.s007.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"industrial%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":801,"cached_input_tokens":0,"output_tokens":110,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002702,"input_uncached_usd":0.001602,"input_cached_usd":0.0,"output_usd":0.0011,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5407246249960735}
{"item_id":"semeval2015.d002.s007.t002","gold_sense_keys":["sector%1:14:00::"],"candidates":[{"index":1,"sense_key":"sector%1:25:00::","synset_id":"sector.n.01"},{"index":2,"sense_key":"sector%1:14:00::","synset_id":"sector.n.02"},{"index":3,"sense_key":"sector%1:09:00::","synset_id":"sector.n.03"},{"index":4,"sense_key":"sector%1:23:00::","synset_id":"sector.n.04"},{"index":5,"sense_key":"sector%1:15:00::","synset_id":"sector.n.05"},{"index":6,"sense_key":"sector%1:06:00::","synset_id":"sector.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"sector%1:14:00::","call_ids":["semeval2015.d002.s007.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"sector%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":884,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001888,"input_uncached_usd":0.0017679999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0340278749354184}
{"item_id":"semeval2015.d002.s007.t003","gold_sense_keys":["metal%1:27:00::"],"candidates":[{"index":1,"sense_key":"metal%1:27:00::","synset_id":"metallic_element.n.01"},{"index":2,"sense_key":"metal%1:27:01::","synset_id":"alloy.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"metal%1:27:00::","call_ids":["semeval2015.d002.s007.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"metal%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":767,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002124,"input_uncached_usd":0.001534,"input_cached_usd":0.0,"output_usd":0.00059,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6945997499860823}
{"item_id":"semeval2015.d002.s007.t004","gold_sense_keys":["industry%1:14:00::"],"candidates":[{"index":1,"sense_key":"industry%1:14:00::","synset_id":"industry.n.01"},{"index":2,"sense_key":"industry%1:04:00::","synset_id":"industry.n.02"},{"index":3,"sense_key":"industry%1:07:00::","synset_id":"diligence.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"industry%1:14:00::","call_ids":["semeval2015.d002.s007.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"industry%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":803,"cached_input_tokens":0,"output_tokens":89,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002496,"input_uncached_usd":0.001606,"input_cached_usd":0.0,"output_usd":0.0008900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2447147079510614}
{"item_id":"semeval2015.d002.s007.t005","gold_sense_keys":["banking%1:04:01::"],"candidates":[{"index":1,"sense_key":"banking%1:04:01::","synset_id":"banking.n.01"},{"index":2,"sense_key":"banking%1:04:00::","synset_id":"banking.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"banking%1:04:01::","call_ids":["semeval2015.d002.s007.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"banking%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":716,"cached_input_tokens":0,"output_tokens":82,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002252,"input_uncached_usd":0.001432,"input_cached_usd":0.0,"output_usd":0.0008200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2153802090324461}
{"item_id":"semeval2015.d002.s007.t006","gold_sense_keys":["sector%1:14:00::"],"candidates":[{"index":1,"sense_key":"sector%1:25:00::","synset_id":"sector.n.01"},{"index":2,"sense_key":"sector%1:14:00::","synset_id":"sector.n.02"},{"index":3,"sense_key":"sector%1:09:00::","synset_id":"sector.n.03"},{"index":4,"sense_key":"sector%1:23:00::","synset_id":"sector.n.04"},{"index":5,"sense_key":"sector%1:15:00::","synset_id":"sector.n.05"},{"index":6,"sense_key":"sector%1:06:00::","synset_id":"sector.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"sector%1:14:00::","call_ids":["semeval2015.d002.s007.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"sector%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":883,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001886,"input_uncached_usd":0.001766,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8062982090050355}
{"item_id":"semeval2015.d002.s007.t008","gold_sense_keys":["include%2:30:00::","include%2:31:00::"],"candidates":[{"index":1,"sense_key":"include%2:42:00::","synset_id":"include.v.01"},{"index":2,"sense_key":"include%2:31:00::","synset_id":"include.v.02"},{"index":3,"sense_key":"include%2:30:00::","synset_id":"include.v.03"},{"index":4,"sense_key":"include%2:41:03::","synset_id":"admit.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"include%2:30:00::","call_ids":["semeval2015.d002.s007.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"include%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":838,"cached_input_tokens":0,"output_tokens":147,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0031460000000000004,"input_uncached_usd":0.001676,"input_cached_usd":0.0,"output_usd":0.0014700000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.140933834016323}
{"item_id":"semeval2015.d002.s008.t000","gold_sense_keys":["speaker%1:18:00::"],"candidates":[{"index":1,"sense_key":"speaker%1:18:00::","synset_id":"speaker.n.01"},{"index":2,"sense_key":"speaker%1:06:00::","synset_id":"loudspeaker.n.01"},{"index":3,"sense_key":"speaker%1:18:01::","synset_id":"speaker.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"speaker%1:18:00::","call_ids":["semeval2015.d002.s008.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"speaker%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":891,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001902,"input_uncached_usd":0.001782,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2310712080216035}
{"item_id":"semeval2015.d002.s008.t001","gold_sense_keys":["social%3:01:00::"],"candidates":[{"index":1,"sense_key":"social%3:01:00::","synset_id":"social.a.01"},{"index":2,"sense_key":"social%3:00:00::","synset_id":"social.a.02"},{"index":3,"sense_key":"social%3:01:02::","synset_id":"social.a.03"},{"index":4,"sense_key":"social%5:00:00:sociable:00","synset_id":"social.s.01"},{"index":5,"sense_key":"social%5:00:00:gregarious:00","synset_id":"social.s.02"},{"index":6,"sense_key":"social%5:00:00:friendly:01","synset_id":"social.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"social%3:01:00::","call_ids":["semeval2015.d002.s008.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"social%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":965,"cached_input_tokens":0,"output_tokens":52,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00245,"input_uncached_usd":0.0019299999999999999,"input_cached_usd":0.0,"output_usd":0.0005200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3363973329542205}
{"item_id":"semeval2015.d002.s008.t002","gold_sense_keys":["exclusion%1:26:00::"],"candidates":[{"index":1,"sense_key":"exclusion%1:26:00::","synset_id":"exclusion.n.01"},{"index":2,"sense_key":"exclusion%1:26:01::","synset_id":"excommunication.n.01"},{"index":3,"sense_key":"exclusion%1:09:00::","synset_id":"exception.n.01"},{"index":4,"sense_key":"exclusion%1:04:01::","synset_id":"ejection.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"exclusion%1:26:00::","call_ids":["semeval2015.d002.s008.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"exclusion%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":854,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001828,"input_uncached_usd":0.0017079999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1732184580760077}
{"item_id":"semeval2015.d002.s008.t003","gold_sense_keys":["conference%1:10:00::","conference%1:14:00::"],"candidates":[{"index":1,"sense_key":"conference%1:14:00::","synset_id":"conference.n.01"},{"index":2,"sense_key":"conference%1:14:01::","synset_id":"league.n.01"},{"index":3,"sense_key":"conference%1:10:00::","synset_id":"conference.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"conference%1:14:00::","call_ids":["semeval2015.d002.s008.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"conference%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":782,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001684,"input_uncached_usd":0.0015639999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8131634580204263}
{"item_id":"semeval2015.d002.s008.t004","gold_sense_keys":["left%1:15:00::"],"candidates":[{"index":1,"sense_key":"left%1:15:00::","synset_id":"left.n.01"},{"index":2,"sense_key":"left%1:14:00::","synset_id":"left.n.02"},{"index":3,"sense_key":"left%1:08:00::","synset_id":"left.n.03"},{"index":4,"sense_key":"left%1:06:00::","synset_id":"left_field.n.01"},{"index":5,"sense_key":"left%1:04:00::","synset_id":"left.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"left%1:15:00::","call_ids":["semeval2015.d002.s008.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"left%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":907,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001934,"input_uncached_usd":0.0018139999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0233338330872357}
{"item_id":"semeval2015.d002.s008.t005","gold_sense_keys":["right%1:15:00::"],"candidates":[{"index":1,"sense_key":"right%1:07:00::","synset_id":"right.n.01"},{"index":2,"sense_key":"right%1:15:00::","synset_id":"right.n.02"},{"index":3,"sense_key":"right%1:06:00::","synset_id":"right_field.n.01"},{"index":4,"sense_key":"right%1:14:00::","synset_id":"right.n.04"},{"index":5,"sense_key":"right%1:08:00::","synset_id":"right.n.05"},{"index":6,"sense_key":"right%1:04:00::","synset_id":"right.n.06"},{"index":7,"sense_key":"right%1:07:01::","synset_id":"right.n.07"},{"index":8,"sense_key":"right%1:21:00::","synset_id":"right.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"right%1:15:00::","call_ids":["semeval2015.d002.s008.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"right%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1157,"cached_input_tokens":0,"output_tokens":52,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002834,"input_uncached_usd":0.002314,"input_cached_usd":0.0,"output_usd":0.0005200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.262930916971527}
{"item_id":"semeval2015.d002.s008.t006","gold_sense_keys":["employment%1:04:00::","employment%1:04:02::","employment%1:26:00::"],"candidates":[{"index":1,"sense_key":"employment%1:26:00::","synset_id":"employment.n.01"},{"index":2,"sense_key":"employment%1:04:00::","synset_id":"employment.n.02"},{"index":3,"sense_key":"employment%1:04:02::","synset_id":"employment.n.03"},{"index":4,"sense_key":"employment%1:04:01::","synset_id":"use.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"employment%1:26:00::","call_ids":["semeval2015.d002.s008.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"employment%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":871,"cached_input_tokens":0,"output_tokens":111,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002852,"input_uncached_usd":0.0017419999999999998,"input_cached_usd":0.0,"output_usd":0.00111,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":5.259889874956571}
{"item_id":"semeval2015.d002.s008.t007","gold_sense_keys":["industrial%3:01:00::"],"candidates":[{"index":1,"sense_key":"industrial%3:01:00::","synset_id":"industrial.a.01"},{"index":2,"sense_key":"industrial%3:00:00::","synset_id":"industrial.a.02"},{"index":3,"sense_key":"industrial%5:00:00:blue-collar:00","synset_id":"industrial.s.01"},{"index":4,"sense_key":"industrial%5:00:00:heavy-duty:00","synset_id":"industrial.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"industrial%3:01:00::","call_ids":["semeval2015.d002.s008.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"industrial%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":828,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002316,"input_uncached_usd":0.001656,"input_cached_usd":0.0,"output_usd":0.0006600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.797267624991946}
{"item_id":"semeval2015.d002.s008.t012","gold_sense_keys":["commission%1:14:00::","commission%1:14:01::"],"candidates":[{"index":1,"sense_key":"commission%1:14:00::","synset_id":"committee.n.01"},{"index":2,"sense_key":"commission%1:21:00::","synset_id":"commission.n.02"},{"index":3,"sense_key":"commission%1:04:02::","synset_id":"commission.n.03"},{"index":4,"sense_key":"commission%1:26:00::","synset_id":"commission.n.04"},{"index":5,"sense_key":"commission%1:14:01::","synset_id":"deputation.n.01"},{"index":6,"sense_key":"commission%1:10:01::","synset_id":"commission.n.06"},{"index":7,"sense_key":"commission%1:10:00::","synset_id":"commission.n.07"},{"index":8,"sense_key":"commission%1:04:01::","synset_id":"perpetration.n.01"},{"index":9,"sense_key":"commission%1:04:00::","synset_id":"mission.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"commission%1:14:00::","call_ids":["semeval2015.d002.s008.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"commission%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1166,"cached_input_tokens":0,"output_tokens":123,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003562,"input_uncached_usd":0.002332,"input_cached_usd":0.0,"output_usd":0.0012300000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":5.8312982090283185}
{"item_id":"semeval2015.d002.s008.t013","gold_sense_keys":["high%3:00:02::"],"candidates":[{"index":1,"sense_key":"high%3:00:02::","synset_id":"high.a.01"},{"index":2,"sense_key":"high%3:00:01::","synset_id":"high.a.02"},{"index":3,"sense_key":"high%5:00:00:superior:01","synset_id":"eminent.s.01"},{"index":4,"sense_key":"high%3:00:03::","synset_id":"high.a.04"},{"index":5,"sense_key":"high%5:00:00:elated:00","synset_id":"high.s.02"},{"index":6,"sense_key":"high%5:00:00:malodorous:00","synset_id":"gamey.s.02"},{"index":7,"sense_key":"high%5:00:00:intoxicated:00","synset_id":"high.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"high%3:00:02::","call_ids":["semeval2015.d002.s008.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"high%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1037,"cached_input_tokens":0,"output_tokens":48,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002554,"input_uncached_usd":0.002074,"input_cached_usd":0.0,"output_usd":0.00048000000000000007,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3621340000536293}
{"item_id":"semeval2015.d002.s008.t014","gold_sense_keys":["committee%1:14:00::"],"candidates":[{"index":1,"sense_key":"committee%1:14:00::","synset_id":"committee.n.01"},{"index":2,"sense_key":"committee%1:14:01::","synset_id":"committee.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"committee%1:14:00::","call_ids":["semeval2015.d002.s008.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"committee%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":749,"cached_input_tokens":0,"output_tokens":84,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002338,"input_uncached_usd":0.001498,"input_cached_usd":0.0,"output_usd":0.00084,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3678546249866486}
{"item_id":"semeval2015.d002.s008.t015","gold_sense_keys":["member%1:18:00::"],"candidates":[{"index":1,"sense_key":"member%1:18:00::","synset_id":"member.n.01"},{"index":2,"sense_key":"member%1:24:00::","synset_id":"member.n.02"},{"index":3,"sense_key":"member%1:08:00::","synset_id":"extremity.n.01"},{"index":4,"sense_key":"member%1:14:00::","synset_id":"member.n.04"},{"index":5,"sense_key":"member%1:08:01::","synset_id":"penis.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"member%1:18:00::","call_ids":["semeval2015.d002.s008.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"member%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":959,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002038,"input_uncached_usd":0.001918,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.185812707990408}
{"item_id":"semeval2015.d002.s008.t017","gold_sense_keys":["parliament%1:14:00::"],"candidates":[{"index":1,"sense_key":"parliament%1:14:00::","synset_id":"parliament.n.01"},{"index":2,"sense_key":"parliament%1:04:00::","synset_id":"fantan.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"parliament%1:14:00::","call_ids":["semeval2015.d002.s008.t017__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"parliament%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":747,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001614,"input_uncached_usd":0.0014939999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0935149999568239}
{"item_id":"semeval2015.d002.s008.t020","gold_sense_keys":["parliament%1:14:00::"],"candidates":[{"index":1,"sense_key":"parliament%1:14:00::","synset_id":"parliament.n.01"},{"index":2,"sense_key":"parliament%1:04:00::","synset_id":"fantan.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"parliament%1:14:00::","call_ids":["semeval2015.d002.s008.t020__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"parliament%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":747,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001614,"input_uncached_usd":0.0014939999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8130842500831932}
{"item_id":"semeval2015.d002.s008.t021","gold_sense_keys":["economic%3:01:01::"],"candidates":[{"index":1,"sense_key":"economic%3:01:01::","synset_id":"economic.a.01"},{"index":2,"sense_key":"economic%3:01:00::","synset_id":"economic.a.02"},{"index":3,"sense_key":"economic%5:00:00:efficient:00","synset_id":"economic.s.01"},{"index":4,"sense_key":"economic%5:00:00:worldly:00","synset_id":"economic.s.02"},{"index":5,"sense_key":"economic%5:00:00:profitable:00","synset_id":"economic.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"economic%3:01:01::","call_ids":["semeval2015.d002.s008.t021__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"economic%3:01:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":966,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002522,"input_uncached_usd":0.001932,"input_cached_usd":0.0,"output_usd":0.00059,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8357328749261796}
{"item_id":"semeval2015.d002.s008.t022","gold_sense_keys":["social%3:01:00::"],"candidates":[{"index":1,"sense_key":"social%3:01:00::","synset_id":"social.a.01"},{"index":2,"sense_key":"social%3:00:00::","synset_id":"social.a.02"},{"index":3,"sense_key":"social%3:01:02::","synset_id":"social.a.03"},{"index":4,"sense_key":"social%5:00:00:sociable:00","synset_id":"social.s.01"},{"index":5,"sense_key":"social%5:00:00:gregarious:00","synset_id":"social.s.02"},{"index":6,"sense_key":"social%5:00:00:friendly:01","synset_id":"social.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"social%3:01:00::","call_ids":["semeval2015.d002.s008.t022__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"social%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":966,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002472,"input_uncached_usd":0.001932,"input_cached_usd":0.0,"output_usd":0.00054,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1038085840409622}
{"item_id":"semeval2015.d002.s008.t023","gold_sense_keys":["committee%1:14:00::"],"candidates":[{"index":1,"sense_key":"committee%1:14:00::","synset_id":"committee.n.01"},{"index":2,"sense_key":"committee%1:14:01::","synset_id":"committee.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"committee%1:14:00::","call_ids":["semeval2015.d002.s008.t023__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"committee%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":749,"cached_input_tokens":0,"output_tokens":76,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002258,"input_uncached_usd":0.001498,"input_cached_usd":0.0,"output_usd":0.00076,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3985180830350146}
{"item_id":"semeval2015.d002.s009.t000","gold_sense_keys":["case_study%1:10:00::"],"candidates":[{"index":1,"sense_key":"case_study%1:10:00::","synset_id":"case_study.n.01"},{"index":2,"sense_key":"case_study%1:04:00::","synset_id":"case_study.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"case_study%1:10:00::","call_ids":["semeval2015.d002.s009.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"case_study%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":748,"cached_input_tokens":0,"output_tokens":156,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003056,"input_uncached_usd":0.001496,"input_cached_usd":0.0,"output_usd":0.0015600000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9101960830157623}
{"item_id":"semeval2015.d002.s009.t001","gold_sense_keys":["national%3:00:00::","national%3:00:01::","national%3:01:00::","national%5:00:00:domestic:00"],"candidates":[{"index":1,"sense_key":"national%3:01:00::","synset_id":"national.a.01"},{"index":2,"sense_key":"national%3:00:00::","synset_id":"national.a.02"},{"index":3,"sense_key":"national%3:00:01::","synset_id":"national.a.03"},{"index":4,"sense_key":"national%5:00:00:public:00","synset_id":"national.s.01"},{"index":5,"sense_key":"national%5:00:00:domestic:00","synset_id":"home.s.01"},{"index":6,"sense_key":"national%3:01:02::","synset_id":"national.a.06"},{"index":7,"sense_key":"national%3:01:01::","synset_id":"national.a.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"national%3:01:00::","call_ids":["semeval2015.d002.s009.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"national%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":994,"cached_input_tokens":0,"output_tokens":131,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0032979999999999997,"input_uncached_usd":0.0019879999999999997,"input_cached_usd":0.0,"output_usd":0.0013100000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2095112079987302}
{"item_id":"semeval2015.d002.s009.t002","gold_sense_keys":["report%1:10:03::"],"candidates":[{"index":1,"sense_key":"report%1:10:03::","synset_id":"report.n.01"},{"index":2,"sense_key":"report%1:10:01::","synset_id":"report.n.02"},{"index":3,"sense_key":"report%1:10:00::","synset_id":"report.n.03"},{"index":4,"sense_key":"report%1:11:00::","synset_id":"report.n.04"},{"index":5,"sense_key":"report%1:10:04::","synset_id":"report_card.n.01"},{"index":6,"sense_key":"report%1:10:02::","synset_id":"composition.n.08"},{"index":7,"sense_key":"report%1:09:00::","synset_id":"reputation.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"report%1:10:03::","call_ids":["semeval2015.d002.s009.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"report%1:10:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1080,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00228,"input_uncached_usd":0.00216,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0411089999834076}
{"item_id":"semeval2015.d002.s009.t003","gold_sense_keys":["contain%2:42:00::"],"candidates":[{"index":1,"sense_key":"contain%2:42:00::","synset_id":"incorporate.v.02"},{"index":2,"sense_key":"contain%2:42:13::","synset_id":"hold.v.11"},{"index":3,"sense_key":"contain%2:41:00::","synset_id":"control.v.02"},{"index":4,"sense_key":"contain%2:42:06::","synset_id":"contain.v.04"},{"index":5,"sense_key":"contain%2:42:14::","synset_id":"contain.v.05"},{"index":6,"sense_key":"contain%2:33:00::","synset_id":"check.v.18"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"contain%2:42:00::","call_ids":["semeval2015.d002.s009.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"contain%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1066,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0022519999999999997,"input_uncached_usd":0.0021319999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0397651250241324}
{"item_id":"semeval2015.d002.s009.t006","gold_sense_keys":["present%2:32:00::"],"candidates":[{"index":1,"sense_key":"present%2:39:00::","synset_id":"show.v.01"},{"index":2,"sense_key":"present%2:32:00::","synset_id":"present.v.02"},{"index":3,"sense_key":"present%2:36:00::","synset_id":"stage.v.01"},{"index":4,"sense_key":"present%2:40:01::","synset_id":"present.v.04"},{"index":5,"sense_key":"present%2:42:00::","synset_id":"present.v.05"},{"index":6,"sense_key":"present%2:40:02::","synset_id":"award.v.01"},{"index":7,"sense_key":"present%2:40:00::","synset_id":"give.v.08"},{"index":8,"sense_key":"present%2:32:02::","synset_id":"deliver.v.01"},{"index":9,"sense_key":"present%2:32:01::","synset_id":"introduce.v.01"},{"index":10,"sense_key":"present%2:36:08::","synset_id":"portray.v.04"},{"index":11,"sense_key":"present%2:32:06::","synset_id":"confront.v.03"},{"index":12,"sense_key":"present%2:32:04::","synset_id":"present.v.12"},{"index":13,"sense_key":"present%2:32:03::","synset_id":"salute.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"present%2:39:00::","call_ids":["semeval2015.d002.s009.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"present%2:39:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1449,"cached_input_tokens":0,"output_tokens":134,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0042380000000000004,"input_uncached_usd":0.002898,"input_cached_usd":0.0,"output_usd":0.00134,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":12.284124540979974}
{"item_id":"semeval2015.d002.s009.t007","gold_sense_keys":["policy%1:09:00::"],"candidates":[{"index":1,"sense_key":"policy%1:09:00::","synset_id":"policy.n.01"},{"index":2,"sense_key":"policy%1:10:00::","synset_id":"policy.n.02"},{"index":3,"sense_key":"policy%1:10:02::","synset_id":"policy.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"policy%1:09:00::","call_ids":["semeval2015.d002.s009.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"policy%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":820,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00176,"input_uncached_usd":0.00164,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":10.37138683302328}
{"item_id":"semeval2015.d002.s009.t008","gold_sense_keys":["practice%1:04:00::"],"candidates":[{"index":1,"sense_key":"practice%1:04:00::","synset_id":"practice.n.01"},{"index":2,"sense_key":"practice%1:04:02::","synset_id":"exercise.n.03"},{"index":3,"sense_key":"practice%1:04:04::","synset_id":"practice.n.03"},{"index":4,"sense_key":"practice%1:04:03::","synset_id":"practice.n.04"},{"index":5,"sense_key":"practice%1:09:00::","synset_id":"practice.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"practice%1:04:00::","call_ids":["semeval2015.d002.s009.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"practice%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":936,"cached_input_tokens":0,"output_tokens":136,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003232,"input_uncached_usd":0.001872,"input_cached_usd":0.0,"output_usd":0.00136,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.3914397089974955}
{"item_id":"semeval2015.d002.s009.t009","gold_sense_keys":["improve%2:30:01::"],"candidates":[{"index":1,"sense_key":"improve%2:30:01::","synset_id":"better.v.02"},{"index":2,"sense_key":"improve%2:30:00::","synset_id":"better.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"improve%2:30:01::","call_ids":["semeval2015.d002.s009.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"improve%2:30:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":752,"cached_input_tokens":0,"output_tokens":86,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0023639999999999998,"input_uncached_usd":0.001504,"input_cached_usd":0.0,"output_usd":0.0008600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4816812500357628}
{"item_id":"semeval2015.d002.s009.t011","gold_sense_keys":["training%1:04:00::"],"candidates":[{"index":1,"sense_key":"training%1:04:00::","synset_id":"training.n.01"},{"index":2,"sense_key":"training%1:07:00::","synset_id":"education.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"training%1:04:00::","call_ids":["semeval2015.d002.s009.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"training%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":746,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001612,"input_uncached_usd":0.0014919999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1538092090049759}
{"item_id":"semeval2015.d002.s009.t012","gold_sense_keys":["development%1:04:01::"],"candidates":[{"index":1,"sense_key":"development%1:04:01::","synset_id":"development.n.01"},{"index":2,"sense_key":"development%1:22:02::","synset_id":"development.n.02"},{"index":3,"sense_key":"development%1:22:01::","synset_id":"growth.n.01"},{"index":4,"sense_key":"development%1:11:01::","synset_id":"development.n.04"},{"index":5,"sense_key":"development%1:04:02::","synset_id":"exploitation.n.01"},{"index":6,"sense_key":"development%1:15:00::","synset_id":"development.n.06"},{"index":7,"sense_key":"development%1:26:00::","synset_id":"development.n.07"},{"index":8,"sense_key":"development%1:22:00::","synset_id":"development.n.08"},{"index":9,"sense_key":"development%1:10:00::","synset_id":"development.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"development%1:04:01::","call_ids":["semeval2015.d002.s009.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"development%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1296,"cached_input_tokens":0,"output_tokens":117,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0037619999999999997,"input_uncached_usd":0.0025919999999999997,"input_cached_usd":0.0,"output_usd":0.00117,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6111756670288742}
{"item_id":"semeval2015.d002.s009.t014","gold_sense_keys":["worker%1:18:00::"],"candidates":[{"index":1,"sense_key":"worker%1:18:00::","synset_id":"worker.n.01"},{"index":2,"sense_key":"worker%1:18:01::","synset_id":"proletarian.n.01"},{"index":3,"sense_key":"worker%1:05:01::","synset_id":"worker.n.03"},{"index":4,"sense_key":"worker%1:18:02::","synset_id":"actor.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"worker%1:18:00::","call_ids":["semeval2015.d002.s009.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"worker%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":854,"cached_input_tokens":0,"output_tokens":58,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002288,"input_uncached_usd":0.0017079999999999999,"input_cached_usd":0.0,"output_usd":0.00058,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4969339590752497}
{"item_id":"semeval2015.d002.s009.t017","gold_sense_keys":["community%1:14:00::"],"candidates":[{"index":1,"sense_key":"community%1:14:00::","synset_id":"community.n.01"},{"index":2,"sense_key":"community%1:21:00::","synset_id":"community.n.02"},{"index":3,"sense_key":"community%1:14:04::","synset_id":"community.n.03"},{"index":4,"sense_key":"community%1:26:00::","synset_id":"community.n.04"},{"index":5,"sense_key":"community%1:15:00::","synset_id":"residential_district.n.01"},{"index":6,"sense_key":"community%1:14:02::","synset_id":"community.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"community%1:14:00::","call_ids":["semeval2015.d002.s009.t017__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"community%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":935,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00199,"input_uncached_usd":0.00187,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1105455410433933}
{"item_id":"semeval2015.d002.s010.t000","gold_sense_keys":["country%1:14:00::"],"candidates":[{"index":1,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":2,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":3,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":4,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":5,"sense_key":"country%1:15:02::","synset_id":"area.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"country%1:14:00::","call_ids":["semeval2015.d002.s010.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"country%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":966,"cached_input_tokens":0,"output_tokens":157,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003502,"input_uncached_usd":0.001932,"input_cached_usd":0.0,"output_usd":0.0015700000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.6915346669266}
{"item_id":"semeval2015.d002.s010.t005","gold_sense_keys":["case_study%1:10:00::"],"candidates":[{"index":1,"sense_key":"case_study%1:10:00::","synset_id":"case_study.n.01"},{"index":2,"sense_key":"case_study%1:04:00::","synset_id":"case_study.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"case_study%1:10:00::","call_ids":["semeval2015.d002.s010.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"case_study%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":728,"cached_input_tokens":0,"output_tokens":119,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002646,"input_uncached_usd":0.0014559999999999998,"input_cached_usd":0.0,"output_usd":0.00119,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7001381670124829}
{"item_id":"semeval2015.d002.s011.t000","gold_sense_keys":["case_study%1:10:00::"],"candidates":[{"index":1,"sense_key":"case_study%1:10:00::","synset_id":"case_study.n.01"},{"index":2,"sense_key":"case_study%1:04:00::","synset_id":"case_study.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"case_study%1:10:00::","call_ids":["semeval2015.d002.s011.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"case_study%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":700,"cached_input_tokens":0,"output_tokens":128,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00268,"input_uncached_usd":0.0014,"input_cached_usd":0.0,"output_usd":0.00128,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6419780420837924}
{"item_id":"semeval2015.d002.s011.t001","gold_sense_keys":["analyse%2:31:00::","analyse%2:31:04::"],"candidates":[{"index":1,"sense_key":"analyse%2:31:00::","synset_id":"analyze.v.01"},{"index":2,"sense_key":"analyse%2:31:04::","synset_id":"analyze.v.03"},{"index":3,"sense_key":"analyse%2:31:01::","synset_id":"analyze.v.02"},{"index":4,"sense_key":"analyse%2:31:02::","synset_id":"analyze.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"analyse%2:31:00::","call_ids":["semeval2015.d002.s011.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"analyse%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":922,"cached_input_tokens":0,"output_tokens":80,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002644,"input_uncached_usd":0.001844,"input_cached_usd":0.0,"output_usd":0.0008,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.100444750045426}
{"item_id":"semeval2015.d002.s011.t002","gold_sense_keys":["background%1:09:01::","background%1:26:00::"],"candidates":[{"index":1,"sense_key":"background%1:07:00::","synset_id":"background.n.01"},{"index":2,"sense_key":"background%1:09:00::","synset_id":"background.n.02"},{"index":3,"sense_key":"background%1:09:01::","synset_id":"background.n.03"},{"index":4,"sense_key":"background%1:11:00::","synset_id":"background.n.04"},{"index":5,"sense_key":"background%1:11:01::","synset_id":"background.n.05"},{"index":6,"sense_key":"background%1:26:00::","synset_id":"setting.n.02"},{"index":7,"sense_key":"background%1:06:01::","synset_id":"background.n.07"},{"index":8,"sense_key":"background%1:06:00::","synset_id":"backdrop.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"background%1:09:01::","call_ids":["semeval2015.d002.s011.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"background%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1083,"cached_input_tokens":0,"output_tokens":73,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002896,"input_uncached_usd":0.002166,"input_cached_usd":0.0,"output_usd":0.0007300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4912008340470493}
{"item_id":"semeval2015.d002.s011.t004","gold_sense_keys":["impact%1:19:00::"],"candidates":[{"index":1,"sense_key":"impact%1:11:00::","synset_id":"impact.n.01"},{"index":2,"sense_key":"impact%1:19:00::","synset_id":"impact.n.02"},{"index":3,"sense_key":"impact%1:04:00::","synset_id":"impingement.n.01"},{"index":4,"sense_key":"impact%1:04:01::","synset_id":"shock.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"impact%1:19:00::","call_ids":["semeval2015.d002.s011.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"impact%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":809,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001738,"input_uncached_usd":0.0016179999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1387927500763908}
{"item_id":"semeval2015.d002.s011.t005","gold_sense_keys":["employment%1:04:00::","employment%1:04:02::","employment%1:26:00::"],"candidates":[{"index":1,"sense_key":"employment%1:26:00::","synset_id":"employment.n.01"},{"index":2,"sense_key":"employment%1:04:00::","synset_id":"employment.n.02"},{"index":3,"sense_key":"employment%1:04:02::","synset_id":"employment.n.03"},{"index":4,"sense_key":"employment%1:04:01::","synset_id":"use.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"employment%1:26:00::","call_ids":["semeval2015.d002.s011.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"employment%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":827,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002374,"input_uncached_usd":0.0016539999999999999,"input_cached_usd":0.0,"output_usd":0.00072,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3625197079963982}
{"item_id":"semeval2015.d002.s011.t007","gold_sense_keys":["work%1:04:01::"],"candidates":[{"index":1,"sense_key":"work%1:04:00::","synset_id":"work.n.01"},{"index":2,"sense_key":"work%1:06:00::","synset_id":"work.n.02"},{"index":3,"sense_key":"work%1:04:01::","synset_id":"employment.n.02"},{"index":4,"sense_key":"work%1:09:00::","synset_id":"study.n.02"},{"index":5,"sense_key":"work%1:19:00::","synset_id":"work.n.05"},{"index":6,"sense_key":"work%1:06:01::","synset_id":"workplace.n.01"},{"index":7,"sense_key":"work%1:06:02::","synset_id":"oeuvre.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"work%1:04:01::","call_ids":["semeval2015.d002.s011.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"work%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1087,"cached_input_tokens":0,"output_tokens":114,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003314,"input_uncached_usd":0.0021739999999999997,"input_cached_usd":0.0,"output_usd":0.0011400000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.3719362909905612}
{"item_id":"semeval2015.d002.s011.t009","gold_sense_keys":["assess%2:31:00::"],"candidates":[{"index":1,"sense_key":"assess%2:31:00::","synset_id":"measure.v.04"},{"index":2,"sense_key":"assess%2:40:01::","synset_id":"assess.v.02"},{"index":3,"sense_key":"assess%2:40:00::","synset_id":"tax.v.02"},{"index":4,"sense_key":"assess%2:31:01::","synset_id":"assess.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"assess%2:31:00::","call_ids":["semeval2015.d002.s011.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"assess%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":834,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001788,"input_uncached_usd":0.001668,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0492477079387754}
{"item_id":"semeval2015.d002.s011.t010","gold_sense_keys":["benefit%1:07:00::"],"candidates":[{"index":1,"sense_key":"benefit%1:21:00::","synset_id":"benefit.n.01"},{"index":2,"sense_key":"benefit%1:07:00::","synset_id":"benefit.n.02"},{"index":3,"sense_key":"benefit%1:10:00::","synset_id":"benefit.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"benefit%1:07:00::","call_ids":["semeval2015.d002.s011.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"benefit%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":712,"cached_input_tokens":0,"output_tokens":90,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002324,"input_uncached_usd":0.001424,"input_cached_usd":0.0,"output_usd":0.0009000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4861977499676868}
{"item_id":"semeval2015.d002.s011.t011","gold_sense_keys":["problem%1:09:00::","problem%1:26:00::"],"candidates":[{"index":1,"sense_key":"problem%1:26:00::","synset_id":"problem.n.01"},{"index":2,"sense_key":"problem%1:10:00::","synset_id":"problem.n.02"},{"index":3,"sense_key":"problem%1:09:00::","synset_id":"trouble.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"problem%1:09:00::","call_ids":["semeval2015.d002.s011.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"problem%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":765,"cached_input_tokens":0,"output_tokens":107,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0026,"input_uncached_usd":0.00153,"input_cached_usd":0.0,"output_usd":0.00107,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.17729366698768}
{"item_id":"semeval2015.d002.s012.t000","gold_sense_keys":["visit%1:04:01::"],"candidates":[{"index":1,"sense_key":"visit%1:04:02::","synset_id":"visit.n.01"},{"index":2,"sense_key":"visit%1:14:00::","synset_id":"visit.n.02"},{"index":3,"sense_key":"visit%1:04:01::","synset_id":"visit.n.03"},{"index":4,"sense_key":"visit%1:04:00::","synset_id":"visit.n.04"},{"index":5,"sense_key":"visit%1:04:03::","synset_id":"sojourn.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"visit%1:04:01::","call_ids":["semeval2015.d002.s012.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"visit%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":833,"cached_input_tokens":0,"output_tokens":121,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002876,"input_uncached_usd":0.001666,"input_cached_usd":0.0,"output_usd":0.0012100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9932345419656485}
{"item_id":"semeval2015.d002.s012.t001","gold_sense_keys":["interview%1:10:01::"],"candidates":[{"index":1,"sense_key":"interview%1:10:01::","synset_id":"interview.n.01"},{"index":2,"sense_key":"interview%1:10:00::","synset_id":"consultation.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"interview%1:10:01::","call_ids":["semeval2015.d002.s012.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"interview%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":713,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001546,"input_uncached_usd":0.001426,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5744081670418382}
{"item_id":"semeval2015.d002.s013.t000","gold_sense_keys":["national%3:00:00::","national%3:00:01::","national%3:01:00::","national%5:00:00:domestic:00"],"candidates":[{"index":1,"sense_key":"national%3:01:00::","synset_id":"national.a.01"},{"index":2,"sense_key":"national%3:00:00::","synset_id":"national.a.02"},{"index":3,"sense_key":"national%3:00:01::","synset_id":"national.a.03"},{"index":4,"sense_key":"national%5:00:00:public:00","synset_id":"national.s.01"},{"index":5,"sense_key":"national%5:00:00:domestic:00","synset_id":"home.s.01"},{"index":6,"sense_key":"national%3:01:02::","synset_id":"national.a.06"},{"index":7,"sense_key":"national%3:01:01::","synset_id":"national.a.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"national%3:01:00::","call_ids":["semeval2015.d002.s013.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"national%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":949,"cached_input_tokens":0,"output_tokens":132,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0032180000000000004,"input_uncached_usd":0.001898,"input_cached_usd":0.0,"output_usd":0.0013200000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.867488499963656}
{"item_id":"semeval2015.d002.s013.t001","gold_sense_keys":["study%1:10:00::"],"candidates":[{"index":1,"sense_key":"study%1:04:00::","synset_id":"survey.n.01"},{"index":2,"sense_key":"study%1:09:03::","synset_id":"study.n.02"},{"index":3,"sense_key":"study%1:10:00::","synset_id":"report.n.01"},{"index":4,"sense_key":"study%1:09:04::","synset_id":"study.n.04"},{"index":5,"sense_key":"study%1:06:00::","synset_id":"study.n.05"},{"index":6,"sense_key":"study%1:09:02::","synset_id":"discipline.n.01"},{"index":7,"sense_key":"study%1:06:01::","synset_id":"sketch.n.01"},{"index":8,"sense_key":"study%1:09:01::","synset_id":"cogitation.n.02"},{"index":9,"sense_key":"study%1:18:00::","synset_id":"study.n.09"},{"index":10,"sense_key":"study%1:10:01::","synset_id":"study.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"study%1:10:00::","call_ids":["semeval2015.d002.s013.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"study%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1175,"cached_input_tokens":0,"output_tokens":153,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00388,"input_uncached_usd":0.00235,"input_cached_usd":0.0,"output_usd":0.0015300000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":9.581938124960288}
{"item_id":"semeval2015.d002.s013.t002","gold_sense_keys":["form%2:42:00::"],"candidates":[{"index":1,"sense_key":"form%2:41:00::","synset_id":"form.v.01"},{"index":2,"sense_key":"form%2:42:00::","synset_id":"form.v.02"},{"index":3,"sense_key":"form%2:42:02::","synset_id":"form.v.03"},{"index":4,"sense_key":"form%2:30:01::","synset_id":"shape.v.03"},{"index":5,"sense_key":"form%2:36:00::","synset_id":"shape.v.02"},{"index":6,"sense_key":"form%2:41:02::","synset_id":"imprint.v.01"},{"index":7,"sense_key":"form%2:30:05::","synset_id":"form.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"form%2:42:00::","call_ids":["semeval2015.d002.s013.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"form%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":989,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002518,"input_uncached_usd":0.0019779999999999997,"input_cached_usd":0.0,"output_usd":0.00054,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.790627958951518}
{"item_id":"semeval2015.d002.s013.t003","gold_sense_keys":["basis%1:09:00::"],"candidates":[{"index":1,"sense_key":"basis%1:24:00::","synset_id":"footing.n.02"},{"index":2,"sense_key":"basis%1:09:00::","synset_id":"basis.n.02"},{"index":3,"sense_key":"basis%1:24:01::","synset_id":"basis.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"basis%1:09:00::","call_ids":["semeval2015.d002.s013.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"basis%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":804,"cached_input_tokens":0,"output_tokens":125,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002858,"input_uncached_usd":0.0016079999999999998,"input_cached_usd":0.0,"output_usd":0.00125,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6214030410628766}
{"item_id":"semeval2015.d002.s013.t005","gold_sense_keys":["report%1:10:03::"],"candidates":[{"index":1,"sense_key":"report%1:10:03::","synset_id":"report.n.01"},{"index":2,"sense_key":"report%1:10:01::","synset_id":"report.n.02"},{"index":3,"sense_key":"report%1:10:00::","synset_id":"report.n.03"},{"index":4,"sense_key":"report%1:11:00::","synset_id":"report.n.04"},{"index":5,"sense_key":"report%1:10:04::","synset_id":"report_card.n.01"},{"index":6,"sense_key":"report%1:10:02::","synset_id":"composition.n.08"},{"index":7,"sense_key":"report%1:09:00::","synset_id":"reputation.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"report%1:10:03::","call_ids":["semeval2015.d002.s013.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"report%1:10:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1035,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0021899999999999997,"input_uncached_usd":0.00207,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2160541251068935}
{"item_id":"semeval2015.d002.s013.t007","gold_sense_keys":["overall%5:00:00:general:00"],"candidates":[{"index":1,"sense_key":"overall%5:00:00:general:00","synset_id":"overall.s.01"},{"index":2,"sense_key":"overall%5:00:00:gross:00","synset_id":"overall.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"overall%5:00:00:general:00","call_ids":["semeval2015.d002.s013.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"overall%5:00:00:general:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":669,"cached_input_tokens":0,"output_tokens":76,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002098,"input_uncached_usd":0.001338,"input_cached_usd":0.0,"output_usd":0.00076,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.555039249942638}
{"item_id":"semeval2015.d002.s013.t008","gold_sense_keys":["practice%1:04:00::"],"candidates":[{"index":1,"sense_key":"practice%1:04:00::","synset_id":"practice.n.01"},{"index":2,"sense_key":"practice%1:04:02::","synset_id":"exercise.n.03"},{"index":3,"sense_key":"practice%1:04:04::","synset_id":"practice.n.03"},{"index":4,"sense_key":"practice%1:04:03::","synset_id":"practice.n.04"},{"index":5,"sense_key":"practice%1:09:00::","synset_id":"practice.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"practice%1:04:00::","call_ids":["semeval2015.d002.s013.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"practice%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":891,"cached_input_tokens":0,"output_tokens":70,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002482,"input_uncached_usd":0.001782,"input_cached_usd":0.0,"output_usd":0.0007000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":14.05860366707202}
{"item_id":"semeval2015.d002.s014.t000","gold_sense_keys":["report%1:10:03::"],"candidates":[{"index":1,"sense_key":"report%1:10:03::","synset_id":"report.n.01"},{"index":2,"sense_key":"report%1:10:01::","synset_id":"report.n.02"},{"index":3,"sense_key":"report%1:10:00::","synset_id":"report.n.03"},{"index":4,"sense_key":"report%1:11:00::","synset_id":"report.n.04"},{"index":5,"sense_key":"report%1:10:04::","synset_id":"report_card.n.01"},{"index":6,"sense_key":"report%1:10:02::","synset_id":"composition.n.08"},{"index":7,"sense_key":"report%1:09:00::","synset_id":"reputation.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"report%1:10:03::","call_ids":["semeval2015.d002.s014.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"report%1:10:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":936,"cached_input_tokens":0,"output_tokens":54,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002412,"input_uncached_usd":0.001872,"input_cached_usd":0.0,"output_usd":0.00054,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":16.015975957969204}
{"item_id":"semeval2015.d002.s014.t003","gold_sense_keys":["publish%2:32:00::","publish%2:36:01::"],"candidates":[{"index":1,"sense_key":"publish%2:36:01::","synset_id":"print.v.01"},{"index":2,"sense_key":"publish%2:32:00::","synset_id":"publish.v.02"},{"index":3,"sense_key":"publish%2:36:00::","synset_id":"publish.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"publish%2:32:00::","call_ids":["semeval2015.d002.s014.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"publish%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":687,"cached_input_tokens":0,"output_tokens":117,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0025440000000000003,"input_uncached_usd":0.001374,"input_cached_usd":0.0,"output_usd":0.00117,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6881430000066757}
{"item_id":"semeval2015.d002.s014.t005","gold_sense_keys":["conference%1:14:00::"],"candidates":[{"index":1,"sense_key":"conference%1:14:00::","synset_id":"conference.n.01"},{"index":2,"sense_key":"conference%1:14:01::","synset_id":"league.n.01"},{"index":3,"sense_key":"conference%1:10:00::","synset_id":"conference.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"conference%1:14:00::","call_ids":["semeval2015.d002.s014.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"conference%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":641,"cached_input_tokens":0,"output_tokens":109,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002372,"input_uncached_usd":0.001282,"input_cached_usd":0.0,"output_usd":0.00109,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7617932920111343}
{"item_id":"semeval2015.d002.s014.t006","gold_sense_keys":["stage%2:36:01::"],"candidates":[{"index":1,"sense_key":"stage%2:36:00::","synset_id":"stage.v.01"},{"index":2,"sense_key":"stage%2:36:01::","synset_id":"stage.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"stage%2:36:01::","call_ids":["semeval2015.d002.s014.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"stage%2:36:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":611,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001342,"input_uncached_usd":0.001222,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9911255419719964}
{"item_id":"semeval2015.d002.s015.t000","gold_sense_keys":["foundation%1:14:00::"],"candidates":[{"index":1,"sense_key":"foundation%1:24:00::","synset_id":"foundation.n.01"},{"index":2,"sense_key":"foundation%1:14:00::","synset_id":"foundation.n.02"},{"index":3,"sense_key":"foundation%1:06:00::","synset_id":"foundation.n.03"},{"index":4,"sense_key":"foundation%1:09:01::","synset_id":"foundation.n.04"},{"index":5,"sense_key":"foundation%1:09:00::","synset_id":"basis.n.02"},{"index":6,"sense_key":"foundation%1:06:01::","synset_id":"foundation_garment.n.01"},{"index":7,"sense_key":"foundation%1:04:00::","synset_id":"initiation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"foundation%1:14:00::","call_ids":["semeval2015.d002.s015.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"foundation%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":953,"cached_input_tokens":0,"output_tokens":82,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002726,"input_uncached_usd":0.001906,"input_cached_usd":0.0,"output_usd":0.0008200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8721125409938395}
{"item_id":"semeval2015.d002.s015.t001","gold_sense_keys":["aim%2:31:01::"],"candidates":[{"index":1,"sense_key":"aim%2:33:00::","synset_id":"aim.v.01"},{"index":2,"sense_key":"aim%2:31:01::","synset_id":"aim.v.02"},{"index":3,"sense_key":"aim%2:32:09::","synset_id":"drive.v.11"},{"index":4,"sense_key":"aim%2:31:02::","synset_id":"calculate.v.05"},{"index":5,"sense_key":"aim%2:33:10::","synset_id":"target.v.01"},{"index":6,"sense_key":"aim%2:32:00::","synset_id":"aim.v.06"},{"index":7,"sense_key":"aim%2:31:00::","synset_id":"draw_a_bead_on.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"aim%2:31:01::","call_ids":["semeval2015.d002.s015.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"aim%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":902,"cached_input_tokens":0,"output_tokens":129,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003094,"input_uncached_usd":0.0018039999999999998,"input_cached_usd":0.0,"output_usd":0.0012900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":14.497495707939379}
{"item_id":"semeval2015.d002.s015.t002","gold_sense_keys":["document%2:32:00::"],"candidates":[{"index":1,"sense_key":"document%2:32:00::","synset_id":"document.v.01"},{"index":2,"sense_key":"document%2:31:00::","synset_id":"document.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"document%2:32:00::","call_ids":["semeval2015.d002.s015.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"document%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":538,"cached_input_tokens":0,"output_tokens":65,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001726,"input_uncached_usd":0.001076,"input_cached_usd":0.0,"output_usd":0.0006500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.284857040969655}
{"item_id":"semeval2015.d002.s015.t003","gold_sense_keys":["characteristic%1:07:00::","characteristic%1:09:00::"],"candidates":[{"index":1,"sense_key":"characteristic%1:09:00::","synset_id":"feature.n.01"},{"index":2,"sense_key":"characteristic%1:07:00::","synset_id":"characteristic.n.02"},{"index":3,"sense_key":"characteristic%1:10:00::","synset_id":"characteristic.n.03"},{"index":4,"sense_key":"characteristic%1:07:01::","synset_id":"characteristic.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"characteristic%1:09:00::","call_ids":["semeval2015.d002.s015.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"characteristic%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":688,"cached_input_tokens":0,"output_tokens":130,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002676,"input_uncached_usd":0.0013759999999999998,"input_cached_usd":0.0,"output_usd":0.0013000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.947263458976522}
{"item_id":"semeval2015.d002.s015.t004","gold_sense_keys":["age%1:07:00::"],"candidates":[{"index":1,"sense_key":"age%1:07:00::","synset_id":"age.n.01"},{"index":2,"sense_key":"age%1:28:02::","synset_id":"historic_period.n.01"},{"index":3,"sense_key":"age%1:28:00::","synset_id":"age.n.03"},{"index":4,"sense_key":"age%1:28:03::","synset_id":"long_time.n.01"},{"index":5,"sense_key":"age%1:28:01::","synset_id":"old_age.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"age%1:07:00::","call_ids":["semeval2015.d002.s015.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"age%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":748,"cached_input_tokens":0,"output_tokens":287,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004366,"input_uncached_usd":0.001496,"input_cached_usd":0.0,"output_usd":0.00287,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.049980124924332}
{"item_id":"semeval2015.d002.s015.t005","gold_sense_keys":["barrier%1:09:00::"],"candidates":[{"index":1,"sense_key":"barrier%1:06:00::","synset_id":"barrier.n.01"},{"index":2,"sense_key":"barrier%1:09:00::","synset_id":"barrier.n.02"},{"index":3,"sense_key":"barrier%1:17:00::","synset_id":"barrier.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"barrier%1:09:00::","call_ids":["semeval2015.d002.s015.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"barrier%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":611,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001342,"input_uncached_usd":0.001222,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5272667499957606}
{"item_id":"semeval2015.d002.s015.t006","gold_sense_keys":["job%1:04:00::"],"candidates":[{"index":1,"sense_key":"job%1:04:00::","synset_id":"occupation.n.01"},{"index":2,"sense_key":"job%1:04:01::","synset_id":"job.n.02"},{"index":3,"sense_key":"job%1:06:00::","synset_id":"job.n.03"},{"index":4,"sense_key":"job%1:06:01::","synset_id":"job.n.04"},{"index":5,"sense_key":"job%1:04:04::","synset_id":"job.n.05"},{"index":6,"sense_key":"job%1:04:05::","synset_id":"job.n.06"},{"index":7,"sense_key":"job%1:04:03::","synset_id":"job.n.07"},{"index":8,"sense_key":"job%1:26:00::","synset_id":"problem.n.01"},{"index":9,"sense_key":"job%1:18:00::","synset_id":"job.n.09"},{"index":10,"sense_key":"job%1:18:01::","synset_id":"job.n.10"},{"index":11,"sense_key":"job%1:10:01::","synset_id":"job.n.11"},{"index":12,"sense_key":"job%1:10:00::","synset_id":"job.n.12"},{"index":13,"sense_key":"job%1:04:02::","synset_id":"caper.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"job%1:04:00::","call_ids":["semeval2015.d002.s015.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"job%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1199,"cached_input_tokens":0,"output_tokens":105,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003448,"input_uncached_usd":0.002398,"input_cached_usd":0.0,"output_usd":0.0010500000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.247769583016634}
{"item_id":"semeval2015.d002.s015.t008","gold_sense_keys":["training%1:04:00::"],"candidates":[{"index":1,"sense_key":"training%1:04:00::","synset_id":"training.n.01"},{"index":2,"sense_key":"training%1:07:00::","synset_id":"education.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"training%1:04:00::","call_ids":["semeval2015.d002.s015.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"training%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":573,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001266,"input_uncached_usd":0.0011459999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0482688749907538}
{"item_id":"semeval2015.d002.s015.t009","gold_sense_keys":["analyse%2:31:00::","analyse%2:31:04::"],"candidates":[{"index":1,"sense_key":"analyse%2:31:00::","synset_id":"analyze.v.01"},{"index":2,"sense_key":"analyse%2:31:04::","synset_id":"analyze.v.03"},{"index":3,"sense_key":"analyse%2:31:01::","synset_id":"analyze.v.02"},{"index":4,"sense_key":"analyse%2:31:02::","synset_id":"analyze.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"analyse%2:31:00::","call_ids":["semeval2015.d002.s015.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"analyse%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":795,"cached_input_tokens":0,"output_tokens":138,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00297,"input_uncached_usd":0.0015899999999999998,"input_cached_usd":0.0,"output_usd":0.0013800000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.043294374947436}
{"item_id":"semeval2015.d002.s015.t010","gold_sense_keys":["cause%1:03:00::","cause%1:11:00::"],"candidates":[{"index":1,"sense_key":"cause%1:11:00::","synset_id":"cause.n.01"},{"index":2,"sense_key":"cause%1:10:00::","synset_id":"cause.n.02"},{"index":3,"sense_key":"cause%1:04:01::","synset_id":"campaign.n.02"},{"index":4,"sense_key":"cause%1:03:00::","synset_id":"causal_agent.n.01"},{"index":5,"sense_key":"cause%1:04:00::","synset_id":"lawsuit.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cause%1:11:00::","call_ids":["semeval2015.d002.s015.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cause%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":815,"cached_input_tokens":0,"output_tokens":128,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0029100000000000003,"input_uncached_usd":0.00163,"input_cached_usd":0.0,"output_usd":0.00128,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8315217080526054}
{"item_id":"semeval2015.d002.s016.t000","gold_sense_keys":["perspective%1:09:00::"],"candidates":[{"index":1,"sense_key":"perspective%1:09:00::","synset_id":"position.n.03"},{"index":2,"sense_key":"perspective%1:07:00::","synset_id":"perspective.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"perspective%1:09:00::","call_ids":["semeval2015.d002.s016.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"perspective%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":601,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001322,"input_uncached_usd":0.001202,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":4.587994124973193}
{"item_id":"semeval2015.d002.s016.t001","gold_sense_keys":["social%3:01:00::"],"candidates":[{"index":1,"sense_key":"social%3:01:00::","synset_id":"social.a.01"},{"index":2,"sense_key":"social%3:00:00::","synset_id":"social.a.02"},{"index":3,"sense_key":"social%3:01:02::","synset_id":"social.a.03"},{"index":4,"sense_key":"social%5:00:00:sociable:00","synset_id":"social.s.01"},{"index":5,"sense_key":"social%5:00:00:gregarious:00","synset_id":"social.s.02"},{"index":6,"sense_key":"social%5:00:00:friendly:01","synset_id":"social.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"social%3:01:00::","call_ids":["semeval2015.d002.s016.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"social%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":815,"cached_input_tokens":0,"output_tokens":75,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00238,"input_uncached_usd":0.00163,"input_cached_usd":0.0,"output_usd":0.00075,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4465262920130044}
{"item_id":"semeval2015.d002.s016.t004","gold_sense_keys":["particular%5:00:00:uncommon:00"],"candidates":[{"index":1,"sense_key":"particular%5:00:00:specific:00","synset_id":"particular.s.01"},{"index":2,"sense_key":"particular%5:00:02:specific:00","synset_id":"particular.s.02"},{"index":3,"sense_key":"particular%5:00:00:uncommon:00","synset_id":"especial.s.01"},{"index":4,"sense_key":"particular%5:00:00:primary:00","synset_id":"particular.s.04"},{"index":5,"sense_key":"particular%5:00:00:fastidious:00","synset_id":"finical.s.01"},{"index":6,"sense_key":"particular%5:00:00:careful:00","synset_id":"particular.s.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"particular%5:00:00:uncommon:00","call_ids":["semeval2015.d002.s016.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"particular%5:00:00:uncommon:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":893,"cached_input_tokens":0,"output_tokens":114,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0029259999999999998,"input_uncached_usd":0.0017859999999999998,"input_cached_usd":0.0,"output_usd":0.0011400000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.4134004999650642}
{"item_id":"semeval2015.d002.s016.t005","gold_sense_keys":["attention%1:09:00::"],"candidates":[{"index":1,"sense_key":"attention%1:09:00::","synset_id":"attention.n.01"},{"index":2,"sense_key":"attention%1:04:01::","synset_id":"care.n.01"},{"index":3,"sense_key":"attention%1:09:02::","synset_id":"attention.n.03"},{"index":4,"sense_key":"attention%1:04:02::","synset_id":"attention.n.04"},{"index":5,"sense_key":"attention%1:09:01::","synset_id":"attention.n.05"},{"index":6,"sense_key":"attention%1:07:00::","synset_id":"attention.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"attention%1:09:00::","call_ids":["semeval2015.d002.s016.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"attention%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":828,"cached_input_tokens":0,"output_tokens":120,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002856,"input_uncached_usd":0.001656,"input_cached_usd":0.0,"output_usd":0.0012000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.3341730419779196}
{"item_id":"semeval2015.d002.s016.t006","gold_sense_keys":["pay%2:31:00::","pay%2:32:00::"],"candidates":[{"index":1,"sense_key":"pay%2:40:00::","synset_id":"pay.v.01"},{"index":2,"sense_key":"pay%2:32:00::","synset_id":"give.v.05"},{"index":3,"sense_key":"pay%2:40:04::","synset_id":"pay_up.v.01"},{"index":4,"sense_key":"pay%2:40:01::","synset_id":"yield.v.10"},{"index":5,"sense_key":"pay%2:40:02::","synset_id":"pay.v.05"},{"index":6,"sense_key":"pay%2:31:00::","synset_id":"give.v.10"},{"index":7,"sense_key":"pay%2:42:00::","synset_id":"pay.v.07"},{"index":8,"sense_key":"pay%2:41:00::","synset_id":"pay.v.08"},{"index":9,"sense_key":"pay%2:31:03::","synset_id":"pay.v.09"},{"index":10,"sense_key":"pay%2:40:13::","synset_id":"pay.v.10"},{"index":11,"sense_key":"pay%2:40:14::","synset_id":"pay.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"pay%2:32:00::","call_ids":["semeval2015.d002.s016.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"pay%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1081,"cached_input_tokens":0,"output_tokens":146,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0036220000000000002,"input_uncached_usd":0.002162,"input_cached_usd":0.0,"output_usd":0.0014600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.4634895409690216}
{"item_id":"semeval2015.d002.s016.t007","gold_sense_keys":["proposal%1:10:00::"],"candidates":[{"index":1,"sense_key":"proposal%1:10:00::","synset_id":"proposal.n.01"},{"index":2,"sense_key":"proposal%1:10:02::","synset_id":"marriage_proposal.n.01"},{"index":3,"sense_key":"proposal%1:10:01::","synset_id":"proposal.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"proposal%1:10:00::","call_ids":["semeval2015.d002.s016.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"proposal%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":628,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001376,"input_uncached_usd":0.001256,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9479047920322046}
{"item_id":"semeval2015.d002.s016.t008","gold_sense_keys":["remove%2:30:00::","remove%2:40:00::"],"candidates":[{"index":1,"sense_key":"remove%2:30:00::","synset_id":"remove.v.01"},{"index":2,"sense_key":"remove%2:41:00::","synset_id":"remove.v.02"},{"index":3,"sense_key":"remove%2:40:00::","synset_id":"get_rid_of.v.01"},{"index":4,"sense_key":"remove%2:41:02::","synset_id":"take_out.v.01"},{"index":5,"sense_key":"remove%2:38:00::","synset_id":"remove.v.05"},{"index":6,"sense_key":"remove%2:30:01::","synset_id":"absent.v.01"},{"index":7,"sense_key":"remove%2:41:01::","synset_id":"murder.v.01"},{"index":8,"sense_key":"remove%2:30:02::","synset_id":"remove.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"remove%2:30:02::","call_ids":["semeval2015.d002.s016.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"remove%2:30:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":989,"cached_input_tokens":0,"output_tokens":105,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003028,"input_uncached_usd":0.0019779999999999997,"input_cached_usd":0.0,"output_usd":0.0010500000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6082640830427408}
{"item_id":"semeval2015.d002.s016.t009","gold_sense_keys":["age%1:07:00::"],"candidates":[{"index":1,"sense_key":"age%1:07:00::","synset_id":"age.n.01"},{"index":2,"sense_key":"age%1:28:02::","synset_id":"historic_period.n.01"},{"index":3,"sense_key":"age%1:28:00::","synset_id":"age.n.03"},{"index":4,"sense_key":"age%1:28:03::","synset_id":"long_time.n.01"},{"index":5,"sense_key":"age%1:28:01::","synset_id":"old_age.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"age%1:07:00::","call_ids":["semeval2015.d002.s016.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"age%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":768,"cached_input_tokens":0,"output_tokens":226,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0037960000000000003,"input_uncached_usd":0.001536,"input_cached_usd":0.0,"output_usd":0.0022600000000000003,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.034544416004792}
{"item_id":"semeval2015.d002.s016.t010","gold_sense_keys":["barrier%1:09:00::"],"candidates":[{"index":1,"sense_key":"barrier%1:06:00::","synset_id":"barrier.n.01"},{"index":2,"sense_key":"barrier%1:09:00::","synset_id":"barrier.n.02"},{"index":3,"sense_key":"barrier%1:17:00::","synset_id":"barrier.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"barrier%1:09:00::","call_ids":["semeval2015.d002.s016.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"barrier%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":631,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001382,"input_uncached_usd":0.0012619999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.992208042065613}
{"item_id":"semeval2015.d002.s017.t002","gold_sense_keys":["assess%2:31:00::"],"candidates":[{"index":1,"sense_key":"assess%2:31:00::","synset_id":"measure.v.04"},{"index":2,"sense_key":"assess%2:40:01::","synset_id":"assess.v.02"},{"index":3,"sense_key":"assess%2:40:00::","synset_id":"tax.v.02"},{"index":4,"sense_key":"assess%2:31:01::","synset_id":"assess.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"assess%2:31:00::","call_ids":["semeval2015.d002.s017.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"assess%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":726,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001572,"input_uncached_usd":0.001452,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":4.377080750069581}
{"item_id":"semeval2015.d002.s017.t005","gold_sense_keys":["private%3:00:00::"],"candidates":[{"index":1,"sense_key":"private%3:00:00::","synset_id":"private.a.01"},{"index":2,"sense_key":"private%5:00:02:personal:00","synset_id":"private.s.01"},{"index":3,"sense_key":"private%5:00:00:personal:00","synset_id":"individual.s.03"},{"index":4,"sense_key":"private%5:00:00:inward:00","synset_id":"secret.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"private%3:00:00::","call_ids":["semeval2015.d002.s017.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"private%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":688,"cached_input_tokens":0,"output_tokens":160,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002976,"input_uncached_usd":0.0013759999999999998,"input_cached_usd":0.0,"output_usd":0.0016,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.323764167027548}
{"item_id":"semeval2015.d002.s017.t006","gold_sense_keys":["sector%1:14:00::"],"candidates":[{"index":1,"sense_key":"sector%1:25:00::","synset_id":"sector.n.01"},{"index":2,"sense_key":"sector%1:14:00::","synset_id":"sector.n.02"},{"index":3,"sense_key":"sector%1:09:00::","synset_id":"sector.n.03"},{"index":4,"sense_key":"sector%1:23:00::","synset_id":"sector.n.04"},{"index":5,"sense_key":"sector%1:15:00::","synset_id":"sector.n.05"},{"index":6,"sense_key":"sector%1:06:00::","synset_id":"sector.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"sector%1:14:00::","call_ids":["semeval2015.d002.s017.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"sector%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":759,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0016380000000000001,"input_uncached_usd":0.001518,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1469145830487832}
{"item_id":"semeval2015.d002.s017.t007","gold_sense_keys":["age%1:07:00::"],"candidates":[{"index":1,"sense_key":"age%1:07:00::","synset_id":"age.n.01"},{"index":2,"sense_key":"age%1:28:02::","synset_id":"historic_period.n.01"},{"index":3,"sense_key":"age%1:28:00::","synset_id":"age.n.03"},{"index":4,"sense_key":"age%1:28:03::","synset_id":"long_time.n.01"},{"index":5,"sense_key":"age%1:28:01::","synset_id":"old_age.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"age%1:28:00::","call_ids":["semeval2015.d002.s017.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"age%1:28:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":767,"cached_input_tokens":0,"output_tokens":196,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0034940000000000006,"input_uncached_usd":0.001534,"input_cached_usd":0.0,"output_usd":0.0019600000000000004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.054933000006713}
{"item_id":"semeval2015.d002.s017.t008","gold_sense_keys":["barrier%1:09:00::"],"candidates":[{"index":1,"sense_key":"barrier%1:06:00::","synset_id":"barrier.n.01"},{"index":2,"sense_key":"barrier%1:09:00::","synset_id":"barrier.n.02"},{"index":3,"sense_key":"barrier%1:17:00::","synset_id":"barrier.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"barrier%1:09:00::","call_ids":["semeval2015.d002.s017.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"barrier%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":630,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00138,"input_uncached_usd":0.0012599999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.7059641659725457}
{"item_id":"semeval2015.d002.s017.t009","gold_sense_keys":["especially%4:02:00::"],"candidates":[{"index":1,"sense_key":"especially%4:02:00::","synset_id":"particularly.r.01"},{"index":2,"sense_key":"especially%4:02:01::","synset_id":"specially.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"especially%4:02:00::","call_ids":["semeval2015.d002.s017.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"especially%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":610,"cached_input_tokens":0,"output_tokens":91,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00213,"input_uncached_usd":0.00122,"input_cached_usd":0.0,"output_usd":0.0009100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7503692500758916}
{"item_id":"semeval2015.d002.s017.t010","gold_sense_keys":["older%5:00:00:experienced:00","older%5:00:00:old:02"],"candidates":[{"index":1,"sense_key":"older%5:00:00:old:02","synset_id":"aged.s.01"},{"index":2,"sense_key":"older%5:00:00:senior:00","synset_id":"elder.s.01"},{"index":3,"sense_key":"older%5:00:00:experienced:00","synset_id":"old.s.02"},{"index":4,"sense_key":"old%3:00:02::","synset_id":"old.a.01"},{"index":5,"sense_key":"old%3:00:01::","synset_id":"old.a.02"},{"index":6,"sense_key":"old%5:00:00:familiar:00","synset_id":"old.s.01"},{"index":7,"sense_key":"older%5:00:00:experienced:00","synset_id":"old.s.02"},{"index":8,"sense_key":"erstwhile%5:00:00:past:00","synset_id":"erstwhile.s.01"},{"index":9,"sense_key":"honest-to-god%5:00:00:genuine:00","synset_id":"honest-to-god.s.01"},{"index":10,"sense_key":"old%5:00:00:early:01","synset_id":"old.s.05"},{"index":11,"sense_key":"previous%5:00:00:preceding:00","synset_id":"previous.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"older%5:00:00:old:02","call_ids":["semeval2015.d002.s017.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"older%5:00:00:old:02","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1250,"cached_input_tokens":0,"output_tokens":150,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004,"input_uncached_usd":0.0025,"input_cached_usd":0.0,"output_usd":0.0015,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0192002919502556}
{"item_id":"semeval2015.d002.s017.t011","gold_sense_keys":["worker%1:18:00::"],"candidates":[{"index":1,"sense_key":"worker%1:18:00::","synset_id":"worker.n.01"},{"index":2,"sense_key":"worker%1:18:01::","synset_id":"proletarian.n.01"},{"index":3,"sense_key":"worker%1:05:01::","synset_id":"worker.n.03"},{"index":4,"sense_key":"worker%1:18:02::","synset_id":"actor.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"worker%1:18:00::","call_ids":["semeval2015.d002.s017.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"worker%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":700,"cached_input_tokens":0,"output_tokens":80,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0022,"input_uncached_usd":0.0014,"input_cached_usd":0.0,"output_usd":0.0008,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2514796670293435}
{"item_id":"semeval2015.d002.s018.t000","gold_sense_keys":["seek%2:35:00::"],"candidates":[{"index":1,"sense_key":"seek%2:40:00::","synset_id":"seek.v.01"},{"index":2,"sense_key":"seek%2:35:00::","synset_id":"search.v.01"},{"index":3,"sense_key":"seek%2:41:00::","synset_id":"try.v.01"},{"index":4,"sense_key":"seek%2:38:00::","synset_id":"seek.v.04"},{"index":5,"sense_key":"seek%2:32:00::","synset_id":"seek.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"seek%2:40:00::","call_ids":["semeval2015.d002.s018.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"seek%2:40:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":765,"cached_input_tokens":0,"output_tokens":92,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00245,"input_uncached_usd":0.00153,"input_cached_usd":0.0,"output_usd":0.00092,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7008615829981863}
{"item_id":"semeval2015.d002.s018.t002","gold_sense_keys":["early%3:00:00::"],"candidates":[{"index":1,"sense_key":"early%3:00:00::","synset_id":"early.a.01"},{"index":2,"sense_key":"early%3:00:02::","synset_id":"early.a.02"},{"index":3,"sense_key":"early%5:00:00:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"early%5:00:00:young:00","synset_id":"early.s.02"},{"index":5,"sense_key":"early%3:00:01::","synset_id":"early.a.05"},{"index":6,"sense_key":"early%5:00:01:future:00","synset_id":"early.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"early%3:00:00::","call_ids":["semeval2015.d002.s018.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"early%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":834,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001788,"input_uncached_usd":0.001668,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.049432208063081}
{"item_id":"semeval2015.d002.s018.t003","gold_sense_keys":["retirement%1:04:00::","retirement%1:26:00::"],"candidates":[{"index":1,"sense_key":"retirement%1:26:00::","synset_id":"retirement.n.01"},{"index":2,"sense_key":"retirement%1:04:00::","synset_id":"retirement.n.02"},{"index":3,"sense_key":"retirement%1:04:01::","synset_id":"retirement.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"retirement%1:04:00::","call_ids":["semeval2015.d002.s018.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"retirement%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":631,"cached_input_tokens":0,"output_tokens":99,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0022519999999999997,"input_uncached_usd":0.0012619999999999999,"input_cached_usd":0.0,"output_usd":0.00099,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.348095666966401}
{"item_id":"semeval2015.d002.s018.t004","gold_sense_keys":["redundancy%1:07:00::"],"candidates":[{"index":1,"sense_key":"redundancy%1:10:00::","synset_id":"redundancy.n.01"},{"index":2,"sense_key":"redundancy%1:07:00::","synset_id":"redundancy.n.02"},{"index":3,"sense_key":"redundancy%1:09:00::","synset_id":"redundancy.n.03"},{"index":4,"sense_key":"redundancy%1:04:00::","synset_id":"redundancy.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"redundancy%1:07:00::","call_ids":["semeval2015.d002.s018.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"redundancy%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":692,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0015040000000000001,"input_uncached_usd":0.001384,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8524791669333354}
{"item_id":"semeval2015.d002.s018.t007","gold_sense_keys":["concerned%5:00:00:involved:00"],"candidates":[{"index":1,"sense_key":"concerned%3:00:00::","synset_id":"concerned.a.01"},{"index":2,"sense_key":"concerned%5:00:00:involved:00","synset_id":"concerned.s.01"},{"index":3,"sense_key":"concerned%5:00:01:involved:00","synset_id":"implicated.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"concerned%5:00:00:involved:00","call_ids":["semeval2015.d002.s018.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"concerned%5:00:00:involved:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":704,"cached_input_tokens":0,"output_tokens":133,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002738,"input_uncached_usd":0.001408,"input_cached_usd":0.0,"output_usd":0.00133,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.403914791997522}
{"item_id":"semeval2015.d002.s018.t008","gold_sense_keys":["retention%1:04:00::"],"candidates":[{"index":1,"sense_key":"retention%1:04:00::","synset_id":"retention.n.01"},{"index":2,"sense_key":"retention%1:09:00::","synset_id":"memory.n.03"},{"index":3,"sense_key":"retention%1:07:00::","synset_id":"retentiveness.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"retention%1:04:00::","call_ids":["semeval2015.d002.s018.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"retention%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":662,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001444,"input_uncached_usd":0.0013239999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0951417080359533}
{"item_id":"semeval2015.d002.s018.t011","gold_sense_keys":["older%5:00:00:experienced:00","older%5:00:00:old:02"],"candidates":[{"index":1,"sense_key":"older%5:00:00:old:02","synset_id":"aged.s.01"},{"index":2,"sense_key":"older%5:00:00:senior:00","synset_id":"elder.s.01"},{"index":3,"sense_key":"older%5:00:00:experienced:00","synset_id":"old.s.02"},{"index":4,"sense_key":"old%3:00:02::","synset_id":"old.a.01"},{"index":5,"sense_key":"old%3:00:01::","synset_id":"old.a.02"},{"index":6,"sense_key":"old%5:00:00:familiar:00","synset_id":"old.s.01"},{"index":7,"sense_key":"older%5:00:00:experienced:00","synset_id":"old.s.02"},{"index":8,"sense_key":"erstwhile%5:00:00:past:00","synset_id":"erstwhile.s.01"},{"index":9,"sense_key":"honest-to-god%5:00:00:genuine:00","synset_id":"honest-to-god.s.01"},{"index":10,"sense_key":"old%5:00:00:early:01","synset_id":"old.s.05"},{"index":11,"sense_key":"previous%5:00:00:preceding:00","synset_id":"previous.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"older%5:00:00:old:02","call_ids":["semeval2015.d002.s018.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"older%5:00:00:old:02","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1262,"cached_input_tokens":0,"output_tokens":150,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004024,"input_uncached_usd":0.0025239999999999998,"input_cached_usd":0.0,"output_usd":0.0015,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":14.821640999987721}
{"item_id":"semeval2015.d002.s018.t012","gold_sense_keys":["worker%1:18:00::"],"candidates":[{"index":1,"sense_key":"worker%1:18:00::","synset_id":"worker.n.01"},{"index":2,"sense_key":"worker%1:18:01::","synset_id":"proletarian.n.01"},{"index":3,"sense_key":"worker%1:05:01::","synset_id":"worker.n.03"},{"index":4,"sense_key":"worker%1:18:02::","synset_id":"actor.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"worker%1:18:00::","call_ids":["semeval2015.d002.s018.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"worker%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":712,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001544,"input_uncached_usd":0.001424,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4467970000114292}
{"item_id":"semeval2015.d002.s019.t000","gold_sense_keys":["gender%1:07:00::"],"candidates":[{"index":1,"sense_key":"gender%1:10:00::","synset_id":"gender.n.01"},{"index":2,"sense_key":"gender%1:07:00::","synset_id":"sex.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"gender%1:07:00::","call_ids":["semeval2015.d002.s019.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"gender%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":681,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001482,"input_uncached_usd":0.001362,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":14.193259124993347}
{"item_id":"semeval2015.d002.s019.t001","gold_sense_keys":["ethnic%5:00:00:social:00"],"candidates":[{"index":1,"sense_key":"ethnic%5:00:00:social:00","synset_id":"cultural.s.01"},{"index":2,"sense_key":"ethnic%5:00:00:irreligious:00","synset_id":"heathen.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ethnic%5:00:00:social:00","call_ids":["semeval2015.d002.s019.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ethnic%5:00:00:social:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":679,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001478,"input_uncached_usd":0.0013579999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.951314750011079}
{"item_id":"semeval2015.d002.s019.t002","gold_sense_keys":["background%1:07:00::"],"candidates":[{"index":1,"sense_key":"background%1:07:00::","synset_id":"background.n.01"},{"index":2,"sense_key":"background%1:09:00::","synset_id":"background.n.02"},{"index":3,"sense_key":"background%1:09:01::","synset_id":"background.n.03"},{"index":4,"sense_key":"background%1:11:00::","synset_id":"background.n.04"},{"index":5,"sense_key":"background%1:11:01::","synset_id":"background.n.05"},{"index":6,"sense_key":"background%1:26:00::","synset_id":"setting.n.02"},{"index":7,"sense_key":"background%1:06:01::","synset_id":"background.n.07"},{"index":8,"sense_key":"background%1:06:00::","synset_id":"backdrop.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"background%1:07:00::","call_ids":["semeval2015.d002.s019.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"background%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1003,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002126,"input_uncached_usd":0.002006,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":13.930500917020254}
{"item_id":"semeval2015.d002.s019.t003","gold_sense_keys":["worker%1:18:00::"],"candidates":[{"index":1,"sense_key":"worker%1:18:00::","synset_id":"worker.n.01"},{"index":2,"sense_key":"worker%1:18:01::","synset_id":"proletarian.n.01"},{"index":3,"sense_key":"worker%1:05:01::","synset_id":"worker.n.03"},{"index":4,"sense_key":"worker%1:18:02::","synset_id":"actor.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"worker%1:18:00::","call_ids":["semeval2015.d002.s019.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"worker%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":724,"cached_input_tokens":0,"output_tokens":103,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002478,"input_uncached_usd":0.0014479999999999999,"input_cached_usd":0.0,"output_usd":0.00103,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.540982833947055}
{"item_id":"semeval2015.d002.s019.t004","gold_sense_keys":["relate%2:42:00::","relate%2:42:01::"],"candidates":[{"index":1,"sense_key":"relate%2:31:00::","synset_id":"associate.v.01"},{"index":2,"sense_key":"relate%2:42:00::","synset_id":"refer.v.02"},{"index":3,"sense_key":"relate%2:32:00::","synset_id":"relate.v.03"},{"index":4,"sense_key":"relate%2:42:01::","synset_id":"relate.v.04"},{"index":5,"sense_key":"relate%2:41:00::","synset_id":"relate.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"relate%2:42:01::","call_ids":["semeval2015.d002.s019.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"relate%2:42:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":801,"cached_input_tokens":0,"output_tokens":176,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003362,"input_uncached_usd":0.001602,"input_cached_usd":0.0,"output_usd":0.00176,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.3067010829690844}
{"item_id":"semeval2015.d002.s019.t005","gold_sense_keys":["age%1:07:00::"],"candidates":[{"index":1,"sense_key":"age%1:07:00::","synset_id":"age.n.01"},{"index":2,"sense_key":"age%1:28:02::","synset_id":"historic_period.n.01"},{"index":3,"sense_key":"age%1:28:00::","synset_id":"age.n.03"},{"index":4,"sense_key":"age%1:28:03::","synset_id":"long_time.n.01"},{"index":5,"sense_key":"age%1:28:01::","synset_id":"old_age.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"age%1:07:00::","call_ids":["semeval2015.d002.s019.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"age%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":791,"cached_input_tokens":0,"output_tokens":300,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004582,"input_uncached_usd":0.0015819999999999999,"input_cached_usd":0.0,"output_usd":0.003,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.28055891697295}
{"item_id":"semeval2015.d002.s019.t006","gold_sense_keys":["discrimination%1:04:00::"],"candidates":[{"index":1,"sense_key":"discrimination%1:04:00::","synset_id":"discrimination.n.01"},{"index":2,"sense_key":"discrimination%1:09:00::","synset_id":"discrimination.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"discrimination%1:04:00::","call_ids":["semeval2015.d002.s019.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"discrimination%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":620,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00136,"input_uncached_usd":0.00124,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.7439894169801846}
{"item_id":"semeval2015.d002.s019.t007","gold_sense_keys":["examine%2:31:00::"],"candidates":[{"index":1,"sense_key":"examine%2:31:00::","synset_id":"analyze.v.01"},{"index":2,"sense_key":"examine%2:39:00::","synset_id":"examine.v.02"},{"index":3,"sense_key":"examine%2:32:01::","synset_id":"probe.v.01"},{"index":4,"sense_key":"examine%2:32:00::","synset_id":"examine.v.04"},{"index":5,"sense_key":"examine%2:41:00::","synset_id":"test.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"examine%2:31:00::","call_ids":["semeval2015.d002.s019.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"examine%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":827,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001774,"input_uncached_usd":0.0016539999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0434806669363752}
{"item_id":"semeval2015.d002.s020.t002","gold_sense_keys":["consider%2:39:00::"],"candidates":[{"index":1,"sense_key":"consider%2:31:00::","synset_id":"see.v.05"},{"index":2,"sense_key":"consider%2:39:00::","synset_id":"study.v.03"},{"index":3,"sense_key":"consider%2:31:01::","synset_id":"consider.v.03"},{"index":4,"sense_key":"consider%2:32:00::","synset_id":"consider.v.04"},{"index":5,"sense_key":"consider%2:32:02::","synset_id":"consider.v.05"},{"index":6,"sense_key":"consider%2:31:08::","synset_id":"think.v.01"},{"index":7,"sense_key":"consider%2:39:04::","synset_id":"regard.v.02"},{"index":8,"sense_key":"consider%2:39:08::","synset_id":"view.v.02"},{"index":9,"sense_key":"consider%2:31:05::","synset_id":"consider.v.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"consider%2:39:00::","call_ids":["semeval2015.d002.s020.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"consider%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1034,"cached_input_tokens":0,"output_tokens":110,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003168,"input_uncached_usd":0.002068,"input_cached_usd":0.0,"output_usd":0.0011,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6566751670325175}
{"item_id":"semeval2015.d002.s020.t004","gold_sense_keys":["age%1:07:00::"],"candidates":[{"index":1,"sense_key":"age%1:07:00::","synset_id":"age.n.01"},{"index":2,"sense_key":"age%1:28:02::","synset_id":"historic_period.n.01"},{"index":3,"sense_key":"age%1:28:00::","synset_id":"age.n.03"},{"index":4,"sense_key":"age%1:28:03::","synset_id":"long_time.n.01"},{"index":5,"sense_key":"age%1:28:01::","synset_id":"old_age.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"age%1:28:01::","call_ids":["semeval2015.d002.s020.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"age%1:28:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":791,"cached_input_tokens":0,"output_tokens":312,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0047020000000000005,"input_uncached_usd":0.0015819999999999999,"input_cached_usd":0.0,"output_usd":0.0031200000000000004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":16.116366458008997}
{"item_id":"semeval2015.d002.s020.t005","gold_sense_keys":["discrimination%1:04:00::"],"candidates":[{"index":1,"sense_key":"discrimination%1:04:00::","synset_id":"discrimination.n.01"},{"index":2,"sense_key":"discrimination%1:09:00::","synset_id":"discrimination.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"discrimination%1:04:00::","call_ids":["semeval2015.d002.s020.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"discrimination%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":620,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00136,"input_uncached_usd":0.00124,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5589696250390261}
{"item_id":"semeval2015.d002.s020.t007","gold_sense_keys":["small%5:00:00:limited:00"],"candidates":[{"index":1,"sense_key":"small%3:00:00::","synset_id":"small.a.01"},{"index":2,"sense_key":"small%5:00:00:limited:00","synset_id":"minor.s.03"},{"index":3,"sense_key":"small%5:00:00:young:00","synset_id":"little.s.01"},{"index":4,"sense_key":"small%5:00:00:little:03","synset_id":"small.s.03"},{"index":5,"sense_key":"small%5:00:00:inferior:01","synset_id":"humble.s.01"},{"index":6,"sense_key":"small%5:00:00:lowercase:00","synset_id":"little.s.04"},{"index":7,"sense_key":"small%5:00:00:soft:04","synset_id":"little.s.03"},{"index":8,"sense_key":"small%5:00:00:fine:00","synset_id":"small.s.07"},{"index":9,"sense_key":"small%5:00:00:moderate:00","synset_id":"modest.s.01"},{"index":10,"sense_key":"small%5:00:00:decreased:00","synset_id":"belittled.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"small%5:00:00:limited:00","call_ids":["semeval2015.d002.s020.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"small%5:00:00:limited:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1109,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002338,"input_uncached_usd":0.002218,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9324345840141177}
{"item_id":"semeval2015.d002.s020.t008","gold_sense_keys":["enterprise%1:14:00::"],"candidates":[{"index":1,"sense_key":"enterprise%1:04:00::","synset_id":"enterprise.n.01"},{"index":2,"sense_key":"enterprise%1:14:00::","synset_id":"enterprise.n.02"},{"index":3,"sense_key":"enterprise%1:07:00::","synset_id":"enterprise.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"enterprise%1:14:00::","call_ids":["semeval2015.d002.s020.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"enterprise%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":700,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00152,"input_uncached_usd":0.0014,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5300842500291765}
{"item_id":"semeval2015.d002.s021.t000","gold_sense_keys":["member%1:14:00::"],"candidates":[{"index":1,"sense_key":"member%1:18:00::","synset_id":"member.n.01"},{"index":2,"sense_key":"member%1:24:00::","synset_id":"member.n.02"},{"index":3,"sense_key":"member%1:08:00::","synset_id":"extremity.n.01"},{"index":4,"sense_key":"member%1:14:00::","synset_id":"member.n.04"},{"index":5,"sense_key":"member%1:08:01::","synset_id":"penis.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"member%1:14:00::","call_ids":["semeval2015.d002.s021.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"member%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":795,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00171,"input_uncached_usd":0.0015899999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1037212499650195}
{"item_id":"semeval2015.d002.s021.t002","gold_sense_keys":["cover%2:42:02::"],"candidates":[{"index":1,"sense_key":"cover%2:35:00::","synset_id":"cover.v.01"},{"index":2,"sense_key":"cover%2:35:01::","synset_id":"cover.v.02"},{"index":3,"sense_key":"cover%2:42:01::","synset_id":"cover.v.03"},{"index":4,"sense_key":"cover%2:42:00::","synset_id":"cover.v.04"},{"index":5,"sense_key":"cover%2:32:01::","synset_id":"cover.v.05"},{"index":6,"sense_key":"cover%2:42:02::","synset_id":"embrace.v.01"},{"index":7,"sense_key":"cover%2:38:00::","synset_id":"traverse.v.01"},{"index":8,"sense_key":"cover%2:32:02::","synset_id":"report.v.05"},{"index":9,"sense_key":"cover%2:33:01::","synset_id":"cover.v.09"},{"index":10,"sense_key":"cover%2:41:01::","synset_id":"cover.v.10"},{"index":11,"sense_key":"cover%2:39:12::","synset_id":"cover.v.11"},{"index":12,"sense_key":"cover%2:33:00::","synset_id":"cover.v.12"},{"index":13,"sense_key":"cover%2:32:12::","synset_id":"cover.v.13"},{"index":14,"sense_key":"cover%2:32:00::","synset_id":"cover.v.14"},{"index":15,"sense_key":"cover%2:42:14::","synset_id":"cover.v.15"},{"index":16,"sense_key":"cover%2:41:08::","synset_id":"cover.v.16"},{"index":17,"sense_key":"cover%2:41:10::","synset_id":"cover.v.17"},{"index":18,"sense_key":"cover%2:40:00::","synset_id":"cover.v.18"},{"index":19,"sense_key":"cover%2:39:14::","synset_id":"cover.v.19"},{"index":20,"sense_key":"cover%2:35:13::","synset_id":"shroud.v.01"},{"index":21,"sense_key":"cover%2:35:02::","synset_id":"breed.v.02"},{"index":22,"sense_key":"cover%2:35:14::","synset_id":"overlay.v.01"},{"index":23,"sense_key":"cover%2:33:07::","synset_id":"cover.v.23"},{"index":24,"sense_key":"cover%2:33:13::","synset_id":"cover.v.24"},{"index":25,"sense_key":"cover%2:29:02::","synset_id":"brood.v.05"},{"index":26,"sense_key":"cover%2:29:15::","synset_id":"cover.v.26"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"cover%2:42:02::","call_ids":["semeval2015.d002.s021.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"cover%2:42:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1978,"cached_input_tokens":0,"output_tokens":63,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004586,"input_uncached_usd":0.0039559999999999994,"input_cached_usd":0.0,"output_usd":0.00063,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4405525421025231}
{"item_id":"semeval2015.d003.s000.t000","gold_sense_keys":["document%1:10:00::"],"candidates":[{"index":1,"sense_key":"document%1:10:00::","synset_id":"document.n.01"},{"index":2,"sense_key":"document%1:06:00::","synset_id":"document.n.02"},{"index":3,"sense_key":"document%1:21:00::","synset_id":"document.n.03"},{"index":4,"sense_key":"document%1:10:01::","synset_id":"text_file.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"document%1:10:00::","call_ids":["semeval2015.d003.s000.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"document%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":494,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001108,"input_uncached_usd":0.000988,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9483526670373976}
{"item_id":"semeval2015.d003.s000.t003","gold_sense_keys":["public%3:00:00::"],"candidates":[{"index":1,"sense_key":"public%3:00:00::","synset_id":"public.a.01"},{"index":2,"sense_key":"public%5:00:00:common:02","synset_id":"public.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"public%3:00:00::","call_ids":["semeval2015.d003.s000.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"public%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":399,"cached_input_tokens":0,"output_tokens":114,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001938,"input_uncached_usd":0.000798,"input_cached_usd":0.0,"output_usd":0.0011400000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":15.114290833007544}
{"item_id":"semeval2015.d003.s000.t004","gold_sense_keys":["assessment%1:04:00::"],"candidates":[{"index":1,"sense_key":"assessment%1:09:00::","synset_id":"appraisal.n.01"},{"index":2,"sense_key":"assessment%1:21:00::","synset_id":"assessment.n.02"},{"index":3,"sense_key":"assessment%1:07:00::","synset_id":"assessment.n.03"},{"index":4,"sense_key":"assessment%1:04:00::","synset_id":"judgment.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"assessment%1:04:00::","call_ids":["semeval2015.d003.s000.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"assessment%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":506,"cached_input_tokens":0,"output_tokens":102,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002032,"input_uncached_usd":0.0010119999999999999,"input_cached_usd":0.0,"output_usd":0.00102,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.4384977919980884}
{"item_id":"semeval2015.d003.s000.t005","gold_sense_keys":["report%1:10:03::"],"candidates":[{"index":1,"sense_key":"report%1:10:03::","synset_id":"report.n.01"},{"index":2,"sense_key":"report%1:10:01::","synset_id":"report.n.02"},{"index":3,"sense_key":"report%1:10:00::","synset_id":"report.n.03"},{"index":4,"sense_key":"report%1:11:00::","synset_id":"report.n.04"},{"index":5,"sense_key":"report%1:10:04::","synset_id":"report_card.n.01"},{"index":6,"sense_key":"report%1:10:02::","synset_id":"composition.n.08"},{"index":7,"sense_key":"report%1:09:00::","synset_id":"reputation.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"report%1:10:03::","call_ids":["semeval2015.d003.s000.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"report%1:10:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":753,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001626,"input_uncached_usd":0.001506,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0960108749568462}
{"item_id":"semeval2015.d003.s001.t000","gold_sense_keys":["explain%2:32:00::","explain%2:32:01::"],"candidates":[{"index":1,"sense_key":"explain%2:32:00::","synset_id":"explain.v.01"},{"index":2,"sense_key":"explain%2:32:01::","synset_id":"explain.v.02"},{"index":3,"sense_key":"explain%2:32:02::","synset_id":"excuse.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"explain%2:32:00::","call_ids":["semeval2015.d003.s001.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"explain%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":502,"cached_input_tokens":0,"output_tokens":114,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002144,"input_uncached_usd":0.001004,"input_cached_usd":0.0,"output_usd":0.0011400000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.891073792008683}
{"item_id":"semeval2015.d003.s001.t001","gold_sense_keys":["committee%1:14:00::"],"candidates":[{"index":1,"sense_key":"committee%1:14:00::","synset_id":"committee.n.01"},{"index":2,"sense_key":"committee%1:14:01::","synset_id":"committee.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"committee%1:14:00::","call_ids":["semeval2015.d003.s001.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"committee%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":432,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.000984,"input_uncached_usd":0.000864,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.026354125002399}
{"item_id":"semeval2015.d003.s001.t003","gold_sense_keys":["product%1:06:01::"],"candidates":[{"index":1,"sense_key":"product%1:06:01::","synset_id":"merchandise.n.01"},{"index":2,"sense_key":"product%1:06:00::","synset_id":"product.n.02"},{"index":3,"sense_key":"product%1:09:00::","synset_id":"product.n.03"},{"index":4,"sense_key":"product%1:27:00::","synset_id":"product.n.04"},{"index":5,"sense_key":"product%1:19:00::","synset_id":"product.n.05"},{"index":6,"sense_key":"product%1:14:00::","synset_id":"intersection.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"product%1:06:01::","call_ids":["semeval2015.d003.s001.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"product%1:06:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":710,"cached_input_tokens":0,"output_tokens":110,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00252,"input_uncached_usd":0.00142,"input_cached_usd":0.0,"output_usd":0.0011,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1724991250084713}
{"item_id":"semeval2015.d003.s001.t005","gold_sense_keys":["use%1:04:00::"],"candidates":[{"index":1,"sense_key":"use%1:04:00::","synset_id":"use.n.01"},{"index":2,"sense_key":"use%1:07:00::","synset_id":"function.n.02"},{"index":3,"sense_key":"use%1:07:02::","synset_id":"use.n.03"},{"index":4,"sense_key":"use%1:22:00::","synset_id":"consumption.n.03"},{"index":5,"sense_key":"use%1:04:01::","synset_id":"habit.n.02"},{"index":6,"sense_key":"use%1:04:02::","synset_id":"manipulation.n.01"},{"index":7,"sense_key":"use%1:07:03::","synset_id":"use.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%1:04:00::","call_ids":["semeval2015.d003.s001.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":840,"cached_input_tokens":0,"output_tokens":149,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00317,"input_uncached_usd":0.0016799999999999999,"input_cached_usd":0.0,"output_usd":0.0014900000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.6511598330689594}
{"item_id":"semeval2015.d003.s001.t006","gold_sense_keys":["assess%2:31:00::"],"candidates":[{"index":1,"sense_key":"assess%2:31:00::","synset_id":"measure.v.04"},{"index":2,"sense_key":"assess%2:40:01::","synset_id":"assess.v.02"},{"index":3,"sense_key":"assess%2:40:00::","synset_id":"tax.v.02"},{"index":4,"sense_key":"assess%2:31:01::","synset_id":"assess.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"assess%2:31:00::","call_ids":["semeval2015.d003.s001.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"assess%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":560,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00124,"input_uncached_usd":0.00112,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1686667499598116}
{"item_id":"semeval2015.d003.s001.t007","gold_sense_keys":["study%1:04:00::"],"candidates":[{"index":1,"sense_key":"study%1:04:00::","synset_id":"survey.n.01"},{"index":2,"sense_key":"study%1:09:03::","synset_id":"study.n.02"},{"index":3,"sense_key":"study%1:10:00::","synset_id":"report.n.01"},{"index":4,"sense_key":"study%1:09:04::","synset_id":"study.n.04"},{"index":5,"sense_key":"study%1:06:00::","synset_id":"study.n.05"},{"index":6,"sense_key":"study%1:09:02::","synset_id":"discipline.n.01"},{"index":7,"sense_key":"study%1:06:01::","synset_id":"sketch.n.01"},{"index":8,"sense_key":"study%1:09:01::","synset_id":"cogitation.n.02"},{"index":9,"sense_key":"study%1:18:00::","synset_id":"study.n.09"},{"index":10,"sense_key":"study%1:10:01::","synset_id":"study.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"study%1:10:00::","call_ids":["semeval2015.d003.s001.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"study%1:10:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":900,"cached_input_tokens":0,"output_tokens":244,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004240000000000001,"input_uncached_usd":0.0018,"input_cached_usd":0.0,"output_usd":0.0024400000000000003,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":15.814431500039063}
{"item_id":"semeval2015.d003.s001.t008","gold_sense_keys":["perform%2:36:00::"],"candidates":[{"index":1,"sense_key":"perform%2:36:00::","synset_id":"perform.v.01"},{"index":2,"sense_key":"perform%2:41:02::","synset_id":"perform.v.02"},{"index":3,"sense_key":"perform%2:36:01::","synset_id":"perform.v.03"},{"index":4,"sense_key":"perform%2:41:00::","synset_id":"do.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"perform%2:36:00::","call_ids":["semeval2015.d003.s001.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"perform%2:36:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":548,"cached_input_tokens":0,"output_tokens":104,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002136,"input_uncached_usd":0.001096,"input_cached_usd":0.0,"output_usd":0.0010400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8393098330125213}
{"item_id":"semeval2015.d003.s001.t009","gold_sense_keys":["reach%2:38:01::","reach%2:38:06::","reach%2:41:00::"],"candidates":[{"index":1,"sense_key":"reach%2:38:01::","synset_id":"reach.v.01"},{"index":2,"sense_key":"reach%2:38:00::","synset_id":"reach.v.02"},{"index":3,"sense_key":"reach%2:35:00::","synset_id":"reach.v.03"},{"index":4,"sense_key":"reach%2:32:00::","synset_id":"reach.v.04"},{"index":5,"sense_key":"reach%2:41:00::","synset_id":"achieve.v.01"},{"index":6,"sense_key":"reach%2:42:00::","synset_id":"reach.v.06"},{"index":7,"sense_key":"reach%2:38:06::","synset_id":"reach.v.07"},{"index":8,"sense_key":"reach%2:40:00::","synset_id":"pass.v.05"},{"index":9,"sense_key":"reach%2:33:00::","synset_id":"strive.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"reach%2:38:01::","call_ids":["semeval2015.d003.s001.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"reach%2:38:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":930,"cached_input_tokens":0,"output_tokens":148,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00334,"input_uncached_usd":0.0018599999999999999,"input_cached_usd":0.0,"output_usd":0.0014800000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":15.144716624985449}
{"item_id":"semeval2015.d003.s001.t010","gold_sense_keys":["recommendation%1:10:00::"],"candidates":[{"index":1,"sense_key":"recommendation%1:10:00::","synset_id":"recommendation.n.01"},{"index":2,"sense_key":"recommendation%1:10:02::","synset_id":"recommendation.n.02"},{"index":3,"sense_key":"recommendation%1:07:00::","synset_id":"recommendation.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"recommendation%1:10:00::","call_ids":["semeval2015.d003.s001.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"recommendation%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":520,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00116,"input_uncached_usd":0.00104,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6663380839163437}
{"item_id":"semeval2015.d003.s001.t011","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":2,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":3,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":4,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":5,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":6,"sense_key":"use%2:41:03::","synset_id":"use.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%2:34:01::","call_ids":["semeval2015.d003.s001.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":698,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001516,"input_uncached_usd":0.0013959999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2247053750324994}
{"item_id":"semeval2015.d003.s001.t012","gold_sense_keys":["medicine%1:06:00::"],"candidates":[{"index":1,"sense_key":"medicine%1:09:00::","synset_id":"medicine.n.01"},{"index":2,"sense_key":"medicine%1:06:00::","synset_id":"medicine.n.02"},{"index":3,"sense_key":"medicine%1:04:00::","synset_id":"medicine.n.03"},{"index":4,"sense_key":"medicine%1:04:02::","synset_id":"music.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"medicine%1:06:00::","call_ids":["semeval2015.d003.s001.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"medicine%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":586,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001292,"input_uncached_usd":0.0011719999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6569132499862462}
{"item_id":"semeval2015.d003.s003.t000","gold_sense_keys":["contain%2:42:00::"],"candidates":[{"index":1,"sense_key":"contain%2:42:00::","synset_id":"incorporate.v.02"},{"index":2,"sense_key":"contain%2:42:13::","synset_id":"hold.v.11"},{"index":3,"sense_key":"contain%2:41:00::","synset_id":"control.v.02"},{"index":4,"sense_key":"contain%2:42:06::","synset_id":"contain.v.04"},{"index":5,"sense_key":"contain%2:42:14::","synset_id":"contain.v.05"},{"index":6,"sense_key":"contain%2:33:00::","synset_id":"check.v.18"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"contain%2:42:00::","call_ids":["semeval2015.d003.s003.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"contain%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":803,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001726,"input_uncached_usd":0.001606,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.714618582976982}
{"item_id":"semeval2015.d003.s003.t001","gold_sense_keys":["active%3:00:07::"],"candidates":[{"index":1,"sense_key":"active%3:00:02::","synset_id":"active.a.01"},{"index":2,"sense_key":"active%5:00:00:operational:00","synset_id":"active.s.01"},{"index":3,"sense_key":"active%3:00:03::","synset_id":"active.a.03"},{"index":4,"sense_key":"active%5:00:00:involved:00","synset_id":"active.s.02"},{"index":5,"sense_key":"active%3:00:01::","synset_id":"active.a.05"},{"index":6,"sense_key":"active%3:00:07::","synset_id":"active.a.06"},{"index":7,"sense_key":"active%3:00:06::","synset_id":"active.a.07"},{"index":8,"sense_key":"active%5:00:00:existent:00","synset_id":"active.s.03"},{"index":9,"sense_key":"active%3:00:10::","synset_id":"active.a.09"},{"index":10,"sense_key":"active%3:00:09::","synset_id":"active.a.10"},{"index":11,"sense_key":"active%3:00:00::","synset_id":"active.a.11"},{"index":12,"sense_key":"active%3:00:05::","synset_id":"active.a.12"},{"index":13,"sense_key":"active%3:00:04::","synset_id":"active.a.13"},{"index":14,"sense_key":"active%3:00:08::","synset_id":"active.a.14"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"active%3:00:07::","call_ids":["semeval2015.d003.s003.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"active%3:00:07::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1125,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0023699999999999997,"input_uncached_usd":0.00225,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1562115419656038}
{"item_id":"semeval2015.d003.s003.t002","gold_sense_keys":["substance%1:03:01::"],"candidates":[{"index":1,"sense_key":"substance%1:03:00::","synset_id":"substance.n.01"},{"index":2,"sense_key":"substance%1:09:01::","synset_id":"kernel.n.03"},{"index":3,"sense_key":"substance%1:09:00::","synset_id":"meaning.n.02"},{"index":4,"sense_key":"substance%1:27:00::","synset_id":"substance.n.04"},{"index":5,"sense_key":"substance%1:21:00::","synset_id":"means.n.03"},{"index":6,"sense_key":"substance%1:10:00::","synset_id":"message.n.02"},{"index":7,"sense_key":"substance%1:03:01::","synset_id":"substance.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"substance%1:03:01::","call_ids":["semeval2015.d003.s003.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"substance%1:03:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":775,"cached_input_tokens":0,"output_tokens":118,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00273,"input_uncached_usd":0.00155,"input_cached_usd":0.0,"output_usd":0.00118,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8349177499767393}
{"item_id":"semeval2015.d003.s003.t003","gold_sense_keys":["available%3:00:00::"],"candidates":[{"index":1,"sense_key":"available%3:00:00::","synset_id":"available.a.01"},{"index":2,"sense_key":"available%5:00:00:free:00","synset_id":"available.s.01"},{"index":3,"sense_key":"available%5:00:00:disposable:02","synset_id":"available.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"available%3:00:00::","call_ids":["semeval2015.d003.s003.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"available%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":567,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0012540000000000001,"input_uncached_usd":0.001134,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5702600840013474}
{"item_id":"semeval2015.d003.s003.t004","gold_sense_keys":["tablet%1:06:00::"],"candidates":[{"index":1,"sense_key":"tablet%1:06:02::","synset_id":"tablet.n.01"},{"index":2,"sense_key":"tablet%1:27:00::","synset_id":"pad.n.01"},{"index":3,"sense_key":"tablet%1:06:01::","synset_id":"tablet.n.03"},{"index":4,"sense_key":"tablet%1:06:00::","synset_id":"pill.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"tablet%1:06:00::","call_ids":["semeval2015.d003.s003.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"tablet%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":571,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001262,"input_uncached_usd":0.001142,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6157427079742774}
{"item_id":"semeval2015.d003.s003.t005","gold_sense_keys":["solution%1:27:00::"],"candidates":[{"index":1,"sense_key":"solution%1:27:00::","synset_id":"solution.n.01"},{"index":2,"sense_key":"solution%1:10:00::","synset_id":"solution.n.02"},{"index":3,"sense_key":"solution%1:09:00::","synset_id":"solution.n.03"},{"index":4,"sense_key":"solution%1:14:00::","synset_id":"solution.n.04"},{"index":5,"sense_key":"solution%1:04:00::","synset_id":"solution.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"solution%1:27:00::","call_ids":["semeval2015.d003.s003.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"solution%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":667,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001454,"input_uncached_usd":0.001334,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":9.240028291940689}
{"item_id":"semeval2015.d003.s003.t006","gold_sense_keys":["injection%1:04:01::"],"candidates":[{"index":1,"sense_key":"injection%1:04:00::","synset_id":"injection.n.01"},{"index":2,"sense_key":"injection%1:27:00::","synset_id":"injection.n.02"},{"index":3,"sense_key":"injection%1:04:01::","synset_id":"injection.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"injection%1:27:00::","call_ids":["semeval2015.d003.s003.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"injection%1:27:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":517,"cached_input_tokens":0,"output_tokens":50,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001534,"input_uncached_usd":0.001034,"input_cached_usd":0.0,"output_usd":0.0005,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7364234579727054}
{"item_id":"semeval2015.d003.s004.t000","gold_sense_keys":["tablet%1:06:00::"],"candidates":[{"index":1,"sense_key":"tablet%1:06:02::","synset_id":"tablet.n.01"},{"index":2,"sense_key":"tablet%1:27:00::","synset_id":"pad.n.01"},{"index":3,"sense_key":"tablet%1:06:01::","synset_id":"tablet.n.03"},{"index":4,"sense_key":"tablet%1:06:00::","synset_id":"pill.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"tablet%1:06:00::","call_ids":["semeval2015.d003.s004.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"tablet%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":614,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001348,"input_uncached_usd":0.001228,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1239328340161592}
{"item_id":"semeval2015.d003.s004.t001","gold_sense_keys":["have%2:40:00::","have%2:42:00::"],"candidates":[{"index":1,"sense_key":"have%2:40:00::","synset_id":"have.v.01"},{"index":2,"sense_key":"have%2:42:00::","synset_id":"have.v.02"},{"index":3,"sense_key":"have%2:39:06::","synset_id":"experience.v.03"},{"index":4,"sense_key":"have%2:40:04::","synset_id":"own.v.01"},{"index":5,"sense_key":"have%2:30:00::","synset_id":"get.v.03"},{"index":6,"sense_key":"have%2:34:00::","synset_id":"consume.v.02"},{"index":7,"sense_key":"have%2:41:00::","synset_id":"have.v.07"},{"index":8,"sense_key":"have%2:36:00::","synset_id":"hold.v.03"},{"index":9,"sense_key":"have%2:40:01::","synset_id":"have.v.09"},{"index":10,"sense_key":"have%2:42:12::","synset_id":"have.v.10"},{"index":11,"sense_key":"have%2:30:01::","synset_id":"have.v.11"},{"index":12,"sense_key":"have%2:29:02::","synset_id":"have.v.12"},{"index":13,"sense_key":"have%2:32:00::","synset_id":"induce.v.02"},{"index":14,"sense_key":"have%2:40:05::","synset_id":"accept.v.02"},{"index":15,"sense_key":"have%2:40:03::","synset_id":"receive.v.01"},{"index":16,"sense_key":"have%2:29:05::","synset_id":"suffer.v.02"},{"index":17,"sense_key":"have%2:40:02::","synset_id":"have.v.17"},{"index":18,"sense_key":"have%2:29:00::","synset_id":"give_birth.v.01"},{"index":19,"sense_key":"have%2:35:00::","synset_id":"take.v.35"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"have%2:42:00::","call_ids":["semeval2015.d003.s004.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"have%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1580,"cached_input_tokens":0,"output_tokens":89,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00405,"input_uncached_usd":0.00316,"input_cached_usd":0.0,"output_usd":0.0008900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5978370000375435}
{"item_id":"semeval2015.d003.s004.t002","gold_sense_keys":["score%1:25:00::"],"candidates":[{"index":1,"sense_key":"score%1:09:00::","synset_id":"mark.n.01"},{"index":2,"sense_key":"score%1:10:00::","synset_id":"score.n.02"},{"index":3,"sense_key":"score%1:23:00::","synset_id":"score.n.03"},{"index":4,"sense_key":"score%1:14:00::","synset_id":"score.n.04"},{"index":5,"sense_key":"score%1:16:00::","synset_id":"score.n.05"},{"index":6,"sense_key":"score%1:09:01::","synset_id":"score.n.06"},{"index":7,"sense_key":"score%1:26:00::","synset_id":"score.n.07"},{"index":8,"sense_key":"score%1:25:00::","synset_id":"score.n.08"},{"index":9,"sense_key":"score%1:12:00::","synset_id":"grudge.n.01"},{"index":10,"sense_key":"score%1:04:00::","synset_id":"score.n.10"},{"index":11,"sense_key":"score%1:04:01::","synset_id":"sexual_conquest.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"score%1:25:00::","call_ids":["semeval2015.d003.s004.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"score%1:25:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1048,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0022159999999999997,"input_uncached_usd":0.0020959999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.201182582997717}
{"item_id":"semeval2015.d003.s004.t003","gold_sense_keys":["line%1:25:01::"],"candidates":[{"index":1,"sense_key":"line%1:14:03::","synset_id":"line.n.01"},{"index":2,"sense_key":"line%1:10:01::","synset_id":"line.n.02"},{"index":3,"sense_key":"line%1:14:01::","synset_id":"line.n.03"},{"index":4,"sense_key":"line%1:25:00::","synset_id":"line.n.04"},{"index":5,"sense_key":"line%1:10:02::","synset_id":"line.n.05"},{"index":6,"sense_key":"line%1:19:00::","synset_id":"line.n.06"},{"index":7,"sense_key":"line%1:15:02::","synset_id":"line.n.07"},{"index":8,"sense_key":"line%1:09:01::","synset_id":"argumentation.n.02"},{"index":9,"sense_key":"line%1:06:01::","synset_id":"cable.n.02"},{"index":10,"sense_key":"line%1:14:02::","synset_id":"course.n.02"},{"index":11,"sense_key":"line%1:15:00::","synset_id":"line.n.11"},{"index":12,"sense_key":"line%1:25:01::","synset_id":"wrinkle.n.01"},{"index":13,"sense_key":"line%1:06:06::","synset_id":"pipeline.n.02"},{"index":14,"sense_key":"line%1:06:03::","synset_id":"line.n.14"},{"index":15,"sense_key":"line%1:06:07::","synset_id":"telephone_line.n.02"},{"index":16,"sense_key":"line%1:04:01::","synset_id":"line.n.16"},{"index":17,"sense_key":"line%1:14:00::","synset_id":"lineage.n.01"},{"index":18,"sense_key":"line%1:06:00::","synset_id":"line.n.18"},{"index":19,"sense_key":"line%1:04:00::","synset_id":"occupation.n.01"},{"index":20,"sense_key":"line%1:15:01::","synset_id":"line.n.20"},{"index":21,"sense_key":"line%1:10:05::","synset_id":"channel.n.05"},{"index":22,"sense_key":"line%1:06:08::","synset_id":"line.n.22"},{"index":23,"sense_key":"line%1:06:05::","synset_id":"line.n.23"},{"index":24,"sense_key":"line%1:23:00::","synset_id":"agate_line.n.01"},{"index":25,"sense_key":"line%1:21:00::","synset_id":"credit_line.n.01"},{"index":26,"sense_key":"line%1:10:03::","synset_id":"tune.n.01"},{"index":27,"sense_key":"line%1:10:06::","synset_id":"line.n.27"},{"index":28,"sense_key":"line%1:10:04::","synset_id":"note.n.02"},{"index":29,"sense_key":"line%1:09:00::","synset_id":"line.n.29"},{"index":30,"sense_key":"line%1:06:09::","synset_id":"production_line.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"line%1:10:01::","call_ids":["semeval2015.d003.s004.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"line%1:10:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":2245,"cached_input_tokens":0,"output_tokens":110,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00559,"input_uncached_usd":0.00449,"input_cached_usd":0.0,"output_usd":0.0011,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.080571416998282}
{"item_id":"semeval2015.d003.s004.t004","gold_sense_keys":["side%1:06:00::","side%1:15:00::"],"candidates":[{"index":1,"sense_key":"side%1:15:02::","synset_id":"side.n.01"},{"index":2,"sense_key":"side%1:14:00::","synset_id":"side.n.02"},{"index":3,"sense_key":"side%1:08:00::","synset_id":"side.n.03"},{"index":4,"sense_key":"side%1:15:00::","synset_id":"side.n.04"},{"index":5,"sense_key":"side%1:06:00::","synset_id":"side.n.05"},{"index":6,"sense_key":"side%1:09:00::","synset_id":"side.n.06"},{"index":7,"sense_key":"side%1:15:01::","synset_id":"side.n.07"},{"index":8,"sense_key":"side%1:14:01::","synset_id":"side.n.08"},{"index":9,"sense_key":"side%1:13:00::","synset_id":"side.n.09"},{"index":10,"sense_key":"side%1:10:00::","synset_id":"side.n.10"},{"index":11,"sense_key":"side%1:17:01::","synset_id":"slope.n.01"},{"index":12,"sense_key":"side%1:11:00::","synset_id":"english.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"side%1:15:00::","call_ids":["semeval2015.d003.s004.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"side%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1100,"cached_input_tokens":0,"output_tokens":132,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0035199999999999997,"input_uncached_usd":0.0021999999999999997,"input_cached_usd":0.0,"output_usd":0.0013200000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9722675830125809}
{"item_id":"semeval2015.d003.s004.t005","gold_sense_keys":["so%4:02:06::"],"candidates":[{"index":1,"sense_key":"so%4:02:02::","synset_id":"so.r.01"},{"index":2,"sense_key":"so%4:02:06::","synset_id":"so.r.02"},{"index":3,"sense_key":"so%4:02:05::","synset_id":"so.r.03"},{"index":4,"sense_key":"so%4:02:07::","synset_id":"so.r.04"},{"index":5,"sense_key":"so%4:02:08::","synset_id":"so.r.05"},{"index":6,"sense_key":"so%4:02:04::","synset_id":"thus.r.02"},{"index":7,"sense_key":"so%4:02:03::","synset_id":"so.r.07"},{"index":8,"sense_key":"so%4:02:09::","synset_id":"then.r.01"},{"index":9,"sense_key":"so%4:02:10::","synset_id":"therefore.r.01"},{"index":10,"sense_key":"so%4:02:00::","synset_id":"indeed.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"so%4:02:06::","call_ids":["semeval2015.d003.s004.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"so%4:02:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1073,"cached_input_tokens":0,"output_tokens":77,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002916,"input_uncached_usd":0.002146,"input_cached_usd":0.0,"output_usd":0.0007700000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6311424999730662}
{"item_id":"semeval2015.d003.s005.t000","gold_sense_keys":["tablet%1:06:00::"],"candidates":[{"index":1,"sense_key":"tablet%1:06:02::","synset_id":"tablet.n.01"},{"index":2,"sense_key":"tablet%1:27:00::","synset_id":"pad.n.01"},{"index":3,"sense_key":"tablet%1:06:01::","synset_id":"tablet.n.03"},{"index":4,"sense_key":"tablet%1:06:00::","synset_id":"pill.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"tablet%1:06:00::","call_ids":["semeval2015.d003.s005.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"tablet%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":624,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001368,"input_uncached_usd":0.001248,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.07173166598659}
{"item_id":"semeval2015.d003.s005.t001","gold_sense_keys":["marked%3:00:00::"],"candidates":[{"index":1,"sense_key":"marked%5:00:00:noticeable:00","synset_id":"marked.s.01"},{"index":2,"sense_key":"marked%5:00:00:conspicuous:00","synset_id":"marked.s.02"},{"index":3,"sense_key":"marked%3:00:00::","synset_id":"marked.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"marked%3:00:00::","call_ids":["semeval2015.d003.s005.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"marked%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":616,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001842,"input_uncached_usd":0.001232,"input_cached_usd":0.0,"output_usd":0.0006100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":5.255884957965463}
{"item_id":"semeval2015.d003.s005.t003","gold_sense_keys":["side%1:06:00::","side%1:15:00::"],"candidates":[{"index":1,"sense_key":"side%1:15:02::","synset_id":"side.n.01"},{"index":2,"sense_key":"side%1:14:00::","synset_id":"side.n.02"},{"index":3,"sense_key":"side%1:08:00::","synset_id":"side.n.03"},{"index":4,"sense_key":"side%1:15:00::","synset_id":"side.n.04"},{"index":5,"sense_key":"side%1:06:00::","synset_id":"side.n.05"},{"index":6,"sense_key":"side%1:09:00::","synset_id":"side.n.06"},{"index":7,"sense_key":"side%1:15:01::","synset_id":"side.n.07"},{"index":8,"sense_key":"side%1:14:01::","synset_id":"side.n.08"},{"index":9,"sense_key":"side%1:13:00::","synset_id":"side.n.09"},{"index":10,"sense_key":"side%1:10:00::","synset_id":"side.n.10"},{"index":11,"sense_key":"side%1:17:01::","synset_id":"slope.n.01"},{"index":12,"sense_key":"side%1:11:00::","synset_id":"english.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"side%1:06:00::","call_ids":["semeval2015.d003.s005.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"side%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1109,"cached_input_tokens":0,"output_tokens":220,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004418,"input_uncached_usd":0.002218,"input_cached_usd":0.0,"output_usd":0.0022,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":4.14966941694729}
{"item_id":"semeval2015.d003.s005.t004","gold_sense_keys":["letter%1:10:01::"],"candidates":[{"index":1,"sense_key":"letter%1:10:00::","synset_id":"letter.n.01"},{"index":2,"sense_key":"letter%1:10:01::","synset_id":"letter.n.02"},{"index":3,"sense_key":"letter%1:18:00::","synset_id":"letter.n.03"},{"index":4,"sense_key":"letter%1:10:03::","synset_id":"letter.n.04"},{"index":5,"sense_key":"letter%1:10:02::","synset_id":"letter.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"letter%1:10:01::","call_ids":["semeval2015.d003.s005.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"letter%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":723,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0015660000000000001,"input_uncached_usd":0.001446,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9782496669795364}
{"item_id":"semeval2015.d003.s005.t005","gold_sense_keys":["quantity%1:03:00::"],"candidates":[{"index":1,"sense_key":"quantity%1:03:00::","synset_id":"measure.n.02"},{"index":2,"sense_key":"quantity%1:07:00::","synset_id":"quantity.n.02"},{"index":3,"sense_key":"quantity%1:09:01::","synset_id":"quantity.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"quantity%1:03:00::","call_ids":["semeval2015.d003.s005.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"quantity%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":577,"cached_input_tokens":0,"output_tokens":95,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002104,"input_uncached_usd":0.0011539999999999999,"input_cached_usd":0.0,"output_usd":0.0009500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7051225419854745}
{"item_id":"semeval2015.d003.s005.t006","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":2,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":3,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"other%3:00:00::","call_ids":["semeval2015.d003.s005.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":656,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0014320000000000001,"input_uncached_usd":0.001312,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":4.722439666977152}
{"item_id":"semeval2015.d003.s005.t007","gold_sense_keys":["side%1:06:00::","side%1:15:00::"],"candidates":[{"index":1,"sense_key":"side%1:15:02::","synset_id":"side.n.01"},{"index":2,"sense_key":"side%1:14:00::","synset_id":"side.n.02"},{"index":3,"sense_key":"side%1:08:00::","synset_id":"side.n.03"},{"index":4,"sense_key":"side%1:15:00::","synset_id":"side.n.04"},{"index":5,"sense_key":"side%1:06:00::","synset_id":"side.n.05"},{"index":6,"sense_key":"side%1:09:00::","synset_id":"side.n.06"},{"index":7,"sense_key":"side%1:15:01::","synset_id":"side.n.07"},{"index":8,"sense_key":"side%1:14:01::","synset_id":"side.n.08"},{"index":9,"sense_key":"side%1:13:00::","synset_id":"side.n.09"},{"index":10,"sense_key":"side%1:10:00::","synset_id":"side.n.10"},{"index":11,"sense_key":"side%1:17:01::","synset_id":"slope.n.01"},{"index":12,"sense_key":"side%1:11:00::","synset_id":"english.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"side%1:06:00::","call_ids":["semeval2015.d003.s005.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"side%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1109,"cached_input_tokens":0,"output_tokens":135,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003568,"input_uncached_usd":0.002218,"input_cached_usd":0.0,"output_usd":0.00135,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.124618708039634}
{"item_id":"semeval2015.d003.s006.t000","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":2,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":3,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":4,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":5,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":6,"sense_key":"use%2:41:03::","synset_id":"use.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%2:34:01::","call_ids":["semeval2015.d003.s006.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":815,"cached_input_tokens":0,"output_tokens":60,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00223,"input_uncached_usd":0.00163,"input_cached_usd":0.0,"output_usd":0.0006000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":4.862467083032243}
{"item_id":"semeval2015.d003.s007.t001","gold_sense_keys":["mean%2:32:03::","mean%2:42:00::"],"candidates":[{"index":1,"sense_key":"mean%2:32:01::","synset_id":"mean.v.01"},{"index":2,"sense_key":"mean%2:42:00::","synset_id":"entail.v.01"},{"index":3,"sense_key":"mean%2:32:03::","synset_id":"mean.v.03"},{"index":4,"sense_key":"mean%2:31:00::","synset_id":"intend.v.01"},{"index":5,"sense_key":"mean%2:42:03::","synset_id":"mean.v.05"},{"index":6,"sense_key":"mean%2:31:01::","synset_id":"think_of.v.04"},{"index":7,"sense_key":"mean%2:31:05::","synset_id":"mean.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"mean%2:42:00::","call_ids":["semeval2015.d003.s007.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"mean%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":863,"cached_input_tokens":0,"output_tokens":209,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003816,"input_uncached_usd":0.0017259999999999999,"input_cached_usd":0.0,"output_usd":0.0020900000000000003,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.997248665895313}
{"item_id":"semeval2015.d003.s007.t002","gold_sense_keys":["stop%2:41:00::"],"candidates":[{"index":1,"sense_key":"stop%2:38:00::","synset_id":"stop.v.01"},{"index":2,"sense_key":"stop%2:42:00::","synset_id":"discontinue.v.01"},{"index":3,"sense_key":"stop%2:41:00::","synset_id":"stop.v.03"},{"index":4,"sense_key":"stop%2:38:02::","synset_id":"stop.v.04"},{"index":5,"sense_key":"stop%2:38:01::","synset_id":"stop.v.05"},{"index":6,"sense_key":"stop%2:30:05::","synset_id":"break.v.10"},{"index":7,"sense_key":"stop%2:33:00::","synset_id":"check.v.18"},{"index":8,"sense_key":"stop%2:35:00::","synset_id":"intercept.v.01"},{"index":9,"sense_key":"stop%2:42:13::","synset_id":"end.v.01"},{"index":10,"sense_key":"stop%2:35:12::","synset_id":"barricade.v.01"},{"index":11,"sense_key":"stop%2:30:12::","synset_id":"hold_on.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"stop%2:41:00::","call_ids":["semeval2015.d003.s007.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"stop%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1193,"cached_input_tokens":0,"output_tokens":311,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.005496000000000001,"input_uncached_usd":0.002386,"input_cached_usd":0.0,"output_usd":0.0031100000000000004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.0275405830470845}
{"item_id":"semeval2015.d003.s008.t000","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":2,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":3,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":4,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":5,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":6,"sense_key":"use%2:41:03::","synset_id":"use.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%2:34:01::","call_ids":["semeval2015.d003.s008.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":869,"cached_input_tokens":0,"output_tokens":78,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0025180000000000003,"input_uncached_usd":0.001738,"input_cached_usd":0.0,"output_usd":0.0007800000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6962584579596296}
{"item_id":"semeval2015.d003.s008.t001","gold_sense_keys":["dog%1:05:00::"],"candidates":[{"index":1,"sense_key":"dog%1:05:00::","synset_id":"dog.n.01"},{"index":2,"sense_key":"dog%1:18:01::","synset_id":"frump.n.01"},{"index":3,"sense_key":"dog%1:18:00::","synset_id":"dog.n.03"},{"index":4,"sense_key":"dog%1:18:02::","synset_id":"cad.n.01"},{"index":5,"sense_key":"dog%1:13:01::","synset_id":"frank.n.02"},{"index":6,"sense_key":"dog%1:06:00::","synset_id":"pawl.n.01"},{"index":7,"sense_key":"dog%1:06:01::","synset_id":"andiron.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"dog%1:05:00::","call_ids":["semeval2015.d003.s008.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"dog%1:05:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":985,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00209,"input_uncached_usd":0.00197,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.1174601670354605}
{"item_id":"semeval2015.d003.s008.t002","gold_sense_keys":["prevent%2:41:01::"],"candidates":[{"index":1,"sense_key":"prevent%2:41:01::","synset_id":"prevent.v.01"},{"index":2,"sense_key":"prevent%2:41:00::","synset_id":"prevent.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"prevent%2:41:01::","call_ids":["semeval2015.d003.s008.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"prevent%2:41:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":664,"cached_input_tokens":0,"output_tokens":126,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002588,"input_uncached_usd":0.001328,"input_cached_usd":0.0,"output_usd":0.00126,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.115125500014983}
{"item_id":"semeval2015.d003.s008.t005","gold_sense_keys":["cause%2:36:00::"],"candidates":[{"index":1,"sense_key":"cause%2:36:00::","synset_id":"cause.v.01"},{"index":2,"sense_key":"cause%2:32:00::","synset_id":"induce.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cause%2:36:00::","call_ids":["semeval2015.d003.s008.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cause%2:36:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":636,"cached_input_tokens":0,"output_tokens":95,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002222,"input_uncached_usd":0.001272,"input_cached_usd":0.0,"output_usd":0.0009500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":14.82128658297006}
{"item_id":"semeval2015.d003.s008.t007","gold_sense_keys":["medicine%1:06:00::"],"candidates":[{"index":1,"sense_key":"medicine%1:09:00::","synset_id":"medicine.n.01"},{"index":2,"sense_key":"medicine%1:06:00::","synset_id":"medicine.n.02"},{"index":3,"sense_key":"medicine%1:04:00::","synset_id":"medicine.n.03"},{"index":4,"sense_key":"medicine%1:04:02::","synset_id":"music.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"medicine%1:06:00::","call_ids":["semeval2015.d003.s008.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"medicine%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":757,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001634,"input_uncached_usd":0.001514,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.067705124965869}
{"item_id":"semeval2015.d003.s008.t008","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":2,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":3,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":4,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":5,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":6,"sense_key":"use%2:41:03::","synset_id":"use.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%2:34:01::","call_ids":["semeval2015.d003.s008.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":869,"cached_input_tokens":0,"output_tokens":102,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002758,"input_uncached_usd":0.001738,"input_cached_usd":0.0,"output_usd":0.00102,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.3553543750895187}
{"item_id":"semeval2015.d003.s008.t009","gold_sense_keys":["treatment%1:04:00::"],"candidates":[{"index":1,"sense_key":"treatment%1:04:00::","synset_id":"treatment.n.01"},{"index":2,"sense_key":"treatment%1:04:01::","synset_id":"treatment.n.02"},{"index":3,"sense_key":"treatment%1:07:00::","synset_id":"treatment.n.03"},{"index":4,"sense_key":"treatment%1:10:00::","synset_id":"discussion.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"treatment%1:04:00::","call_ids":["semeval2015.d003.s008.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"treatment%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":757,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001634,"input_uncached_usd":0.001514,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.544622707995586}
{"item_id":"semeval2015.d003.s008.t010","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["semeval2015.d003.s008.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":801,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001722,"input_uncached_usd":0.001602,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2399469170486555}
{"item_id":"semeval2015.d003.s008.t012","gold_sense_keys":["cause%2:36:00::"],"candidates":[{"index":1,"sense_key":"cause%2:36:00::","synset_id":"cause.v.01"},{"index":2,"sense_key":"cause%2:32:00::","synset_id":"induce.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cause%2:36:00::","call_ids":["semeval2015.d003.s008.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cause%2:36:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":636,"cached_input_tokens":0,"output_tokens":72,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001992,"input_uncached_usd":0.001272,"input_cached_usd":0.0,"output_usd":0.00072,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6849147499306127}
{"item_id":"semeval2015.d003.s009.t002","gold_sense_keys":["measure%1:04:01::"],"candidates":[{"index":1,"sense_key":"measure%1:04:01::","synset_id":"measure.n.01"},{"index":2,"sense_key":"measure%1:03:00::","synset_id":"measure.n.02"},{"index":3,"sense_key":"measure%1:10:02::","synset_id":"bill.n.01"},{"index":4,"sense_key":"measure%1:04:00::","synset_id":"measurement.n.01"},{"index":5,"sense_key":"measure%1:10:03::","synset_id":"standard.n.01"},{"index":6,"sense_key":"measure%1:10:01::","synset_id":"meter.n.03"},{"index":7,"sense_key":"measure%1:10:00::","synset_id":"measure.n.07"},{"index":8,"sense_key":"measure%1:06:00::","synset_id":"measuring_stick.n.01"},{"index":9,"sense_key":"measure%1:06:01::","synset_id":"measure.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"measure%1:04:01::","call_ids":["semeval2015.d003.s009.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"measure%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1074,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0022679999999999996,"input_uncached_usd":0.0021479999999999997,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9027340420288965}
{"item_id":"semeval2015.d003.s009.t003","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":2,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":3,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"other%3:00:00::","call_ids":["semeval2015.d003.s009.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":706,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001532,"input_uncached_usd":0.0014119999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0177430839976296}
{"item_id":"semeval2015.d003.s009.t005","gold_sense_keys":["treatment%1:04:00::"],"candidates":[{"index":1,"sense_key":"treatment%1:04:00::","synset_id":"treatment.n.01"},{"index":2,"sense_key":"treatment%1:04:01::","synset_id":"treatment.n.02"},{"index":3,"sense_key":"treatment%1:07:00::","synset_id":"treatment.n.03"},{"index":4,"sense_key":"treatment%1:10:00::","synset_id":"discussion.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"treatment%1:04:00::","call_ids":["semeval2015.d003.s009.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"treatment%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":746,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001612,"input_uncached_usd":0.0014919999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.525848750025034}
{"item_id":"semeval2015.d003.s009.t006","gold_sense_keys":["special%5:00:00:specialized:00","special%5:00:01:specific:00"],"candidates":[{"index":1,"sense_key":"special%5:00:01:specific:00","synset_id":"particular.s.01"},{"index":2,"sense_key":"special%5:00:00:extraordinary:00","synset_id":"special.s.02"},{"index":3,"sense_key":"special%5:00:00:uncommon:00","synset_id":"especial.s.01"},{"index":4,"sense_key":"special%5:00:00:specialized:00","synset_id":"special.s.04"},{"index":5,"sense_key":"special%5:00:00:specific:00","synset_id":"limited.s.05"},{"index":6,"sense_key":"special%5:00:00:primary:00","synset_id":"particular.s.04"},{"index":7,"sense_key":"special%5:00:00:unscheduled:00","synset_id":"extra.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"special%5:00:00:specialized:00","call_ids":["semeval2015.d003.s009.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"special%5:00:00:specialized:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":933,"cached_input_tokens":0,"output_tokens":85,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0027159999999999997,"input_uncached_usd":0.0018659999999999998,"input_cached_usd":0.0,"output_usd":0.0008500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0311680829618126}
{"item_id":"semeval2015.d003.s009.t007","gold_sense_keys":["diet%1:13:00::"],"candidates":[{"index":1,"sense_key":"diet%1:13:00::","synset_id":"diet.n.01"},{"index":2,"sense_key":"diet%1:14:00::","synset_id":"diet.n.02"},{"index":3,"sense_key":"diet%1:13:01::","synset_id":"diet.n.03"},{"index":4,"sense_key":"diet%1:04:00::","synset_id":"diet.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"diet%1:13:00::","call_ids":["semeval2015.d003.s009.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"diet%1:13:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":642,"cached_input_tokens":0,"output_tokens":48,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001764,"input_uncached_usd":0.001284,"input_cached_usd":0.0,"output_usd":0.00048000000000000007,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":11.537316625006497}
{"item_id":"semeval2015.d003.s009.t009","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":2,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":3,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":4,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":5,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":6,"sense_key":"use%2:41:03::","synset_id":"use.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%2:34:01::","call_ids":["semeval2015.d003.s009.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":858,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001836,"input_uncached_usd":0.0017159999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.6627053330885246}
{"item_id":"semeval2015.d003.s009.t010","gold_sense_keys":["treatment%1:04:00::"],"candidates":[{"index":1,"sense_key":"treatment%1:04:00::","synset_id":"treatment.n.01"},{"index":2,"sense_key":"treatment%1:04:01::","synset_id":"treatment.n.02"},{"index":3,"sense_key":"treatment%1:07:00::","synset_id":"treatment.n.03"},{"index":4,"sense_key":"treatment%1:10:00::","synset_id":"discussion.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"treatment%1:04:00::","call_ids":["semeval2015.d003.s009.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"treatment%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":746,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001612,"input_uncached_usd":0.0014919999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5882115419954062}
{"item_id":"semeval2015.d003.s009.t012","gold_sense_keys":["injection%1:04:01::"],"candidates":[{"index":1,"sense_key":"injection%1:04:00::","synset_id":"injection.n.01"},{"index":2,"sense_key":"injection%1:27:00::","synset_id":"injection.n.02"},{"index":3,"sense_key":"injection%1:04:01::","synset_id":"injection.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"injection%1:04:01::","call_ids":["semeval2015.d003.s009.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"injection%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":620,"cached_input_tokens":0,"output_tokens":119,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00243,"input_uncached_usd":0.00124,"input_cached_usd":0.0,"output_usd":0.00119,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9681998339947313}
{"item_id":"semeval2015.d003.s009.t013","gold_sense_keys":["injection%1:04:01::"],"candidates":[{"index":1,"sense_key":"injection%1:04:00::","synset_id":"injection.n.01"},{"index":2,"sense_key":"injection%1:27:00::","synset_id":"injection.n.02"},{"index":3,"sense_key":"injection%1:04:01::","synset_id":"injection.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"injection%1:04:01::","call_ids":["semeval2015.d003.s009.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"injection%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":620,"cached_input_tokens":0,"output_tokens":66,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0019000000000000002,"input_uncached_usd":0.00124,"input_cached_usd":0.0,"output_usd":0.0006600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3524952080333605}
{"item_id":"semeval2015.d003.s009.t015","gold_sense_keys":["tablet%1:06:00::"],"candidates":[{"index":1,"sense_key":"tablet%1:06:02::","synset_id":"tablet.n.01"},{"index":2,"sense_key":"tablet%1:27:00::","synset_id":"pad.n.01"},{"index":3,"sense_key":"tablet%1:06:01::","synset_id":"tablet.n.03"},{"index":4,"sense_key":"tablet%1:06:00::","synset_id":"pill.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"tablet%1:06:00::","call_ids":["semeval2015.d003.s009.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"tablet%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":674,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0014680000000000001,"input_uncached_usd":0.001348,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9153724999632686}
{"item_id":"semeval2015.d003.s010.t000","gold_sense_keys":["purpose%1:07:00::"],"candidates":[{"index":1,"sense_key":"purpose%1:09:00::","synset_id":"purpose.n.01"},{"index":2,"sense_key":"purpose%1:07:00::","synset_id":"function.n.02"},{"index":3,"sense_key":"purpose%1:07:01::","synset_id":"determination.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"purpose%1:07:00::","call_ids":["semeval2015.d003.s010.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"purpose%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":687,"cached_input_tokens":0,"output_tokens":59,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001964,"input_uncached_usd":0.001374,"input_cached_usd":0.0,"output_usd":0.00059,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3785510420566425}
{"item_id":"semeval2015.d003.s010.t001","gold_sense_keys":["give%2:40:13::"],"candidates":[{"index":1,"sense_key":"give%2:40:03::","synset_id":"give.v.01"},{"index":2,"sense_key":"give%2:40:05::","synset_id":"yield.v.01"},{"index":3,"sense_key":"give%2:40:00::","synset_id":"give.v.03"},{"index":4,"sense_key":"give%2:40:07::","synset_id":"give.v.04"},{"index":5,"sense_key":"give%2:32:01::","synset_id":"give.v.05"},{"index":6,"sense_key":"give%2:36:02::","synset_id":"hold.v.03"},{"index":7,"sense_key":"give%2:32:00::","synset_id":"give.v.07"},{"index":8,"sense_key":"give%2:40:01::","synset_id":"give.v.08"},{"index":9,"sense_key":"give%2:36:03::","synset_id":"give.v.09"},{"index":10,"sense_key":"give%2:31:02::","synset_id":"give.v.10"},{"index":11,"sense_key":"give%2:36:00::","synset_id":"render.v.04"},{"index":12,"sense_key":"give%2:40:09::","synset_id":"impart.v.01"},{"index":13,"sense_key":"give%2:36:01::","synset_id":"establish.v.05"},{"index":14,"sense_key":"give%2:40:02::","synset_id":"give.v.14"},{"index":15,"sense_key":"give%2:41:00::","synset_id":"give.v.15"},{"index":16,"sense_key":"give%2:40:10::","synset_id":"sacrifice.v.01"},{"index":17,"sense_key":"give%2:40:06::","synset_id":"pass.v.05"},{"index":18,"sense_key":"give%2:32:03::","synset_id":"give.v.18"},{"index":19,"sense_key":"give%2:40:13::","synset_id":"give.v.19"},{"index":20,"sense_key":"give%2:40:14::","synset_id":"give.v.20"},{"index":21,"sense_key":"give%2:32:02::","synset_id":"give.v.21"},{"index":22,"sense_key":"give%2:40:11::","synset_id":"grant.v.05"},{"index":23,"sense_key":"give%2:38:00::","synset_id":"move_over.v.01"},{"index":24,"sense_key":"give%2:34:00::","synset_id":"feed.v.02"},{"index":25,"sense_key":"give%2:40:08::","synset_id":"contribute.v.02"},{"index":26,"sense_key":"give%2:38:13::","synset_id":"collapse.v.01"},{"index":27,"sense_key":"give%2:31:00::","synset_id":"give.v.27"},{"index":28,"sense_key":"give%2:41:10::","synset_id":"give.v.28"},{"index":29,"sense_key":"give%2:40:12::","synset_id":"give.v.29"},{"index":30,"sense_key":"give%2:40:15::","synset_id":"afford.v.04"},{"index":31,"sense_key":"give%2:36:13::","synset_id":"give.v.31"},{"index":32,"sense_key":"give%2:36:12::","synset_id":"give.v.32"},{"index":33,"sense_key":"give%2:35:00::","synset_id":"give.v.33"},{"index":34,"sense_key":"give%2:34:10::","synset_id":"give.v.34"},{"index":35,"sense_key":"give%2:32:11::","synset_id":"give.v.35"},{"index":36,"sense_key":"give%2:32:08::","synset_id":"give.v.36"},{"index":37,"sense_key":"give%2:32:15::","synset_id":"give.v.37"},{"index":38,"sense_key":"give%2:32:14::","synset_id":"give.v.38"},{"index":39,"sense_key":"give%2:32:09::","synset_id":"give.v.39"},{"index":40,"sense_key":"give%2:32:07::","synset_id":"give.v.40"},{"index":41,"sense_key":"give%2:32:10::","synset_id":"give.v.41"},{"index":42,"sense_key":"give%2:30:13::","synset_id":"give.v.42"},{"index":43,"sense_key":"give%2:29:11::","synset_id":"give.v.43"},{"index":44,"sense_key":"give%2:29:10::","synset_id":"give.v.44"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":19,"chosen_sense_key":"give%2:40:13::","call_ids":["semeval2015.d003.s010.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":19,"predicted_sense_key":"give%2:40:13::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2853,"cached_input_tokens":0,"output_tokens":61,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.006316,"input_uncached_usd":0.005706,"input_cached_usd":0.0,"output_usd":0.0006100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":14.709450416034088}
{"item_id":"semeval2015.d003.s010.t002","gold_sense_keys":["day%1:28:00::"],"candidates":[{"index":1,"sense_key":"day%1:28:00::","synset_id":"day.n.01"},{"index":2,"sense_key":"day%1:28:03::","synset_id":"day.n.02"},{"index":3,"sense_key":"day%1:28:01::","synset_id":"day.n.03"},{"index":4,"sense_key":"day%1:28:02::","synset_id":"day.n.04"},{"index":5,"sense_key":"day%1:28:04::","synset_id":"day.n.05"},{"index":6,"sense_key":"day%1:28:05::","synset_id":"day.n.06"},{"index":7,"sense_key":"day%1:28:07::","synset_id":"day.n.07"},{"index":8,"sense_key":"day%1:28:06::","synset_id":"sidereal_day.n.01"},{"index":9,"sense_key":"day%1:26:00::","synset_id":"day.n.09"},{"index":10,"sense_key":"day%1:18:00::","synset_id":"day.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"day%1:28:00::","call_ids":["semeval2015.d003.s010.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"day%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1107,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0023339999999999997,"input_uncached_usd":0.002214,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.868242249940522}
{"item_id":"semeval2015.d003.s012.t001","gold_sense_keys":["receptor%1:08:01::"],"candidates":[{"index":1,"sense_key":"receptor%1:08:01::","synset_id":"receptor.n.01"},{"index":2,"sense_key":"receptor%1:08:00::","synset_id":"sense_organ.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"receptor%1:08:01::","call_ids":["semeval2015.d003.s012.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"receptor%1:08:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":598,"cached_input_tokens":0,"output_tokens":85,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002046,"input_uncached_usd":0.001196,"input_cached_usd":0.0,"output_usd":0.0008500000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8715495419455692}
{"item_id":"semeval2015.d003.s013.t000","gold_sense_keys":["study%2:31:02::"],"candidates":[{"index":1,"sense_key":"study%2:31:02::","synset_id":"analyze.v.01"},{"index":2,"sense_key":"study%2:31:03::","synset_id":"study.v.02"},{"index":3,"sense_key":"study%2:39:00::","synset_id":"study.v.03"},{"index":4,"sense_key":"study%2:31:00::","synset_id":"learn.v.04"},{"index":5,"sense_key":"study%2:31:01::","synset_id":"study.v.05"},{"index":6,"sense_key":"study%2:31:04::","synset_id":"study.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"study%2:31:02::","call_ids":["semeval2015.d003.s013.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"study%2:31:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":855,"cached_input_tokens":0,"output_tokens":98,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00269,"input_uncached_usd":0.00171,"input_cached_usd":0.0,"output_usd":0.0009800000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":14.400984833016992}
{"item_id":"semeval2015.d003.s014.t001","gold_sense_keys":["study%1:04:00::"],"candidates":[{"index":1,"sense_key":"study%1:04:00::","synset_id":"survey.n.01"},{"index":2,"sense_key":"study%1:09:03::","synset_id":"study.n.02"},{"index":3,"sense_key":"study%1:10:00::","synset_id":"report.n.01"},{"index":4,"sense_key":"study%1:09:04::","synset_id":"study.n.04"},{"index":5,"sense_key":"study%1:06:00::","synset_id":"study.n.05"},{"index":6,"sense_key":"study%1:09:02::","synset_id":"discipline.n.01"},{"index":7,"sense_key":"study%1:06:01::","synset_id":"sketch.n.01"},{"index":8,"sense_key":"study%1:09:01::","synset_id":"cogitation.n.02"},{"index":9,"sense_key":"study%1:18:00::","synset_id":"study.n.09"},{"index":10,"sense_key":"study%1:10:01::","synset_id":"study.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"study%1:10:00::","call_ids":["semeval2015.d003.s014.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"study%1:10:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1020,"cached_input_tokens":0,"output_tokens":173,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00377,"input_uncached_usd":0.0020399999999999997,"input_cached_usd":0.0,"output_usd":0.0017300000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1742012089816853}
{"item_id":"semeval2015.d003.s014.t002","gold_sense_keys":["carry_out%2:36:00::"],"candidates":[{"index":1,"sense_key":"carry_out%2:36:00::","synset_id":"carry_through.v.01"},{"index":2,"sense_key":"carry_out%2:30:00::","synset_id":"follow_through.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"carry_out%2:36:00::","call_ids":["semeval2015.d003.s014.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"carry_out%2:36:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":605,"cached_input_tokens":0,"output_tokens":118,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0023899999999999998,"input_uncached_usd":0.00121,"input_cached_usd":0.0,"output_usd":0.00118,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.3654501670971513}
{"item_id":"semeval2015.d003.s014.t003","gold_sense_keys":["laboratory%1:06:00::"],"candidates":[{"index":1,"sense_key":"laboratory%1:06:00::","synset_id":"lab.n.01"},{"index":2,"sense_key":"laboratory%1:15:00::","synset_id":"testing_ground.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"laboratory%1:06:00::","call_ids":["semeval2015.d003.s014.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"laboratory%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":622,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001364,"input_uncached_usd":0.0012439999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":13.844301291974261}
{"item_id":"semeval2015.d003.s014.t004","gold_sense_keys":["dog%1:05:00::"],"candidates":[{"index":1,"sense_key":"dog%1:05:00::","synset_id":"dog.n.01"},{"index":2,"sense_key":"dog%1:18:01::","synset_id":"frump.n.01"},{"index":3,"sense_key":"dog%1:18:00::","synset_id":"dog.n.03"},{"index":4,"sense_key":"dog%1:18:02::","synset_id":"cad.n.01"},{"index":5,"sense_key":"dog%1:13:01::","synset_id":"frank.n.02"},{"index":6,"sense_key":"dog%1:06:00::","synset_id":"pawl.n.01"},{"index":7,"sense_key":"dog%1:06:01::","synset_id":"andiron.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"dog%1:05:00::","call_ids":["semeval2015.d003.s014.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"dog%1:05:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":934,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0019879999999999997,"input_uncached_usd":0.0018679999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6589301669737324}
{"item_id":"semeval2015.d003.s014.t007","gold_sense_keys":["several%5:00:00:some:00"],"candidates":[{"index":1,"sense_key":"several%5:00:00:some:00","synset_id":"several.s.01"},{"index":2,"sense_key":"several%5:00:00:individual:00","synset_id":"respective.s.01"},{"index":3,"sense_key":"several%5:00:00:different:00","synset_id":"several.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"several%5:00:00:some:00","call_ids":["semeval2015.d003.s014.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"several%5:00:00:some:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":598,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0013160000000000001,"input_uncached_usd":0.001196,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.212472749990411}
{"item_id":"semeval2015.d003.s015.t000","gold_sense_keys":["benefit%1:07:00::"],"candidates":[{"index":1,"sense_key":"benefit%1:21:00::","synset_id":"benefit.n.01"},{"index":2,"sense_key":"benefit%1:07:00::","synset_id":"benefit.n.02"},{"index":3,"sense_key":"benefit%1:10:00::","synset_id":"benefit.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"benefit%1:07:00::","call_ids":["semeval2015.d003.s015.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"benefit%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":570,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00126,"input_uncached_usd":0.00114,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.258368708076887}
{"item_id":"semeval2015.d003.s015.t001","gold_sense_keys":["show%2:32:00::","show%2:39:00::"],"candidates":[{"index":1,"sense_key":"show%2:39:02::","synset_id":"show.v.01"},{"index":2,"sense_key":"show%2:31:00::","synset_id":"prove.v.02"},{"index":3,"sense_key":"show%2:32:00::","synset_id":"testify.v.02"},{"index":4,"sense_key":"show%2:39:00::","synset_id":"show.v.04"},{"index":5,"sense_key":"show%2:36:00::","synset_id":"picture.v.02"},{"index":6,"sense_key":"show%2:32:02::","synset_id":"express.v.01"},{"index":7,"sense_key":"show%2:32:01::","synset_id":"indicate.v.02"},{"index":8,"sense_key":"show%2:39:01::","synset_id":"show.v.08"},{"index":9,"sense_key":"show%2:32:04::","synset_id":"read.v.08"},{"index":10,"sense_key":"show%2:32:03::","synset_id":"show.v.10"},{"index":11,"sense_key":"show%2:38:00::","synset_id":"usher.v.01"},{"index":12,"sense_key":"show%2:33:00::","synset_id":"show.v.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"show%2:31:00::","call_ids":["semeval2015.d003.s015.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"show%2:31:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1258,"cached_input_tokens":0,"output_tokens":309,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.005606,"input_uncached_usd":0.002516,"input_cached_usd":0.0,"output_usd":0.0030900000000000003,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.535204541985877}
{"item_id":"semeval2015.d003.s015.t002","gold_sense_keys":["study%1:04:00::"],"candidates":[{"index":1,"sense_key":"study%1:04:00::","synset_id":"survey.n.01"},{"index":2,"sense_key":"study%1:09:03::","synset_id":"study.n.02"},{"index":3,"sense_key":"study%1:10:00::","synset_id":"report.n.01"},{"index":4,"sense_key":"study%1:09:04::","synset_id":"study.n.04"},{"index":5,"sense_key":"study%1:06:00::","synset_id":"study.n.05"},{"index":6,"sense_key":"study%1:09:02::","synset_id":"discipline.n.01"},{"index":7,"sense_key":"study%1:06:01::","synset_id":"sketch.n.01"},{"index":8,"sense_key":"study%1:09:01::","synset_id":"cogitation.n.02"},{"index":9,"sense_key":"study%1:18:00::","synset_id":"study.n.09"},{"index":10,"sense_key":"study%1:10:01::","synset_id":"study.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"study%1:10:00::","call_ids":["semeval2015.d003.s015.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"study%1:10:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1032,"cached_input_tokens":0,"output_tokens":121,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003274,"input_uncached_usd":0.002064,"input_cached_usd":0.0,"output_usd":0.0012100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.032215000013821}
{"item_id":"semeval2015.d003.s016.t000","gold_sense_keys":["result%1:11:00::"],"candidates":[{"index":1,"sense_key":"result%1:19:00::","synset_id":"consequence.n.01"},{"index":2,"sense_key":"result%1:10:00::","synset_id":"solution.n.02"},{"index":3,"sense_key":"result%1:11:00::","synset_id":"result.n.03"},{"index":4,"sense_key":"result%1:10:01::","synset_id":"resultant_role.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"result%1:11:00::","call_ids":["semeval2015.d003.s016.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"result%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":796,"cached_input_tokens":0,"output_tokens":118,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002772,"input_uncached_usd":0.0015919999999999999,"input_cached_usd":0.0,"output_usd":0.00118,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.363259124918841}
{"item_id":"semeval2015.d003.s016.t001","gold_sense_keys":["study%1:04:00::"],"candidates":[{"index":1,"sense_key":"study%1:04:00::","synset_id":"survey.n.01"},{"index":2,"sense_key":"study%1:09:03::","synset_id":"study.n.02"},{"index":3,"sense_key":"study%1:10:00::","synset_id":"report.n.01"},{"index":4,"sense_key":"study%1:09:04::","synset_id":"study.n.04"},{"index":5,"sense_key":"study%1:06:00::","synset_id":"study.n.05"},{"index":6,"sense_key":"study%1:09:02::","synset_id":"discipline.n.01"},{"index":7,"sense_key":"study%1:06:01::","synset_id":"sketch.n.01"},{"index":8,"sense_key":"study%1:09:01::","synset_id":"cogitation.n.02"},{"index":9,"sense_key":"study%1:18:00::","synset_id":"study.n.09"},{"index":10,"sense_key":"study%1:10:01::","synset_id":"study.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"study%1:10:00::","call_ids":["semeval2015.d003.s016.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"study%1:10:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1067,"cached_input_tokens":0,"output_tokens":104,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003174,"input_uncached_usd":0.002134,"input_cached_usd":0.0,"output_usd":0.0010400000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.095083750085905}
{"item_id":"semeval2015.d003.s016.t002","gold_sense_keys":["show%2:31:00::","show%2:32:00::"],"candidates":[{"index":1,"sense_key":"show%2:39:02::","synset_id":"show.v.01"},{"index":2,"sense_key":"show%2:31:00::","synset_id":"prove.v.02"},{"index":3,"sense_key":"show%2:32:00::","synset_id":"testify.v.02"},{"index":4,"sense_key":"show%2:39:00::","synset_id":"show.v.04"},{"index":5,"sense_key":"show%2:36:00::","synset_id":"picture.v.02"},{"index":6,"sense_key":"show%2:32:02::","synset_id":"express.v.01"},{"index":7,"sense_key":"show%2:32:01::","synset_id":"indicate.v.02"},{"index":8,"sense_key":"show%2:39:01::","synset_id":"show.v.08"},{"index":9,"sense_key":"show%2:32:04::","synset_id":"read.v.08"},{"index":10,"sense_key":"show%2:32:03::","synset_id":"show.v.10"},{"index":11,"sense_key":"show%2:38:00::","synset_id":"usher.v.01"},{"index":12,"sense_key":"show%2:33:00::","synset_id":"show.v.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"show%2:32:00::","call_ids":["semeval2015.d003.s016.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"show%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1293,"cached_input_tokens":0,"output_tokens":136,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003946,"input_uncached_usd":0.0025859999999999998,"input_cached_usd":0.0,"output_usd":0.00136,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":14.59667904197704}
{"item_id":"semeval2015.d003.s016.t003","gold_sense_keys":["more%4:02:00::"],"candidates":[{"index":1,"sense_key":"more%4:02:00::","synset_id":"more.r.01"},{"index":2,"sense_key":"more%4:02:01::","synset_id":"more.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"more%4:02:00::","call_ids":["semeval2015.d003.s016.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"more%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":595,"cached_input_tokens":0,"output_tokens":46,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00165,"input_uncached_usd":0.0011899999999999999,"input_cached_usd":0.0,"output_usd":0.00046,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9472788330167532}
{"item_id":"semeval2015.d003.s016.t004","gold_sense_keys":["effective%3:00:00::","effective%5:00:00:competent:00","effective%5:00:00:efficacious:00"],"candidates":[{"index":1,"sense_key":"effective%3:00:00::","synset_id":"effective.a.01"},{"index":2,"sense_key":"effective%5:00:00:competent:00","synset_id":"effective.s.01"},{"index":3,"sense_key":"effective%5:00:00:efficacious:00","synset_id":"effective.s.02"},{"index":4,"sense_key":"effective%5:00:00:operative:00","synset_id":"effective.s.03"},{"index":5,"sense_key":"effective%5:00:00:actual:00","synset_id":"effective.s.04"},{"index":6,"sense_key":"effective%5:00:00:operational:00","synset_id":"effective.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"effective%5:00:00:efficacious:00","call_ids":["semeval2015.d003.s016.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"effective%5:00:00:efficacious:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":946,"cached_input_tokens":0,"output_tokens":121,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003102,"input_uncached_usd":0.001892,"input_cached_usd":0.0,"output_usd":0.0012100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.3917882919777185}
{"item_id":"semeval2015.d003.s016.t005","gold_sense_keys":["placebo%1:06:00::"],"candidates":[{"index":1,"sense_key":"placebo%1:06:00::","synset_id":"placebo.n.01"},{"index":2,"sense_key":"placebo%1:04:00::","synset_id":"placebo.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"placebo%1:06:00::","call_ids":["semeval2015.d003.s016.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"placebo%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":586,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001292,"input_uncached_usd":0.0011719999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8699427079409361}
{"item_id":"semeval2015.d003.s016.t006","gold_sense_keys":["less%3:00:00::"],"candidates":[{"index":1,"sense_key":"less%3:00:00::","synset_id":"less.a.01"},{"index":2,"sense_key":"less%5:00:00:inferior:02","synset_id":"less.s.01"},{"index":3,"sense_key":"less%5:00:00:fewer:00","synset_id":"less.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"less%3:00:00::","call_ids":["semeval2015.d003.s016.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"less%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":675,"cached_input_tokens":0,"output_tokens":84,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00219,"input_uncached_usd":0.0013499999999999999,"input_cached_usd":0.0,"output_usd":0.00084,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8653964579571038}
{"item_id":"semeval2015.d003.s016.t008","gold_sense_keys":["see%2:39:02::"],"candidates":[{"index":1,"sense_key":"see%2:39:00::","synset_id":"see.v.01"},{"index":2,"sense_key":"see%2:31:01::","synset_id":"understand.v.02"},{"index":3,"sense_key":"see%2:39:02::","synset_id":"witness.v.02"},{"index":4,"sense_key":"see%2:36:00::","synset_id":"visualize.v.01"},{"index":5,"sense_key":"see%2:31:00::","synset_id":"see.v.05"},{"index":6,"sense_key":"see%2:31:02::","synset_id":"learn.v.02"},{"index":7,"sense_key":"see%2:39:03::","synset_id":"watch.v.03"},{"index":8,"sense_key":"see%2:38:00::","synset_id":"meet.v.01"},{"index":9,"sense_key":"see%2:32:00::","synset_id":"determine.v.08"},{"index":10,"sense_key":"see%2:31:03::","synset_id":"see.v.10"},{"index":11,"sense_key":"see%2:41:11::","synset_id":"see.v.11"},{"index":12,"sense_key":"see%2:41:12::","synset_id":"see.v.12"},{"index":13,"sense_key":"see%2:41:10::","synset_id":"visit.v.01"},{"index":14,"sense_key":"see%2:41:14::","synset_id":"attend.v.02"},{"index":15,"sense_key":"see%2:35:09::","synset_id":"see.v.15"},{"index":16,"sense_key":"see%2:41:00::","synset_id":"go_steady.v.01"},{"index":17,"sense_key":"see%2:39:05::","synset_id":"see.v.17"},{"index":18,"sense_key":"see%2:31:08::","synset_id":"see.v.18"},{"index":19,"sense_key":"see%2:39:11::","synset_id":"see.v.19"},{"index":20,"sense_key":"see%2:39:13::","synset_id":"examine.v.02"},{"index":21,"sense_key":"see%2:39:12::","synset_id":"experience.v.01"},{"index":22,"sense_key":"see%2:38:03::","synset_id":"see.v.22"},{"index":23,"sense_key":"see%2:33:00::","synset_id":"see.v.23"},{"index":24,"sense_key":"see%2:31:13::","synset_id":"interpret.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"see%2:39:02::","call_ids":["semeval2015.d003.s016.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"see%2:39:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2034,"cached_input_tokens":0,"output_tokens":129,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.005358,"input_uncached_usd":0.004068,"input_cached_usd":0.0,"output_usd":0.0012900000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.3300556250615045}
{"item_id":"semeval2015.d003.s016.t009","gold_sense_keys":["dog%1:05:00::"],"candidates":[{"index":1,"sense_key":"dog%1:05:00::","synset_id":"dog.n.01"},{"index":2,"sense_key":"dog%1:18:01::","synset_id":"frump.n.01"},{"index":3,"sense_key":"dog%1:18:00::","synset_id":"dog.n.03"},{"index":4,"sense_key":"dog%1:18:02::","synset_id":"cad.n.01"},{"index":5,"sense_key":"dog%1:13:01::","synset_id":"frank.n.02"},{"index":6,"sense_key":"dog%1:06:00::","synset_id":"pawl.n.01"},{"index":7,"sense_key":"dog%1:06:01::","synset_id":"andiron.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"dog%1:05:00::","call_ids":["semeval2015.d003.s016.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"dog%1:05:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":981,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0020819999999999996,"input_uncached_usd":0.0019619999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9624791670357808}
{"item_id":"semeval2015.d003.s016.t011","gold_sense_keys":["medicine%1:06:00::"],"candidates":[{"index":1,"sense_key":"medicine%1:09:00::","synset_id":"medicine.n.01"},{"index":2,"sense_key":"medicine%1:06:00::","synset_id":"medicine.n.02"},{"index":3,"sense_key":"medicine%1:04:00::","synset_id":"medicine.n.03"},{"index":4,"sense_key":"medicine%1:04:02::","synset_id":"music.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"medicine%1:06:00::","call_ids":["semeval2015.d003.s016.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"medicine%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":753,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001626,"input_uncached_usd":0.001506,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.52834966708906}
{"item_id":"semeval2015.d003.s016.t012","gold_sense_keys":["dog%1:05:00::"],"candidates":[{"index":1,"sense_key":"dog%1:05:00::","synset_id":"dog.n.01"},{"index":2,"sense_key":"dog%1:18:01::","synset_id":"frump.n.01"},{"index":3,"sense_key":"dog%1:18:00::","synset_id":"dog.n.03"},{"index":4,"sense_key":"dog%1:18:02::","synset_id":"cad.n.01"},{"index":5,"sense_key":"dog%1:13:01::","synset_id":"frank.n.02"},{"index":6,"sense_key":"dog%1:06:00::","synset_id":"pawl.n.01"},{"index":7,"sense_key":"dog%1:06:01::","synset_id":"andiron.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"dog%1:05:00::","call_ids":["semeval2015.d003.s016.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"dog%1:05:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":981,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0020819999999999996,"input_uncached_usd":0.0019619999999999998,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0872272080741823}
{"item_id":"semeval2015.d003.s016.t014","gold_sense_keys":["placebo%1:06:00::"],"candidates":[{"index":1,"sense_key":"placebo%1:06:00::","synset_id":"placebo.n.01"},{"index":2,"sense_key":"placebo%1:04:00::","synset_id":"placebo.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"placebo%1:06:00::","call_ids":["semeval2015.d003.s016.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"placebo%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":586,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001292,"input_uncached_usd":0.0011719999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.3776699169538915}
{"item_id":"semeval2015.d003.s016.t015","gold_sense_keys":["treatment%1:04:00::"],"candidates":[{"index":1,"sense_key":"treatment%1:04:00::","synset_id":"treatment.n.01"},{"index":2,"sense_key":"treatment%1:04:01::","synset_id":"treatment.n.02"},{"index":3,"sense_key":"treatment%1:07:00::","synset_id":"treatment.n.03"},{"index":4,"sense_key":"treatment%1:10:00::","synset_id":"discussion.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"treatment%1:04:00::","call_ids":["semeval2015.d003.s016.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"treatment%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":753,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001626,"input_uncached_usd":0.001506,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2115761670283973}
{"item_id":"semeval2015.d003.s017.t000","gold_sense_keys":["treatment%1:04:00::"],"candidates":[{"index":1,"sense_key":"treatment%1:04:00::","synset_id":"treatment.n.01"},{"index":2,"sense_key":"treatment%1:04:01::","synset_id":"treatment.n.02"},{"index":3,"sense_key":"treatment%1:07:00::","synset_id":"treatment.n.03"},{"index":4,"sense_key":"treatment%1:10:00::","synset_id":"discussion.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"treatment%1:04:00::","call_ids":["semeval2015.d003.s017.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"treatment%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":770,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00166,"input_uncached_usd":0.00154,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.2443759160814807}
{"item_id":"semeval2015.d003.s017.t002","gold_sense_keys":["dog%1:05:00::"],"candidates":[{"index":1,"sense_key":"dog%1:05:00::","synset_id":"dog.n.01"},{"index":2,"sense_key":"dog%1:18:01::","synset_id":"frump.n.01"},{"index":3,"sense_key":"dog%1:18:00::","synset_id":"dog.n.03"},{"index":4,"sense_key":"dog%1:18:02::","synset_id":"cad.n.01"},{"index":5,"sense_key":"dog%1:13:01::","synset_id":"frank.n.02"},{"index":6,"sense_key":"dog%1:06:00::","synset_id":"pawl.n.01"},{"index":7,"sense_key":"dog%1:06:01::","synset_id":"andiron.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"dog%1:05:00::","call_ids":["semeval2015.d003.s017.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"dog%1:05:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":997,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002114,"input_uncached_usd":0.001994,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5929467090172693}
{"item_id":"semeval2015.d003.s017.t003","gold_sense_keys":["start%2:30:01::","start%2:41:04::"],"candidates":[{"index":1,"sense_key":"start%2:30:00::","synset_id":"get_down.v.07"},{"index":2,"sense_key":"start%2:30:01::","synset_id":"begin.v.03"},{"index":3,"sense_key":"start%2:38:02::","synset_id":"depart.v.03"},{"index":4,"sense_key":"start%2:42:00::","synset_id":"begin.v.02"},{"index":5,"sense_key":"start%2:36:00::","synset_id":"originate.v.02"},{"index":6,"sense_key":"start%2:36:01::","synset_id":"start.v.06"},{"index":7,"sense_key":"start%2:38:04::","synset_id":"startle.v.02"},{"index":8,"sense_key":"start%2:38:01::","synset_id":"start.v.08"},{"index":9,"sense_key":"start%2:38:00::","synset_id":"start.v.09"},{"index":10,"sense_key":"start%2:41:00::","synset_id":"start.v.10"},{"index":11,"sense_key":"start%2:33:00::","synset_id":"start.v.11"},{"index":12,"sense_key":"start%2:42:03::","synset_id":"begin.v.07"},{"index":13,"sense_key":"start%2:41:04::","synset_id":"begin.v.08"},{"index":14,"sense_key":"start%2:38:03::","synset_id":"start.v.14"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"start%2:30:01::","call_ids":["semeval2015.d003.s017.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"start%2:30:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1380,"cached_input_tokens":0,"output_tokens":278,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00554,"input_uncached_usd":0.00276,"input_cached_usd":0.0,"output_usd":0.0027800000000000004,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.297885333071463}
{"item_id":"semeval2015.d003.s017.t004","gold_sense_keys":["injection%1:04:01::"],"candidates":[{"index":1,"sense_key":"injection%1:04:00::","synset_id":"injection.n.01"},{"index":2,"sense_key":"injection%1:27:00::","synset_id":"injection.n.02"},{"index":3,"sense_key":"injection%1:04:01::","synset_id":"injection.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"injection%1:04:01::","call_ids":["semeval2015.d003.s017.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"injection%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":644,"cached_input_tokens":0,"output_tokens":110,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002388,"input_uncached_usd":0.0012879999999999999,"input_cached_usd":0.0,"output_usd":0.0011,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":14.691936874995008}
{"item_id":"semeval2015.d003.s017.t008","gold_sense_keys":["able%3:00:00::"],"candidates":[{"index":1,"sense_key":"able%3:00:00::","synset_id":"able.a.01"},{"index":2,"sense_key":"able%5:00:00:competent:00","synset_id":"able.s.01"},{"index":3,"sense_key":"able%5:00:00:capable:00","synset_id":"able.s.02"},{"index":4,"sense_key":"able%5:00:00:fit:01","synset_id":"able.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"able%3:00:00::","call_ids":["semeval2015.d003.s017.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"able%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":749,"cached_input_tokens":0,"output_tokens":115,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002648,"input_uncached_usd":0.001498,"input_cached_usd":0.0,"output_usd":0.0011500000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.9270939159905538}
{"item_id":"semeval2015.d003.s017.t009","gold_sense_keys":["tablet%1:06:00::"],"candidates":[{"index":1,"sense_key":"tablet%1:06:02::","synset_id":"tablet.n.01"},{"index":2,"sense_key":"tablet%1:27:00::","synset_id":"pad.n.01"},{"index":3,"sense_key":"tablet%1:06:01::","synset_id":"tablet.n.03"},{"index":4,"sense_key":"tablet%1:06:00::","synset_id":"pill.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"tablet%1:06:00::","call_ids":["semeval2015.d003.s017.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"tablet%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":698,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001516,"input_uncached_usd":0.0013959999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5055458749411628}
{"item_id":"semeval2015.d003.s017.t010","gold_sense_keys":["intact%5:00:00:undamaged:00","intact%5:00:00:whole:00"],"candidates":[{"index":1,"sense_key":"intact%5:00:00:whole:00","synset_id":"integral.s.02"},{"index":2,"sense_key":"intact%5:00:00:uninjured:00","synset_id":"intact.s.02"},{"index":3,"sense_key":"intact%5:00:00:uncastrated:00","synset_id":"entire.s.04"},{"index":4,"sense_key":"intact%5:00:00:undamaged:00","synset_id":"intact.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"intact%5:00:00:undamaged:00","call_ids":["semeval2015.d003.s017.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"intact%5:00:00:undamaged:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":777,"cached_input_tokens":0,"output_tokens":108,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002634,"input_uncached_usd":0.0015539999999999998,"input_cached_usd":0.0,"output_usd":0.00108,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":11.063708709087223}
{"item_id":"semeval2015.d003.s017.t011","gold_sense_keys":["tablet%1:06:00::"],"candidates":[{"index":1,"sense_key":"tablet%1:06:02::","synset_id":"tablet.n.01"},{"index":2,"sense_key":"tablet%1:27:00::","synset_id":"pad.n.01"},{"index":3,"sense_key":"tablet%1:06:01::","synset_id":"tablet.n.03"},{"index":4,"sense_key":"tablet%1:06:00::","synset_id":"pill.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"tablet%1:06:00::","call_ids":["semeval2015.d003.s017.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"tablet%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":698,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001516,"input_uncached_usd":0.0013959999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.28302558301948}
{"item_id":"semeval2015.d003.s018.t000","gold_sense_keys":["follow-up%1:04:00::","follow-up%1:04:01::"],"candidates":[{"index":1,"sense_key":"follow-up%1:06:00::","synset_id":"follow-up.n.01"},{"index":2,"sense_key":"follow-up%1:04:01::","synset_id":"follow-up.n.02"},{"index":3,"sense_key":"follow-up%1:04:00::","synset_id":"follow-up.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"follow-up%1:04:01::","call_ids":["semeval2015.d003.s018.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"follow-up%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":684,"cached_input_tokens":0,"output_tokens":108,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0024479999999999997,"input_uncached_usd":0.0013679999999999999,"input_cached_usd":0.0,"output_usd":0.00108,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":14.399574875016697}
{"item_id":"semeval2015.d003.s018.t001","gold_sense_keys":["treatment%1:04:00::"],"candidates":[{"index":1,"sense_key":"treatment%1:04:00::","synset_id":"treatment.n.01"},{"index":2,"sense_key":"treatment%1:04:01::","synset_id":"treatment.n.02"},{"index":3,"sense_key":"treatment%1:07:00::","synset_id":"treatment.n.03"},{"index":4,"sense_key":"treatment%1:10:00::","synset_id":"discussion.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"treatment%1:04:00::","call_ids":["semeval2015.d003.s018.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"treatment%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":782,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001684,"input_uncached_usd":0.0015639999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.0715831669513136}
{"item_id":"semeval2015.d003.s018.t002","gold_sense_keys":["owner%1:18:02::"],"candidates":[{"index":1,"sense_key":"owner%1:18:00::","synset_id":"owner.n.01"},{"index":2,"sense_key":"owner%1:18:02::","synset_id":"owner.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"owner%1:18:02::","call_ids":["semeval2015.d003.s018.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"owner%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":644,"cached_input_tokens":0,"output_tokens":55,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0018379999999999998,"input_uncached_usd":0.0012879999999999999,"input_cached_usd":0.0,"output_usd":0.00055,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8142333339201286}
{"item_id":"semeval2015.d003.s018.t003","gold_sense_keys":["administer%2:29:00::"],"candidates":[{"index":1,"sense_key":"administer%2:41:00::","synset_id":"administer.v.01"},{"index":2,"sense_key":"administer%2:40:01::","synset_id":"administer.v.02"},{"index":3,"sense_key":"administer%2:40:00::","synset_id":"distribute.v.01"},{"index":4,"sense_key":"administer%2:29:00::","synset_id":"administer.v.04"},{"index":5,"sense_key":"administer%2:41:01::","synset_id":"administer.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"administer%2:29:00::","call_ids":["semeval2015.d003.s018.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"administer%2:29:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":785,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00169,"input_uncached_usd":0.00157,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":13.746023165993392}
{"item_id":"semeval2015.d003.s018.t004","gold_sense_keys":["tablet%1:06:00::"],"candidates":[{"index":1,"sense_key":"tablet%1:06:02::","synset_id":"tablet.n.01"},{"index":2,"sense_key":"tablet%1:27:00::","synset_id":"pad.n.01"},{"index":3,"sense_key":"tablet%1:06:01::","synset_id":"tablet.n.03"},{"index":4,"sense_key":"tablet%1:06:00::","synset_id":"pill.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"tablet%1:06:00::","call_ids":["semeval2015.d003.s018.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"tablet%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":710,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0015400000000000001,"input_uncached_usd":0.00142,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.167684750049375}
{"item_id":"semeval2015.d003.s018.t005","gold_sense_keys":["dog%1:05:00::"],"candidates":[{"index":1,"sense_key":"dog%1:05:00::","synset_id":"dog.n.01"},{"index":2,"sense_key":"dog%1:18:01::","synset_id":"frump.n.01"},{"index":3,"sense_key":"dog%1:18:00::","synset_id":"dog.n.03"},{"index":4,"sense_key":"dog%1:18:02::","synset_id":"cad.n.01"},{"index":5,"sense_key":"dog%1:13:01::","synset_id":"frank.n.02"},{"index":6,"sense_key":"dog%1:06:00::","synset_id":"pawl.n.01"},{"index":7,"sense_key":"dog%1:06:01::","synset_id":"andiron.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"dog%1:05:00::","call_ids":["semeval2015.d003.s018.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"dog%1:05:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1009,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0021379999999999997,"input_uncached_usd":0.002018,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.06492487504147}
{"item_id":"semeval2015.d003.s019.t001","gold_sense_keys":["treatment%1:04:00::"],"candidates":[{"index":1,"sense_key":"treatment%1:04:00::","synset_id":"treatment.n.01"},{"index":2,"sense_key":"treatment%1:04:01::","synset_id":"treatment.n.02"},{"index":3,"sense_key":"treatment%1:07:00::","synset_id":"treatment.n.03"},{"index":4,"sense_key":"treatment%1:10:00::","synset_id":"discussion.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"treatment%1:04:00::","call_ids":["semeval2015.d003.s019.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"treatment%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":785,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00169,"input_uncached_usd":0.00157,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5260502499295399}
{"item_id":"semeval2015.d003.s019.t003","gold_sense_keys":["only%4:02:01::"],"candidates":[{"index":1,"sense_key":"only%4:02:02::","synset_id":"merely.r.01"},{"index":2,"sense_key":"only%4:02:01::","synset_id":"entirely.r.02"},{"index":3,"sense_key":"only%4:02:00::","synset_id":"only.r.03"},{"index":4,"sense_key":"only%4:02:04::","synset_id":"only.r.04"},{"index":5,"sense_key":"only%4:02:06::","synset_id":"only.r.05"},{"index":6,"sense_key":"only%4:02:03::","synset_id":"only.r.06"},{"index":7,"sense_key":"only%4:02:05::","synset_id":"only.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"only%4:02:03::","call_ids":["semeval2015.d003.s019.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"only%4:02:03::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":877,"cached_input_tokens":0,"output_tokens":148,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003234,"input_uncached_usd":0.001754,"input_cached_usd":0.0,"output_usd":0.0014800000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.913290375028737}
{"item_id":"semeval2015.d003.s019.t005","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":2,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":3,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"other%3:00:00::","call_ids":["semeval2015.d003.s019.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":745,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00161,"input_uncached_usd":0.00149,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.8972056250786409}
{"item_id":"semeval2015.d003.s019.t007","gold_sense_keys":["measure%1:04:01::"],"candidates":[{"index":1,"sense_key":"measure%1:04:01::","synset_id":"measure.n.01"},{"index":2,"sense_key":"measure%1:03:00::","synset_id":"measure.n.02"},{"index":3,"sense_key":"measure%1:10:02::","synset_id":"bill.n.01"},{"index":4,"sense_key":"measure%1:04:00::","synset_id":"measurement.n.01"},{"index":5,"sense_key":"measure%1:10:03::","synset_id":"standard.n.01"},{"index":6,"sense_key":"measure%1:10:01::","synset_id":"meter.n.03"},{"index":7,"sense_key":"measure%1:10:00::","synset_id":"measure.n.07"},{"index":8,"sense_key":"measure%1:06:00::","synset_id":"measuring_stick.n.01"},{"index":9,"sense_key":"measure%1:06:01::","synset_id":"measure.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"measure%1:04:01::","call_ids":["semeval2015.d003.s019.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"measure%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1113,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002346,"input_uncached_usd":0.002226,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.951253000064753}
{"item_id":"semeval2015.d003.s019.t008","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":2,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":3,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"other%3:00:00::","call_ids":["semeval2015.d003.s019.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":745,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00161,"input_uncached_usd":0.00149,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.4984718749765307}
{"item_id":"semeval2015.d003.s019.t011","gold_sense_keys":["address%2:34:00::"],"candidates":[{"index":1,"sense_key":"address%2:32:00::","synset_id":"address.v.01"},{"index":2,"sense_key":"address%2:32:02::","synset_id":"address.v.02"},{"index":3,"sense_key":"address%2:32:03::","synset_id":"address.v.03"},{"index":4,"sense_key":"address%2:33:00::","synset_id":"address.v.04"},{"index":5,"sense_key":"address%2:34:00::","synset_id":"address.v.05"},{"index":6,"sense_key":"address%2:41:08::","synset_id":"address.v.06"},{"index":7,"sense_key":"address%2:40:00::","synset_id":"address.v.07"},{"index":8,"sense_key":"address%2:32:04::","synset_id":"cover.v.05"},{"index":9,"sense_key":"address%2:32:01::","synset_id":"address.v.09"},{"index":10,"sense_key":"address%2:30:00::","synset_id":"address.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"address%2:34:00::","call_ids":["semeval2015.d003.s019.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"address%2:34:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":980,"cached_input_tokens":0,"output_tokens":118,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00314,"input_uncached_usd":0.00196,"input_cached_usd":0.0,"output_usd":0.00118,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.6566108330152929}
{"item_id":"semeval2015.d003.s019.t012","gold_sense_keys":["underlying%5:00:00:implicit:00"],"candidates":[{"index":1,"sense_key":"underlying%5:00:00:implicit:00","synset_id":"implicit_in.s.01"},{"index":2,"sense_key":"underlying%5:00:00:subjacent:00","synset_id":"underlying.s.02"},{"index":3,"sense_key":"underlying%5:00:00:basic:00","synset_id":"fundamental.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"underlying%5:00:00:basic:00","call_ids":["semeval2015.d003.s019.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"underlying%5:00:00:basic:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":710,"cached_input_tokens":0,"output_tokens":189,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0033100000000000004,"input_uncached_usd":0.00142,"input_cached_usd":0.0,"output_usd":0.0018900000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.276735167019069}
{"item_id":"semeval2015.d003.s019.t013","gold_sense_keys":["cause%1:10:00::","cause%1:11:00::"],"candidates":[{"index":1,"sense_key":"cause%1:11:00::","synset_id":"cause.n.01"},{"index":2,"sense_key":"cause%1:10:00::","synset_id":"cause.n.02"},{"index":3,"sense_key":"cause%1:04:01::","synset_id":"campaign.n.02"},{"index":4,"sense_key":"cause%1:03:00::","synset_id":"causal_agent.n.01"},{"index":5,"sense_key":"cause%1:04:00::","synset_id":"lawsuit.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cause%1:11:00::","call_ids":["semeval2015.d003.s019.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cause%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":867,"cached_input_tokens":0,"output_tokens":131,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003044,"input_uncached_usd":0.0017339999999999999,"input_cached_usd":0.0,"output_usd":0.0013100000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.2007317919051275}
{"item_id":"semeval2015.d003.s020.t000","gold_sense_keys":["risk%1:26:00::"],"candidates":[{"index":1,"sense_key":"risk%1:26:00::","synset_id":"hazard.n.01"},{"index":2,"sense_key":"risk%1:04:00::","synset_id":"risk.n.02"},{"index":3,"sense_key":"risk%1:07:02::","synset_id":"risk.n.03"},{"index":4,"sense_key":"risk%1:07:01::","synset_id":"risk.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"risk%1:26:00::","call_ids":["semeval2015.d003.s020.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"risk%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":800,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00172,"input_uncached_usd":0.0015999999999999999,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5541758330073208}
{"item_id":"semeval2015.d003.s021.t000","gold_sense_keys":["generally%4:02:02::"],"candidates":[{"index":1,"sense_key":"generally%4:02:02::","synset_id":"by_and_large.r.01"},{"index":2,"sense_key":"generally%4:02:01::","synset_id":"generally.r.02"},{"index":3,"sense_key":"generally%4:02:00::","synset_id":"broadly.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"generally%4:02:02::","call_ids":["semeval2015.d003.s021.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"generally%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":721,"cached_input_tokens":0,"output_tokens":120,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0026420000000000003,"input_uncached_usd":0.001442,"input_cached_usd":0.0,"output_usd":0.0012000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":15.429026375059038}
{"item_id":"semeval2015.d003.s021.t001","gold_sense_keys":["well%4:02:00::"],"candidates":[{"index":1,"sense_key":"well%4:02:00::","synset_id":"well.r.01"},{"index":2,"sense_key":"well%4:02:05::","synset_id":"well.r.02"},{"index":3,"sense_key":"well%4:02:03::","synset_id":"well.r.03"},{"index":4,"sense_key":"well%4:02:15::","synset_id":"well.r.04"},{"index":5,"sense_key":"well%4:02:08::","synset_id":"well.r.05"},{"index":6,"sense_key":"well%4:02:07::","synset_id":"well.r.06"},{"index":7,"sense_key":"well%4:02:11::","synset_id":"well.r.07"},{"index":8,"sense_key":"well%4:02:14::","synset_id":"well.r.08"},{"index":9,"sense_key":"well%4:02:13::","synset_id":"well.r.09"},{"index":10,"sense_key":"well%4:02:12::","synset_id":"well.r.10"},{"index":11,"sense_key":"well%4:02:10::","synset_id":"well.r.11"},{"index":12,"sense_key":"well%4:02:09::","synset_id":"well.r.12"},{"index":13,"sense_key":"well%4:02:01::","synset_id":"well.r.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"well%4:02:00::","call_ids":["semeval2015.d003.s021.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"well%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1341,"cached_input_tokens":0,"output_tokens":176,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.004442,"input_uncached_usd":0.002682,"input_cached_usd":0.0,"output_usd":0.00176,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.1506186249898747}
{"item_id":"semeval2015.d003.s021.t002","gold_sense_keys":["tolerate%2:39:00::"],"candidates":[{"index":1,"sense_key":"tolerate%2:31:00::","synset_id":"digest.v.03"},{"index":2,"sense_key":"tolerate%2:41:01::","synset_id":"tolerate.v.02"},{"index":3,"sense_key":"tolerate%2:39:00::","synset_id":"tolerate.v.03"},{"index":4,"sense_key":"tolerate%2:32:01::","synset_id":"allow.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"tolerate%2:39:00::","call_ids":["semeval2015.d003.s021.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"tolerate%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":859,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001838,"input_uncached_usd":0.001718,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5491298750275746}
{"item_id":"semeval2015.d003.s021.t003","gold_sense_keys":["administer%2:29:00::"],"candidates":[{"index":1,"sense_key":"administer%2:41:00::","synset_id":"administer.v.01"},{"index":2,"sense_key":"administer%2:40:01::","synset_id":"administer.v.02"},{"index":3,"sense_key":"administer%2:40:00::","synset_id":"distribute.v.01"},{"index":4,"sense_key":"administer%2:29:00::","synset_id":"administer.v.04"},{"index":5,"sense_key":"administer%2:41:01::","synset_id":"administer.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"administer%2:29:00::","call_ids":["semeval2015.d003.s021.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"administer%2:29:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":811,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001742,"input_uncached_usd":0.001622,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.385482291923836}
{"item_id":"semeval2015.d003.s021.t004","gold_sense_keys":["daily%4:02:00::"],"candidates":[{"index":1,"sense_key":"daily%4:02:00::","synset_id":"daily.r.01"},{"index":2,"sense_key":"daily%4:02:01::","synset_id":"day_by_day.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"daily%4:02:00::","call_ids":["semeval2015.d003.s021.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"daily%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":625,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0013700000000000001,"input_uncached_usd":0.00125,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.0142875000601634}
{"item_id":"semeval2015.d003.s021.t005","gold_sense_keys":["dosage%1:06:00::"],"candidates":[{"index":1,"sense_key":"dosage%1:23:00::","synset_id":"dose.n.02"},{"index":2,"sense_key":"dosage%1:06:00::","synset_id":"dose.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"dosage%1:06:00::","call_ids":["semeval2015.d003.s021.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"dosage%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":643,"cached_input_tokens":0,"output_tokens":120,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0024860000000000004,"input_uncached_usd":0.001286,"input_cached_usd":0.0,"output_usd":0.0012000000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.274018916999921}
{"item_id":"semeval2015.d003.s021.t008","gold_sense_keys":["dose%1:06:00::"],"candidates":[{"index":1,"sense_key":"dose%1:06:00::","synset_id":"dose.n.01"},{"index":2,"sense_key":"dose%1:23:00::","synset_id":"dose.n.02"},{"index":3,"sense_key":"dose%1:26:00::","synset_id":"venereal_disease.n.01"},{"index":4,"sense_key":"dose%1:06:01::","synset_id":"acid.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"dose%1:06:00::","call_ids":["semeval2015.d003.s021.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"dose%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":794,"cached_input_tokens":0,"output_tokens":100,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002588,"input_uncached_usd":0.001588,"input_cached_usd":0.0,"output_usd":0.001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.7477502920664847}
{"item_id":"semeval2015.d003.s021.t009","gold_sense_keys":["extended%5:00:00:long:02"],"candidates":[{"index":1,"sense_key":"extended%5:00:00:long:02","synset_id":"drawn-out.s.01"},{"index":2,"sense_key":"extended%3:00:00::","synset_id":"extended.a.02"},{"index":3,"sense_key":"extended%5:00:00:long:01","synset_id":"elongated.s.01"},{"index":4,"sense_key":"extended%5:00:00:figurative:00","synset_id":"extended.s.03"},{"index":5,"sense_key":"extended%5:00:00:large:00","synset_id":"extensive.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"extended%5:00:00:long:02","call_ids":["semeval2015.d003.s021.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"extended%5:00:00:long:02","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":899,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001918,"input_uncached_usd":0.001798,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.002048584050499}
{"item_id":"semeval2015.d003.s021.t013","gold_sense_keys":["duration%1:07:00::","duration%1:28:02::"],"candidates":[{"index":1,"sense_key":"duration%1:28:02::","synset_id":"duration.n.01"},{"index":2,"sense_key":"duration%1:28:00::","synset_id":"duration.n.02"},{"index":3,"sense_key":"duration%1:07:00::","synset_id":"duration.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"duration%1:07:00::","call_ids":["semeval2015.d003.s021.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"duration%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":676,"cached_input_tokens":0,"output_tokens":162,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002972,"input_uncached_usd":0.001352,"input_cached_usd":0.0,"output_usd":0.0016200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.8689126250101253}
{"item_id":"semeval2015.d003.s021.t014","gold_sense_keys":["treatment%1:04:00::"],"candidates":[{"index":1,"sense_key":"treatment%1:04:00::","synset_id":"treatment.n.01"},{"index":2,"sense_key":"treatment%1:04:01::","synset_id":"treatment.n.02"},{"index":3,"sense_key":"treatment%1:07:00::","synset_id":"treatment.n.03"},{"index":4,"sense_key":"treatment%1:10:00::","synset_id":"discussion.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"treatment%1:04:00::","call_ids":["semeval2015.d003.s021.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"treatment%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":808,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.0017360000000000001,"input_uncached_usd":0.001616,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.5770649999612942}
{"item_id":"semeval2015.d003.s022.t000","gold_sense_keys":["precaution%1:04:00::"],"candidates":[{"index":1,"sense_key":"precaution%1:04:00::","synset_id":"precaution.n.01"},{"index":2,"sense_key":"precaution%1:07:00::","synset_id":"precaution.n.02"},{"index":3,"sense_key":"precaution%1:09:00::","synset_id":"caution.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"precaution%1:04:00::","call_ids":["semeval2015.d003.s022.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"precaution%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":686,"cached_input_tokens":0,"output_tokens":86,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002232,"input_uncached_usd":0.001372,"input_cached_usd":0.0,"output_usd":0.0008600000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.029334624996409}
{"item_id":"semeval2015.d003.s022.t001","gold_sense_keys":["person%1:03:00::"],"candidates":[{"index":1,"sense_key":"person%1:03:00::","synset_id":"person.n.01"},{"index":2,"sense_key":"person%1:08:00::","synset_id":"person.n.02"},{"index":3,"sense_key":"person%1:10:00::","synset_id":"person.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"person%1:03:00::","call_ids":["semeval2015.d003.s022.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"person%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":673,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001466,"input_uncached_usd":0.001346,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":4.636511249933392}
{"item_id":"semeval2015.d003.s022.t002","gold_sense_keys":["give%2:40:13::"],"candidates":[{"index":1,"sense_key":"give%2:40:03::","synset_id":"give.v.01"},{"index":2,"sense_key":"give%2:40:05::","synset_id":"yield.v.01"},{"index":3,"sense_key":"give%2:40:00::","synset_id":"give.v.03"},{"index":4,"sense_key":"give%2:40:07::","synset_id":"give.v.04"},{"index":5,"sense_key":"give%2:32:01::","synset_id":"give.v.05"},{"index":6,"sense_key":"give%2:36:02::","synset_id":"hold.v.03"},{"index":7,"sense_key":"give%2:32:00::","synset_id":"give.v.07"},{"index":8,"sense_key":"give%2:40:01::","synset_id":"give.v.08"},{"index":9,"sense_key":"give%2:36:03::","synset_id":"give.v.09"},{"index":10,"sense_key":"give%2:31:02::","synset_id":"give.v.10"},{"index":11,"sense_key":"give%2:36:00::","synset_id":"render.v.04"},{"index":12,"sense_key":"give%2:40:09::","synset_id":"impart.v.01"},{"index":13,"sense_key":"give%2:36:01::","synset_id":"establish.v.05"},{"index":14,"sense_key":"give%2:40:02::","synset_id":"give.v.14"},{"index":15,"sense_key":"give%2:41:00::","synset_id":"give.v.15"},{"index":16,"sense_key":"give%2:40:10::","synset_id":"sacrifice.v.01"},{"index":17,"sense_key":"give%2:40:06::","synset_id":"pass.v.05"},{"index":18,"sense_key":"give%2:32:03::","synset_id":"give.v.18"},{"index":19,"sense_key":"give%2:40:13::","synset_id":"give.v.19"},{"index":20,"sense_key":"give%2:40:14::","synset_id":"give.v.20"},{"index":21,"sense_key":"give%2:32:02::","synset_id":"give.v.21"},{"index":22,"sense_key":"give%2:40:11::","synset_id":"grant.v.05"},{"index":23,"sense_key":"give%2:38:00::","synset_id":"move_over.v.01"},{"index":24,"sense_key":"give%2:34:00::","synset_id":"feed.v.02"},{"index":25,"sense_key":"give%2:40:08::","synset_id":"contribute.v.02"},{"index":26,"sense_key":"give%2:38:13::","synset_id":"collapse.v.01"},{"index":27,"sense_key":"give%2:31:00::","synset_id":"give.v.27"},{"index":28,"sense_key":"give%2:41:10::","synset_id":"give.v.28"},{"index":29,"sense_key":"give%2:40:12::","synset_id":"give.v.29"},{"index":30,"sense_key":"give%2:40:15::","synset_id":"afford.v.04"},{"index":31,"sense_key":"give%2:36:13::","synset_id":"give.v.31"},{"index":32,"sense_key":"give%2:36:12::","synset_id":"give.v.32"},{"index":33,"sense_key":"give%2:35:00::","synset_id":"give.v.33"},{"index":34,"sense_key":"give%2:34:10::","synset_id":"give.v.34"},{"index":35,"sense_key":"give%2:32:11::","synset_id":"give.v.35"},{"index":36,"sense_key":"give%2:32:08::","synset_id":"give.v.36"},{"index":37,"sense_key":"give%2:32:15::","synset_id":"give.v.37"},{"index":38,"sense_key":"give%2:32:14::","synset_id":"give.v.38"},{"index":39,"sense_key":"give%2:32:09::","synset_id":"give.v.39"},{"index":40,"sense_key":"give%2:32:07::","synset_id":"give.v.40"},{"index":41,"sense_key":"give%2:32:10::","synset_id":"give.v.41"},{"index":42,"sense_key":"give%2:30:13::","synset_id":"give.v.42"},{"index":43,"sense_key":"give%2:29:11::","synset_id":"give.v.43"},{"index":44,"sense_key":"give%2:29:10::","synset_id":"give.v.44"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":19,"chosen_sense_key":"give%2:40:13::","call_ids":["semeval2015.d003.s022.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":19,"predicted_sense_key":"give%2:40:13::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2879,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.005878,"input_uncached_usd":0.005758,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":0.9381794170476496}
{"item_id":"semeval2015.d003.s022.t003","gold_sense_keys":["medicine%1:06:00::"],"candidates":[{"index":1,"sense_key":"medicine%1:09:00::","synset_id":"medicine.n.01"},{"index":2,"sense_key":"medicine%1:06:00::","synset_id":"medicine.n.02"},{"index":3,"sense_key":"medicine%1:04:00::","synset_id":"medicine.n.03"},{"index":4,"sense_key":"medicine%1:04:02::","synset_id":"music.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"medicine%1:06:00::","call_ids":["semeval2015.d003.s022.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"medicine%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":753,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001626,"input_uncached_usd":0.001506,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":4.531057249987498}
{"item_id":"semeval2015.d003.s022.t005","gold_sense_keys":["contact%1:04:00::","contact%1:26:00::"],"candidates":[{"index":1,"sense_key":"contact%1:04:02::","synset_id":"contact.n.01"},{"index":2,"sense_key":"contact%1:04:00::","synset_id":"contact.n.02"},{"index":3,"sense_key":"contact%1:26:00::","synset_id":"contact.n.03"},{"index":4,"sense_key":"contact%1:11:00::","synset_id":"contact.n.04"},{"index":5,"sense_key":"contact%1:18:00::","synset_id":"contact.n.05"},{"index":6,"sense_key":"contact%1:10:00::","synset_id":"liaison.n.02"},{"index":7,"sense_key":"contact%1:06:00::","synset_id":"contact.n.07"},{"index":8,"sense_key":"contact%1:10:01::","synset_id":"contact.n.08"},{"index":9,"sense_key":"contact%1:06:01::","synset_id":"contact.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"contact%1:04:00::","call_ids":["semeval2015.d003.s022.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"contact%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1058,"cached_input_tokens":0,"output_tokens":121,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003326,"input_uncached_usd":0.002116,"input_cached_usd":0.0,"output_usd":0.0012100000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":5.427213957998902}
{"item_id":"semeval2015.d003.s023.t001","gold_sense_keys":["represent%2:42:00::"],"candidates":[{"index":1,"sense_key":"represent%2:42:02::","synset_id":"represent.v.01"},{"index":2,"sense_key":"represent%2:32:02::","synset_id":"typify.v.02"},{"index":3,"sense_key":"represent%2:41:02::","synset_id":"represent.v.03"},{"index":4,"sense_key":"represent%2:41:00::","synset_id":"represent.v.04"},{"index":5,"sense_key":"represent%2:32:00::","synset_id":"represent.v.05"},{"index":6,"sense_key":"represent%2:42:01::","synset_id":"exemplify.v.01"},{"index":7,"sense_key":"represent%2:42:00::","synset_id":"constitute.v.01"},{"index":8,"sense_key":"represent%2:41:06::","synset_id":"defend.v.06"},{"index":9,"sense_key":"represent%2:36:01::","synset_id":"represent.v.09"},{"index":10,"sense_key":"represent%2:36:02::","synset_id":"act.v.03"},{"index":11,"sense_key":"represent%2:36:11::","synset_id":"stage.v.01"},{"index":12,"sense_key":"represent%2:32:12::","synset_id":"represent.v.12"},{"index":13,"sense_key":"represent%2:32:10::","synset_id":"represent.v.13"},{"index":14,"sense_key":"represent%2:32:11::","synset_id":"present.v.02"},{"index":15,"sense_key":"represent%2:30:06::","synset_id":"map.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"represent%2:42:00::","call_ids":["semeval2015.d003.s023.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"represent%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1405,"cached_input_tokens":0,"output_tokens":82,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.00363,"input_uncached_usd":0.00281,"input_cached_usd":0.0,"output_usd":0.0008200000000000001,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":1.581784375011921}
{"item_id":"semeval2015.d003.s023.t002","gold_sense_keys":["particular%5:00:00:uncommon:00","particular%5:00:02:specific:00"],"candidates":[{"index":1,"sense_key":"particular%5:00:00:specific:00","synset_id":"particular.s.01"},{"index":2,"sense_key":"particular%5:00:02:specific:00","synset_id":"particular.s.02"},{"index":3,"sense_key":"particular%5:00:00:uncommon:00","synset_id":"especial.s.01"},{"index":4,"sense_key":"particular%5:00:00:primary:00","synset_id":"particular.s.04"},{"index":5,"sense_key":"particular%5:00:00:fastidious:00","synset_id":"finical.s.01"},{"index":6,"sense_key":"particular%5:00:00:careful:00","synset_id":"particular.s.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"particular%5:00:00:uncommon:00","call_ids":["semeval2015.d003.s023.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"particular%5:00:00:uncommon:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":839,"cached_input_tokens":0,"output_tokens":147,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.003148,"input_uncached_usd":0.001678,"input_cached_usd":0.0,"output_usd":0.0014700000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":2.01925133401528}
{"item_id":"semeval2015.d003.s023.t003","gold_sense_keys":["risk%1:26:00::"],"candidates":[{"index":1,"sense_key":"risk%1:26:00::","synset_id":"hazard.n.01"},{"index":2,"sense_key":"risk%1:04:00::","synset_id":"risk.n.02"},{"index":3,"sense_key":"risk%1:07:02::","synset_id":"risk.n.03"},{"index":4,"sense_key":"risk%1:07:01::","synset_id":"risk.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"risk%1:26:00::","call_ids":["semeval2015.d003.s023.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"risk%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":701,"cached_input_tokens":0,"output_tokens":101,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.002412,"input_uncached_usd":0.001402,"input_cached_usd":0.0,"output_usd":0.00101,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":3.758547457982786}
{"item_id":"semeval2015.d003.s023.t004","gold_sense_keys":["people%1:14:00::","people%1:14:03::"],"candidates":[{"index":1,"sense_key":"people%1:14:00::","synset_id":"people.n.01"},{"index":2,"sense_key":"people%1:14:01::","synset_id":"citizenry.n.01"},{"index":3,"sense_key":"people%1:14:02::","synset_id":"people.n.03"},{"index":4,"sense_key":"people%1:14:03::","synset_id":"multitude.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"people%1:14:00::","call_ids":["semeval2015.d003.s023.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"people%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":658,"cached_input_tokens":0,"output_tokens":12,"reasoning_output_tokens":0},"cost":{"currency":"USD","total_usd":0.001436,"input_uncached_usd":0.001316,"input_cached_usd":0.0,"output_usd":0.00012000000000000002,"input_uncached_unit_price_usd":2e-6,"input_cached_unit_price_usd":1e-7,"output_unit_price_usd":0.00001,"source":"litellm_estimate"},"latency_seconds":6.365192750003189}