{ "generatedAt": "2026-10-11T10:03:57.060Z", "seed": 20260926, "models": [ "jev", "laya", "clef-flash", "clef", "microsoft-decision-1" ], "jevModel": "typesafe-ai/jev", "layaVersion": "0.3.20", "layaDevice": "mps", "tasks": { "banking77": { "name": "Banking77 intent (77 options)", "source": "https://github.com/PolyAI-LDN/task-specific-datasets", "license": "CC BY 4.0", "split": "test", "question": { "type": "choice", "options": 77 }, "jev": { "n": 100, "failed": 0, "accuracy": 0.79, "accuracy95": [ 0.7001985069803768, 0.8583445046835422 ], "meanConfidence": 0.8933999999999997, "ece": 0.10339999999999985, "predictionShare": { "card_payment_not_recognised": 0.02, "lost_or_stolen_phone": 0.01, "exchange_rate": 0.02, "top_up_failed": 0.04, "apple_pay_or_google_pay": 0.01, "passcode_forgotten": 0.01, "pending_cash_withdrawal": 0.03, "compromised_card": 0.02, "disposable_card_limits": 0.01, "exchange_charge": 0.02, "verify_top_up": 0.03, "request_refund": 0.02, "Refund_not_showing_up": 0.05, "pending_top_up": 0.01, "card_payment_wrong_exchange_rate": 0.01, "country_support": 0.01, "card_linking": 0.03, "unable_to_verify_identity": 0.03, "activate_my_card": 0.02, "change_pin": 0.06, "transfer_timing": 0.04, "transfer_not_received_by_recipient": 0.01, "get_disposable_virtual_card": 0.01, "getting_virtual_card": 0.04, "verify_source_of_funds": 0.02, "reverted_card_payment?": 0.01, "top_up_by_card_charge": 0.02, "card_not_working": 0.01, "verify_my_identity": 0.02, "direct_debit_payment_not_recognised": 0.01, "balance_not_updated_after_bank_transfer": 0.01, "declined_card_payment": 0.02, "pending_transfer": 0.05, "cash_withdrawal_charge": 0.01, "top_up_by_cash_or_cheque": 0.01, "exchange_via_app": 0.01, "get_physical_card": 0.01, "top_up_reverted": 0.01, "pin_blocked": 0.01, "atm_support": 0.03, "contactless_not_working": 0.01, "card_swallowed": 0.02, "transfer_into_account": 0.02, "top_up_limits": 0.01, "balance_not_updated_after_cheque_or_cash_deposit": 0.01, "card_about_to_expire": 0.01, "transfer_fee_charged": 0.03, "transaction_charged_twice": 0.01, "wrong_exchange_rate_for_cash_withdrawal": 0.01, "fiat_currency_support": 0.01, "wrong_amount_of_cash_received": 0.01, "lost_or_stolen_card": 0.01, "card_delivery_estimate": 0.01, "extra_charge_on_statement": 0.01 }, "highConfidenceShare": 0.71, "highConfidenceAccuracy": 0.9295774647887324, "latencyP50Ms": 1680, "latencyP95Ms": 3915, "meanInputTokens": 1687.44, "usdPer1000Calls": 0.07087248000000002, "reliability": [ { "from": 0.4, "to": 0.5, "n": 3, "accuracy": 0.333, "confidence": 0.467 }, { "from": 0.5, "to": 0.6, "n": 8, "accuracy": 0.375, "confidence": 0.541 }, { "from": 0.6, "to": 0.7, "n": 3, "accuracy": 0, "confidence": 0.637 }, { "from": 0.7, "to": 0.8, "n": 7, "accuracy": 0.714, "confidence": 0.747 }, { "from": 0.8, "to": 0.9, "n": 8, "accuracy": 0.5, "confidence": 0.848 }, { "from": 0.9, "to": 1, "n": 71, "accuracy": 0.93, "confidence": 0.982 } ] }, "laya": { "n": 100, "failed": 0, "accuracy": 0.35, "accuracy95": [ 0.2636410465347802, 0.4474573957080203 ], "meanConfidence": 0.9012379999999998, "ece": 0.5568779999999999, "predictionShare": { "transaction_charged_twice": 0.05, "apple_pay_or_google_pay": 0.01, "exchange_rate": 0.04, "declined_transfer": 0.06, "top_up_reverted": 0.08, "passcode_forgotten": 0.04, "disposable_card_limits": 0.05, "exchange_via_app": 0.04, "verify_top_up": 0.03, "request_refund": 0.04, "Refund_not_showing_up": 0.02, "wrong_exchange_rate_for_cash_withdrawal": 0.02, "supported_cards_and_currencies": 0.02, "card_linking": 0.03, "verify_my_identity": 0.05, "activate_my_card": 0.02, "pin_blocked": 0.03, "transfer_timing": 0.08, "edit_personal_details": 0.02, "virtual_card_not_working": 0.04, "verify_source_of_funds": 0.02, "reverted_card_payment?": 0.01, "topping_up_by_card": 0.02, "direct_debit_payment_not_recognised": 0.01, "pending_transfer": 0.02, "atm_support": 0.04, "pending_cash_withdrawal": 0.03, "order_physical_card": 0.02, "contactless_not_working": 0.01, "lost_or_stolen_phone": 0.02, "transfer_fee_charged": 0.01, "declined_cash_withdrawal": 0.01, "extra_charge_on_statement": 0.01 }, "highConfidenceShare": 0.78, "highConfidenceAccuracy": 0.41025641025641024, "latencyP50Ms": 198.6, "latencyP95Ms": 230.2, "meanInputTokens": null, "usdPer1000Calls": null, "reliability": [ { "from": 0.1, "to": 0.2, "n": 5, "accuracy": 0, "confidence": 0.136 }, { "from": 0.4, "to": 0.5, "n": 3, "accuracy": 0.333, "confidence": 0.472 }, { "from": 0.5, "to": 0.6, "n": 2, "accuracy": 0, "confidence": 0.57 }, { "from": 0.6, "to": 0.7, "n": 3, "accuracy": 0, "confidence": 0.636 }, { "from": 0.7, "to": 0.8, "n": 1, "accuracy": 1, "confidence": 0.718 }, { "from": 0.8, "to": 0.9, "n": 8, "accuracy": 0.125, "confidence": 0.865 }, { "from": 0.9, "to": 1, "n": 78, "accuracy": 0.41, "confidence": 0.992 } ] }, "clef-flash": { "n": 100, "failed": 0, "accuracy": 0.97, "accuracy95": [ 0.915479219195973, 0.9897456617765851 ], "meanConfidence": 0.9032909999999998, "ece": 0.06670900000000002, "predictionShare": { "card_payment_not_recognised": 0.01, "lost_or_stolen_phone": 0.01, "card_payment_wrong_exchange_rate": 0.02, "top_up_failed": 0.03, "apple_pay_or_google_pay": 0.01, "get_physical_card": 0.03, "pending_cash_withdrawal": 0.03, "compromised_card": 0.02, "disposable_card_limits": 0.01, "exchange_charge": 0.02, "verify_top_up": 0.03, "request_refund": 0.02, "Refund_not_showing_up": 0.05, "pending_top_up": 0.01, "country_support": 0.01, "card_linking": 0.03, "why_verify_identity": 0.02, "activate_my_card": 0.02, "change_pin": 0.04, "transfer_timing": 0.03, "transfer_not_received_by_recipient": 0.01, "get_disposable_virtual_card": 0.01, "direct_debit_payment_not_recognised": 0.02, "exchange_rate": 0.01, "getting_virtual_card": 0.04, "verify_source_of_funds": 0.02, "cancel_transfer": 0.01, "top_up_by_card_charge": 0.02, "card_not_working": 0.01, "verify_my_identity": 0.02, "balance_not_updated_after_bank_transfer": 0.03, "declined_transfer": 0.01, "pending_transfer": 0.02, "wrong_exchange_rate_for_cash_withdrawal": 0.02, "top_up_by_cash_or_cheque": 0.01, "top_up_reverted": 0.02, "exchange_via_app": 0.02, "order_physical_card": 0.01, "pin_blocked": 0.01, "atm_support": 0.02, "contactless_not_working": 0.01, "card_swallowed": 0.02, "failed_transfer": 0.01, "top_up_by_bank_transfer_charge": 0.02, "top_up_limits": 0.01, "balance_not_updated_after_cheque_or_cash_deposit": 0.01, "pending_card_payment": 0.01, "card_about_to_expire": 0.01, "transfer_fee_charged": 0.03, "transaction_charged_twice": 0.01, "declined_card_payment": 0.01, "unable_to_verify_identity": 0.01, "wrong_amount_of_cash_received": 0.01, "lost_or_stolen_card": 0.01, "declined_cash_withdrawal": 0.01, "card_delivery_estimate": 0.01, "extra_charge_on_statement": 0.01 }, "highConfidenceShare": 0.75, "highConfidenceAccuracy": 1, "latencyP50Ms": 881, "latencyP95Ms": 5274, "meanInputTokens": 1833.43, "usdPer1000Calls": 0.06967034, "reliability": [ { "from": 0.4, "to": 0.5, "n": 2, "accuracy": 0.5, "confidence": 0.481 }, { "from": 0.6, "to": 0.7, "n": 5, "accuracy": 0.8, "confidence": 0.661 }, { "from": 0.7, "to": 0.8, "n": 6, "accuracy": 0.833, "confidence": 0.757 }, { "from": 0.8, "to": 0.9, "n": 12, "accuracy": 1, "confidence": 0.872 }, { "from": 0.9, "to": 1, "n": 75, "accuracy": 1, "confidence": 0.948 } ] }, "clef": { "n": 100, "failed": 0, "accuracy": 0.96, "accuracy95": [ 0.9016278276739262, 0.984336949448152 ], "meanConfidence": 0.9105199999999998, "ece": 0.06873000000000008, "predictionShare": { "card_payment_not_recognised": 0.01, "lost_or_stolen_phone": 0.01, "card_payment_wrong_exchange_rate": 0.02, "top_up_failed": 0.03, "apple_pay_or_google_pay": 0.01, "get_physical_card": 0.03, "pending_cash_withdrawal": 0.03, "compromised_card": 0.02, "disposable_card_limits": 0.01, "exchange_charge": 0.02, "verify_top_up": 0.03, "request_refund": 0.02, "Refund_not_showing_up": 0.05, "pending_top_up": 0.01, "country_support": 0.01, "card_linking": 0.03, "why_verify_identity": 0.02, "activate_my_card": 0.02, "change_pin": 0.04, "transfer_timing": 0.03, "transfer_not_received_by_recipient": 0.01, "get_disposable_virtual_card": 0.01, "direct_debit_payment_not_recognised": 0.02, "exchange_rate": 0.01, "getting_virtual_card": 0.04, "verify_source_of_funds": 0.02, "cancel_transfer": 0.01, "top_up_by_card_charge": 0.02, "card_not_working": 0.01, "verify_my_identity": 0.02, "balance_not_updated_after_bank_transfer": 0.02, "declined_transfer": 0.01, "pending_transfer": 0.04, "wrong_exchange_rate_for_cash_withdrawal": 0.02, "top_up_by_cash_or_cheque": 0.01, "top_up_reverted": 0.02, "exchange_via_app": 0.02, "order_physical_card": 0.01, "pin_blocked": 0.01, "atm_support": 0.02, "contactless_not_working": 0.01, "card_swallowed": 0.02, "top_up_by_bank_transfer_charge": 0.02, "top_up_limits": 0.01, "balance_not_updated_after_cheque_or_cash_deposit": 0.01, "pending_card_payment": 0.01, "card_about_to_expire": 0.01, "transfer_fee_charged": 0.03, "transaction_charged_twice": 0.01, "declined_card_payment": 0.01, "unable_to_verify_identity": 0.01, "wrong_amount_of_cash_received": 0.01, "lost_or_stolen_card": 0.01, "declined_cash_withdrawal": 0.01, "card_delivery_estimate": 0.01, "extra_charge_on_statement": 0.01 }, "highConfidenceShare": 0.74, "highConfidenceAccuracy": 1, "latencyP50Ms": 579, "latencyP95Ms": 1182, "meanInputTokens": 1833.43, "usdPer1000Calls": 0.44002319999999995, "reliability": [ { "from": 0.4, "to": 0.5, "n": 2, "accuracy": 0, "confidence": 0.429 }, { "from": 0.5, "to": 0.6, "n": 2, "accuracy": 0.5, "confidence": 0.552 }, { "from": 0.6, "to": 0.7, "n": 3, "accuracy": 1, "confidence": 0.666 }, { "from": 0.7, "to": 0.8, "n": 6, "accuracy": 0.833, "confidence": 0.748 }, { "from": 0.8, "to": 0.9, "n": 13, "accuracy": 1, "confidence": 0.866 }, { "from": 0.9, "to": 1, "n": 74, "accuracy": 1, "confidence": 0.964 } ] }, "microsoft-decision-1": { "n": 100, "failed": 0, "accuracy": 0.91, "accuracy95": [ 0.8377362530335634, 0.9519280048361151 ], "meanConfidence": 0.8783710985438136, "ece": 0.04867585408443033, "predictionShare": { "card_payment_not_recognised": 0.01, "lost_or_stolen_phone": 0.01, "exchange_rate": 0.02, "top_up_failed": 0.03, "apple_pay_or_google_pay": 0.01, "get_physical_card": 0.03, "pending_cash_withdrawal": 0.03, "compromised_card": 0.02, "disposable_card_limits": 0.01, "exchange_charge": 0.02, "verify_top_up": 0.03, "request_refund": 0.02, "Refund_not_showing_up": 0.05, "pending_top_up": 0.01, "card_payment_wrong_exchange_rate": 0.01, "country_support": 0.01, "card_linking": 0.03, "why_verify_identity": 0.02, "activate_my_card": 0.02, "change_pin": 0.05, "transfer_timing": 0.03, "transfer_not_received_by_recipient": 0.01, "get_disposable_virtual_card": 0.01, "direct_debit_payment_not_recognised": 0.02, "getting_virtual_card": 0.04, "verify_source_of_funds": 0.02, "cancel_transfer": 0.01, "top_up_by_card_charge": 0.02, "card_not_working": 0.01, "verify_my_identity": 0.02, "balance_not_updated_after_bank_transfer": 0.02, "declined_transfer": 0.01, "pending_transfer": 0.04, "cash_withdrawal_charge": 0.01, "top_up_by_cash_or_cheque": 0.01, "top_up_reverted": 0.02, "exchange_via_app": 0.01, "pin_blocked": 0.01, "atm_support": 0.02, "contactless_not_working": 0.01, "card_swallowed": 0.02, "transfer_into_account": 0.01, "top_up_limits": 0.01, "balance_not_updated_after_cheque_or_cash_deposit": 0.01, "pending_card_payment": 0.01, "card_about_to_expire": 0.01, "transfer_fee_charged": 0.03, "transaction_charged_twice": 0.01, "wrong_exchange_rate_for_cash_withdrawal": 0.01, "fiat_currency_support": 0.01, "declined_card_payment": 0.01, "unable_to_verify_identity": 0.01, "wrong_amount_of_cash_received": 0.01, "lost_or_stolen_card": 0.01, "declined_cash_withdrawal": 0.01, "top_up_by_bank_transfer_charge": 0.01, "card_delivery_estimate": 0.01, "extra_charge_on_statement": 0.01 }, "highConfidenceShare": 0.66, "highConfidenceAccuracy": 0.9848484848484849, "latencyP50Ms": 735, "latencyP95Ms": 1409, "meanInputTokens": 772.2, "usdPer1000Calls": 0.0324324, "reliability": [ { "from": 0.2, "to": 0.3, "n": 1, "accuracy": 0, "confidence": 0.227 }, { "from": 0.4, "to": 0.5, "n": 4, "accuracy": 0.75, "confidence": 0.453 }, { "from": 0.5, "to": 0.6, "n": 1, "accuracy": 0, "confidence": 0.523 }, { "from": 0.6, "to": 0.7, "n": 11, "accuracy": 0.727, "confidence": 0.656 }, { "from": 0.7, "to": 0.8, "n": 4, "accuracy": 1, "confidence": 0.763 }, { "from": 0.8, "to": 0.9, "n": 13, "accuracy": 0.846, "confidence": 0.854 }, { "from": 0.9, "to": 1, "n": 66, "accuracy": 0.985, "confidence": 0.968 } ] } }, "ag_news": { "name": "AG News topic (4 options)", "source": "https://huggingface.co/datasets/fancyzhx/ag_news", "license": "Non-commercial research use (see dataset card)", "split": "test", "question": { "type": "choice", "options": 4 }, "jev": { "n": 98, "failed": 2, "accuracy": 0.8367346938775511, "accuracy95": [ 0.751132386961264, 0.8969328437499582 ], "meanConfidence": 0.9732653061224489, "ece": 0.15244897959183662, "predictionShare": { "World": 0.316, "Sci/Tech": 0.204, "Business": 0.276, "Sports": 0.204 }, "highConfidenceShare": 0.9183673469387755, "highConfidenceAccuracy": 0.8555555555555555, "latencyP50Ms": 1741, "latencyP95Ms": 3795, "meanInputTokens": 431.4183673469388, "usdPer1000Calls": 0.01811957142857143, "reliability": [ { "from": 0.5, "to": 0.6, "n": 1, "accuracy": 1, "confidence": 0.55 }, { "from": 0.6, "to": 0.7, "n": 1, "accuracy": 1, "confidence": 0.67 }, { "from": 0.7, "to": 0.8, "n": 4, "accuracy": 0.75, "confidence": 0.753 }, { "from": 0.8, "to": 0.9, "n": 2, "accuracy": 0, "confidence": 0.85 }, { "from": 0.9, "to": 1, "n": 90, "accuracy": 0.856, "confidence": 0.994 } ] }, "laya": { "n": 100, "failed": 0, "accuracy": 0.92, "accuracy95": [ 0.8500173312059693, 0.9589070305141891 ], "meanConfidence": 0.8780660000000003, "ece": 0.0419339999999999, "predictionShare": { "World": 0.28, "Sci/Tech": 0.29, "Business": 0.22, "Sports": 0.21 }, "highConfidenceShare": 0.63, "highConfidenceAccuracy": 0.9682539682539683, "latencyP50Ms": 80.7, "latencyP95Ms": 158.2, "meanInputTokens": null, "usdPer1000Calls": null, "reliability": [ { "from": 0.4, "to": 0.5, "n": 4, "accuracy": 0.75, "confidence": 0.466 }, { "from": 0.5, "to": 0.6, "n": 3, "accuracy": 0.667, "confidence": 0.549 }, { "from": 0.6, "to": 0.7, "n": 4, "accuracy": 1, "confidence": 0.652 }, { "from": 0.7, "to": 0.8, "n": 6, "accuracy": 0.833, "confidence": 0.75 }, { "from": 0.8, "to": 0.9, "n": 20, "accuracy": 0.85, "confidence": 0.85 }, { "from": 0.9, "to": 1, "n": 63, "accuracy": 0.968, "confidence": 0.956 } ] }, "clef-flash": { "n": 100, "failed": 0, "accuracy": 0.91, "accuracy95": [ 0.8377362530335634, 0.9519280048361151 ], "meanConfidence": 0.8884059999999998, "ece": 0.04074799999999989, "predictionShare": { "World": 0.31, "Sci/Tech": 0.28, "Business": 0.2, "Sports": 0.21 }, "highConfidenceShare": 0.64, "highConfidenceAccuracy": 0.9375, "latencyP50Ms": 546, "latencyP95Ms": 2074, "meanInputTokens": 257.15, "usdPer1000Calls": 0.0097717, "reliability": [ { "from": 0.5, "to": 0.6, "n": 3, "accuracy": 0.667, "confidence": 0.538 }, { "from": 0.6, "to": 0.7, "n": 4, "accuracy": 0.5, "confidence": 0.646 }, { "from": 0.7, "to": 0.8, "n": 7, "accuracy": 1, "confidence": 0.751 }, { "from": 0.8, "to": 0.9, "n": 22, "accuracy": 0.909, "confidence": 0.864 }, { "from": 0.9, "to": 1, "n": 64, "accuracy": 0.938, "confidence": 0.943 } ] }, "clef": { "n": 100, "failed": 0, "accuracy": 0.91, "accuracy95": [ 0.8377362530335634, 0.9519280048361151 ], "meanConfidence": 0.9254290000000001, "ece": 0.021467000000000215, "predictionShare": { "World": 0.31, "Sci/Tech": 0.29, "Business": 0.19, "Sports": 0.21 }, "highConfidenceShare": 0.82, "highConfidenceAccuracy": 0.9512195121951219, "latencyP50Ms": 585, "latencyP95Ms": 1146, "meanInputTokens": 257.15, "usdPer1000Calls": 0.06171599999999999, "reliability": [ { "from": 0.3, "to": 0.4, "n": 1, "accuracy": 0, "confidence": 0.368 }, { "from": 0.5, "to": 0.6, "n": 2, "accuracy": 0.5, "confidence": 0.588 }, { "from": 0.6, "to": 0.7, "n": 3, "accuracy": 0.667, "confidence": 0.663 }, { "from": 0.7, "to": 0.8, "n": 5, "accuracy": 0.8, "confidence": 0.742 }, { "from": 0.8, "to": 0.9, "n": 7, "accuracy": 0.857, "confidence": 0.857 }, { "from": 0.9, "to": 1, "n": 82, "accuracy": 0.951, "confidence": 0.967 } ] }, "microsoft-decision-1": { "n": 100, "failed": 0, "accuracy": 0.83, "accuracy95": [ 0.7445181876635703, 0.8910652394022684 ], "meanConfidence": 0.9531907920473276, "ece": 0.13912869082649218, "predictionShare": { "World": 0.31, "Sci/Tech": 0.19, "Business": 0.29, "Sports": 0.21 }, "highConfidenceShare": 0.86, "highConfidenceAccuracy": 0.872093023255814, "latencyP50Ms": 699, "latencyP95Ms": 1359, "meanInputTokens": 139.44, "usdPer1000Calls": 0.00585648, "reliability": [ { "from": 0.4, "to": 0.5, "n": 1, "accuracy": 0, "confidence": 0.499 }, { "from": 0.5, "to": 0.6, "n": 2, "accuracy": 0.5, "confidence": 0.561 }, { "from": 0.7, "to": 0.8, "n": 7, "accuracy": 0.857, "confidence": 0.743 }, { "from": 0.8, "to": 0.9, "n": 4, "accuracy": 0.25, "confidence": 0.851 }, { "from": 0.9, "to": 1, "n": 86, "accuracy": 0.872, "confidence": 0.989 } ] } }, "emotion": { "name": "Emotion (6 options)", "source": "https://huggingface.co/datasets/dair-ai/emotion", "license": "Educational and research use (see dataset card)", "split": "test", "question": { "type": "choice", "options": 6 }, "jev": { "n": 49, "failed": 0, "accuracy": 0.6530612244897959, "accuracy95": [ 0.5131093521943979, 0.7707579039068555 ], "meanConfidence": 0.8497959183673471, "ece": 0.22530612244897968, "predictionShare": { "sadness": 0.408, "joy": 0.245, "fear": 0.204, "anger": 0.082, "surprise": 0.041, "love": 0.02 }, "highConfidenceShare": 0.5102040816326531, "highConfidenceAccuracy": 0.76, "latencyP50Ms": 1778, "latencyP95Ms": 4225, "meanInputTokens": 445.16326530612247, "usdPer1000Calls": 0.018696857142857144, "reliability": [ { "from": 0.3, "to": 0.4, "n": 1, "accuracy": 1, "confidence": 0.37 }, { "from": 0.4, "to": 0.5, "n": 2, "accuracy": 0.5, "confidence": 0.465 }, { "from": 0.5, "to": 0.6, "n": 3, "accuracy": 0.333, "confidence": 0.567 }, { "from": 0.6, "to": 0.7, "n": 2, "accuracy": 0.5, "confidence": 0.63 }, { "from": 0.7, "to": 0.8, "n": 5, "accuracy": 0.6, "confidence": 0.742 }, { "from": 0.8, "to": 0.9, "n": 11, "accuracy": 0.545, "confidence": 0.84 }, { "from": 0.9, "to": 1, "n": 25, "accuracy": 0.76, "confidence": 0.977 } ] }, "laya": { "n": 49, "failed": 0, "accuracy": 0.5714285714285714, "accuracy95": [ 0.4327024751701555, 0.6997689110104294 ], "meanConfidence": 0.8224530612244897, "ece": 0.3004408163265305, "predictionShare": { "sadness": 0.367, "joy": 0.367, "anger": 0.061, "fear": 0.143, "love": 0.061 }, "highConfidenceShare": 0.5102040816326531, "highConfidenceAccuracy": 0.64, "latencyP50Ms": 81.8, "latencyP95Ms": 108.9, "meanInputTokens": null, "usdPer1000Calls": null, "reliability": [ { "from": 0.3, "to": 0.4, "n": 1, "accuracy": 0, "confidence": 0.391 }, { "from": 0.4, "to": 0.5, "n": 4, "accuracy": 0, "confidence": 0.468 }, { "from": 0.5, "to": 0.6, "n": 3, "accuracy": 0.667, "confidence": 0.53 }, { "from": 0.6, "to": 0.7, "n": 5, "accuracy": 0.4, "confidence": 0.641 }, { "from": 0.7, "to": 0.8, "n": 7, "accuracy": 0.857, "confidence": 0.743 }, { "from": 0.8, "to": 0.9, "n": 4, "accuracy": 0.5, "confidence": 0.873 }, { "from": 0.9, "to": 1, "n": 25, "accuracy": 0.64, "confidence": 0.982 } ] }, "clef-flash": { "n": 49, "failed": 0, "accuracy": 0.5714285714285714, "accuracy95": [ 0.4327024751701555, 0.6997689110104294 ], "meanConfidence": 0.7794857142857141, "ece": 0.23355102040816328, "predictionShare": { "sadness": 0.449, "joy": 0.224, "fear": 0.102, "anger": 0.122, "surprise": 0.041, "love": 0.061 }, "highConfidenceShare": 0.2857142857142857, "highConfidenceAccuracy": 0.7857142857142857, "latencyP50Ms": 510, "latencyP95Ms": 2235, "meanInputTokens": 276.83673469387753, "usdPer1000Calls": 0.010519795918367344, "reliability": [ { "from": 0.3, "to": 0.4, "n": 1, "accuracy": 1, "confidence": 0.375 }, { "from": 0.4, "to": 0.5, "n": 3, "accuracy": 0, "confidence": 0.475 }, { "from": 0.5, "to": 0.6, "n": 7, "accuracy": 0.286, "confidence": 0.552 }, { "from": 0.6, "to": 0.7, "n": 4, "accuracy": 0.25, "confidence": 0.646 }, { "from": 0.7, "to": 0.8, "n": 5, "accuracy": 0.6, "confidence": 0.742 }, { "from": 0.8, "to": 0.9, "n": 15, "accuracy": 0.667, "confidence": 0.864 }, { "from": 0.9, "to": 1, "n": 14, "accuracy": 0.786, "confidence": 0.948 } ] }, "clef": { "n": 49, "failed": 0, "accuracy": 0.5510204081632653, "accuracy95": [ 0.41314866799678107, 0.6814737507036366 ], "meanConfidence": 0.769342857142857, "ece": 0.21832244897959188, "predictionShare": { "sadness": 0.449, "joy": 0.224, "anger": 0.143, "surprise": 0.041, "fear": 0.102, "love": 0.041 }, "highConfidenceShare": 0.3469387755102041, "highConfidenceAccuracy": 0.7647058823529411, "latencyP50Ms": 572, "latencyP95Ms": 1033, "meanInputTokens": 276.83673469387753, "usdPer1000Calls": 0.06644081632653061, "reliability": [ { "from": 0.3, "to": 0.4, "n": 1, "accuracy": 0, "confidence": 0.35 }, { "from": 0.4, "to": 0.5, "n": 5, "accuracy": 0, "confidence": 0.435 }, { "from": 0.5, "to": 0.6, "n": 6, "accuracy": 0.167, "confidence": 0.55 }, { "from": 0.6, "to": 0.7, "n": 4, "accuracy": 0.25, "confidence": 0.62 }, { "from": 0.7, "to": 0.8, "n": 5, "accuracy": 0.6, "confidence": 0.738 }, { "from": 0.8, "to": 0.9, "n": 11, "accuracy": 0.818, "confidence": 0.868 }, { "from": 0.9, "to": 1, "n": 17, "accuracy": 0.765, "confidence": 0.95 } ] }, "microsoft-decision-1": { "n": 49, "failed": 0, "accuracy": 0.6326530612244898, "accuracy95": [ 0.49266811386889864, 0.7533501747521878 ], "meanConfidence": 0.7424982259868806, "ece": 0.21111413018114972, "predictionShare": { "sadness": 0.347, "joy": 0.245, "fear": 0.184, "anger": 0.102, "surprise": 0.061, "love": 0.061 }, "highConfidenceShare": 0.24489795918367346, "highConfidenceAccuracy": 0.75, "latencyP50Ms": 720, "latencyP95Ms": 1347, "meanInputTokens": 135.83673469387756, "usdPer1000Calls": 0.005705142857142857, "reliability": [ { "from": 0.2, "to": 0.3, "n": 1, "accuracy": 1, "confidence": 0.257 }, { "from": 0.3, "to": 0.4, "n": 3, "accuracy": 0, "confidence": 0.355 }, { "from": 0.4, "to": 0.5, "n": 3, "accuracy": 1, "confidence": 0.434 }, { "from": 0.5, "to": 0.6, "n": 5, "accuracy": 0.2, "confidence": 0.536 }, { "from": 0.6, "to": 0.7, "n": 3, "accuracy": 0.667, "confidence": 0.653 }, { "from": 0.7, "to": 0.8, "n": 12, "accuracy": 0.583, "confidence": 0.756 }, { "from": 0.8, "to": 0.9, "n": 10, "accuracy": 0.8, "confidence": 0.863 }, { "from": 0.9, "to": 1, "n": 12, "accuracy": 0.75, "confidence": 0.951 } ] } }, "sst2": { "name": "SST-2 sentiment (yes/no)", "source": "https://huggingface.co/datasets/stanfordnlp/sst2", "license": "See dataset card", "split": "validation", "question": { "type": "boolean" }, "laya": { "n": 200, "failed": 0, "accuracy": 0.48, "accuracy95": [ 0.4117904407948983, 0.5489633994418346 ], "meanConfidence": 0.9725279999999994, "ece": 0.5013979999999996, "predictionShare": { "negative": 0.975, "positive": 0.025 }, "highConfidenceShare": 0.92, "highConfidenceAccuracy": 0.4945652173913043, "latencyP50Ms": 116.8, "latencyP95Ms": 600.7, "meanInputTokens": null, "usdPer1000Calls": null, "reliability": [ { "from": 0.5, "to": 0.6, "n": 2, "accuracy": 1, "confidence": 0.557 }, { "from": 0.6, "to": 0.7, "n": 4, "accuracy": 0.25, "confidence": 0.65 }, { "from": 0.7, "to": 0.8, "n": 4, "accuracy": 0, "confidence": 0.736 }, { "from": 0.8, "to": 0.9, "n": 6, "accuracy": 0.333, "confidence": 0.849 }, { "from": 0.9, "to": 1, "n": 184, "accuracy": 0.495, "confidence": 0.993 } ] }, "clef-flash": { "n": 200, "failed": 0, "accuracy": 0.96, "accuracy95": [ 0.9230670617058012, 0.9795946128493436 ], "meanConfidence": 0.9524995000000002, "ece": 0.028740500000000297, "predictionShare": { "negative": 0.475, "positive": 0.525 }, "highConfidenceShare": 0.94, "highConfidenceAccuracy": 0.9840425531914894, "latencyP50Ms": 397, "latencyP95Ms": 1459, "meanInputTokens": 169.615, "usdPer1000Calls": 0.006445370000000001, "reliability": [ { "from": 0.5, "to": 0.6, "n": 2, "accuracy": 0.5, "confidence": 0.527 }, { "from": 0.6, "to": 0.7, "n": 1, "accuracy": 0, "confidence": 0.682 }, { "from": 0.7, "to": 0.8, "n": 3, "accuracy": 0.667, "confidence": 0.749 }, { "from": 0.8, "to": 0.9, "n": 6, "accuracy": 0.667, "confidence": 0.857 }, { "from": 0.9, "to": 1, "n": 188, "accuracy": 0.984, "confidence": 0.965 } ] }, "clef": { "n": 200, "failed": 0, "accuracy": 0.965, "accuracy95": [ 0.9295285391333187, 0.9829446753626429 ], "meanConfidence": 0.943518, "ece": 0.033638000000000154, "predictionShare": { "negative": 0.47, "positive": 0.53 }, "highConfidenceShare": 0.87, "highConfidenceAccuracy": 0.9827586206896551, "latencyP50Ms": 530, "latencyP95Ms": 1285, "meanInputTokens": 169.615, "usdPer1000Calls": 0.0407076, "reliability": [ { "from": 0.5, "to": 0.6, "n": 4, "accuracy": 0.75, "confidence": 0.539 }, { "from": 0.7, "to": 0.8, "n": 7, "accuracy": 0.571, "confidence": 0.745 }, { "from": 0.8, "to": 0.9, "n": 15, "accuracy": 1, "confidence": 0.86 }, { "from": 0.9, "to": 1, "n": 174, "accuracy": 0.983, "confidence": 0.968 } ] }, "microsoft-decision-1": { "n": 200, "failed": 0, "accuracy": 0.97, "accuracy95": [ 0.9361048802675267, 0.9861798741692518 ], "meanConfidence": 0.9745210820515401, "ece": 0.009497933326935718, "predictionShare": { "negative": 0.465, "positive": 0.535 }, "highConfidenceShare": 0.93, "highConfidenceAccuracy": 0.9838709677419355, "latencyP50Ms": 716, "latencyP95Ms": 1480, "meanInputTokens": 77.615, "usdPer1000Calls": 0.00325983, "reliability": [ { "from": 0.5, "to": 0.6, "n": 1, "accuracy": 1, "confidence": 0.531 }, { "from": 0.6, "to": 0.7, "n": 1, "accuracy": 0, "confidence": 0.651 }, { "from": 0.7, "to": 0.8, "n": 4, "accuracy": 0.75, "confidence": 0.759 }, { "from": 0.8, "to": 0.9, "n": 8, "accuracy": 0.875, "confidence": 0.871 }, { "from": 0.9, "to": 1, "n": 186, "accuracy": 0.984, "confidence": 0.988 } ] } }, "sst2_choice": { "name": "SST-2 sentiment as a choice (2 options)", "source": "https://huggingface.co/datasets/stanfordnlp/sst2", "license": "See dataset card", "split": "validation", "question": { "type": "choice", "options": 2 }, "laya": { "n": 200, "failed": 0, "accuracy": 0.93, "accuracy95": [ 0.8859442178331628, 0.9578482170770813 ], "meanConfidence": 0.8872254999999997, "ece": 0.08814550000000002, "predictionShare": { "negative": 0.435, "positive": 0.565 }, "highConfidenceShare": 0.67, "highConfidenceAccuracy": 0.9850746268656716, "latencyP50Ms": 55.6, "latencyP95Ms": 69.5, "meanInputTokens": null, "usdPer1000Calls": null, "reliability": [ { "from": 0.5, "to": 0.6, "n": 12, "accuracy": 0.167, "confidence": 0.545 }, { "from": 0.6, "to": 0.7, "n": 12, "accuracy": 0.917, "confidence": 0.652 }, { "from": 0.7, "to": 0.8, "n": 13, "accuracy": 1, "confidence": 0.751 }, { "from": 0.8, "to": 0.9, "n": 29, "accuracy": 0.966, "confidence": 0.859 }, { "from": 0.9, "to": 1, "n": 134, "accuracy": 0.985, "confidence": 0.958 } ] }, "clef-flash": { "n": 200, "failed": 0, "accuracy": 0.97, "accuracy95": [ 0.9361048802675267, 0.9861798741692518 ], "meanConfidence": 0.9615554999999997, "ece": 0.02088950000000042, "predictionShare": { "negative": 0.455, "positive": 0.545 }, "highConfidenceShare": 0.945, "highConfidenceAccuracy": 0.9841269841269841, "latencyP50Ms": 423, "latencyP95Ms": 1058, "meanInputTokens": 167.615, "usdPer1000Calls": 0.00636937, "reliability": [ { "from": 0.5, "to": 0.6, "n": 2, "accuracy": 1, "confidence": 0.515 }, { "from": 0.6, "to": 0.7, "n": 3, "accuracy": 0.667, "confidence": 0.609 }, { "from": 0.7, "to": 0.8, "n": 1, "accuracy": 1, "confidence": 0.778 }, { "from": 0.8, "to": 0.9, "n": 5, "accuracy": 0.6, "confidence": 0.849 }, { "from": 0.9, "to": 1, "n": 189, "accuracy": 0.984, "confidence": 0.976 } ] }, "clef": { "n": 200, "failed": 0, "accuracy": 0.965, "accuracy95": [ 0.9295285391333187, 0.9829446753626429 ], "meanConfidence": 0.9609354999999996, "ece": 0.007686500000000072, "predictionShare": { "negative": 0.47, "positive": 0.53 }, "highConfidenceShare": 0.915, "highConfidenceAccuracy": 0.9836065573770492, "latencyP50Ms": 562, "latencyP95Ms": 1256, "meanInputTokens": 167.615, "usdPer1000Calls": 0.0402276, "reliability": [ { "from": 0.6, "to": 0.7, "n": 3, "accuracy": 0.667, "confidence": 0.668 }, { "from": 0.7, "to": 0.8, "n": 10, "accuracy": 0.7, "confidence": 0.736 }, { "from": 0.8, "to": 0.9, "n": 4, "accuracy": 1, "confidence": 0.882 }, { "from": 0.9, "to": 1, "n": 183, "accuracy": 0.984, "confidence": 0.98 } ] }, "microsoft-decision-1": { "n": 200, "failed": 0, "accuracy": 0.97, "accuracy95": [ 0.9361048802675267, 0.9861798741692518 ], "meanConfidence": 0.9681180530273291, "ece": 0.007993319366377708, "predictionShare": { "negative": 0.465, "positive": 0.535 }, "highConfidenceShare": 0.895, "highConfidenceAccuracy": 0.9888268156424581, "latencyP50Ms": 688, "latencyP95Ms": 1381, "meanInputTokens": 75.615, "usdPer1000Calls": 0.00317583, "reliability": [ { "from": 0.5, "to": 0.6, "n": 2, "accuracy": 0.5, "confidence": 0.577 }, { "from": 0.6, "to": 0.7, "n": 2, "accuracy": 1, "confidence": 0.651 }, { "from": 0.7, "to": 0.8, "n": 7, "accuracy": 0.714, "confidence": 0.764 }, { "from": 0.8, "to": 0.9, "n": 10, "accuracy": 0.9, "confidence": 0.871 }, { "from": 0.9, "to": 1, "n": 179, "accuracy": 0.989, "confidence": 0.989 } ] } } } }