{
  "generatedAt": "2026-09-26T12:59:49.133Z",
  "seed": 20260926,
  "jevModel": "typesafe-ai/jev",
  "layaVersion": "0.3.20",
  "layaDevice": "mps",
  "tasks": {
    "banking77": {
      "name": "Banking77 intent (77 options)",
      "source": "https://github.com/PolyAI-LDN/task-specific-datasets",
      "license": "CC BY 4.0",
      "split": "test",
      "question": {
        "type": "choice",
        "options": 77
      },
      "jev": {
        "n": 100,
        "failed": 0,
        "accuracy": 0.79,
        "accuracy95": [
          0.7001985069803768,
          0.8583445046835422
        ],
        "meanConfidence": 0.8933999999999997,
        "ece": 0.10339999999999985,
        "predictionShare": {
          "card_payment_not_recognised": 0.02,
          "lost_or_stolen_phone": 0.01,
          "exchange_rate": 0.02,
          "top_up_failed": 0.04,
          "apple_pay_or_google_pay": 0.01,
          "passcode_forgotten": 0.01,
          "pending_cash_withdrawal": 0.03,
          "compromised_card": 0.02,
          "disposable_card_limits": 0.01,
          "exchange_charge": 0.02,
          "verify_top_up": 0.03,
          "request_refund": 0.02,
          "Refund_not_showing_up": 0.05,
          "pending_top_up": 0.01,
          "card_payment_wrong_exchange_rate": 0.01,
          "country_support": 0.01,
          "card_linking": 0.03,
          "unable_to_verify_identity": 0.03,
          "activate_my_card": 0.02,
          "change_pin": 0.06,
          "transfer_timing": 0.04,
          "transfer_not_received_by_recipient": 0.01,
          "get_disposable_virtual_card": 0.01,
          "getting_virtual_card": 0.04,
          "verify_source_of_funds": 0.02,
          "reverted_card_payment?": 0.01,
          "top_up_by_card_charge": 0.02,
          "card_not_working": 0.01,
          "verify_my_identity": 0.02,
          "direct_debit_payment_not_recognised": 0.01,
          "balance_not_updated_after_bank_transfer": 0.01,
          "declined_card_payment": 0.02,
          "pending_transfer": 0.05,
          "cash_withdrawal_charge": 0.01,
          "top_up_by_cash_or_cheque": 0.01,
          "exchange_via_app": 0.01,
          "get_physical_card": 0.01,
          "top_up_reverted": 0.01,
          "pin_blocked": 0.01,
          "atm_support": 0.03,
          "contactless_not_working": 0.01,
          "card_swallowed": 0.02,
          "transfer_into_account": 0.02,
          "top_up_limits": 0.01,
          "balance_not_updated_after_cheque_or_cash_deposit": 0.01,
          "card_about_to_expire": 0.01,
          "transfer_fee_charged": 0.03,
          "transaction_charged_twice": 0.01,
          "wrong_exchange_rate_for_cash_withdrawal": 0.01,
          "fiat_currency_support": 0.01,
          "wrong_amount_of_cash_received": 0.01,
          "lost_or_stolen_card": 0.01,
          "card_delivery_estimate": 0.01,
          "extra_charge_on_statement": 0.01
        },
        "highConfidenceShare": 0.71,
        "highConfidenceAccuracy": 0.9295774647887324,
        "latencyP50Ms": 1680,
        "latencyP95Ms": 3915,
        "meanInputTokens": 1687.44,
        "usdPer1000Calls": 0.07087248000000002,
        "reliability": [
          {
            "from": 0.4,
            "to": 0.5,
            "n": 3,
            "accuracy": 0.333,
            "confidence": 0.467
          },
          {
            "from": 0.5,
            "to": 0.6,
            "n": 8,
            "accuracy": 0.375,
            "confidence": 0.541
          },
          {
            "from": 0.6,
            "to": 0.7,
            "n": 3,
            "accuracy": 0,
            "confidence": 0.637
          },
          {
            "from": 0.7,
            "to": 0.8,
            "n": 7,
            "accuracy": 0.714,
            "confidence": 0.747
          },
          {
            "from": 0.8,
            "to": 0.9,
            "n": 8,
            "accuracy": 0.5,
            "confidence": 0.848
          },
          {
            "from": 0.9,
            "to": 1,
            "n": 71,
            "accuracy": 0.93,
            "confidence": 0.982
          }
        ]
      },
      "laya": {
        "n": 100,
        "failed": 0,
        "accuracy": 0.35,
        "accuracy95": [
          0.2636410465347802,
          0.4474573957080203
        ],
        "meanConfidence": 0.9012379999999998,
        "ece": 0.5568779999999999,
        "predictionShare": {
          "transaction_charged_twice": 0.05,
          "apple_pay_or_google_pay": 0.01,
          "exchange_rate": 0.04,
          "declined_transfer": 0.06,
          "top_up_reverted": 0.08,
          "passcode_forgotten": 0.04,
          "disposable_card_limits": 0.05,
          "exchange_via_app": 0.04,
          "verify_top_up": 0.03,
          "request_refund": 0.04,
          "Refund_not_showing_up": 0.02,
          "wrong_exchange_rate_for_cash_withdrawal": 0.02,
          "supported_cards_and_currencies": 0.02,
          "card_linking": 0.03,
          "verify_my_identity": 0.05,
          "activate_my_card": 0.02,
          "pin_blocked": 0.03,
          "transfer_timing": 0.08,
          "edit_personal_details": 0.02,
          "virtual_card_not_working": 0.04,
          "verify_source_of_funds": 0.02,
          "reverted_card_payment?": 0.01,
          "topping_up_by_card": 0.02,
          "direct_debit_payment_not_recognised": 0.01,
          "pending_transfer": 0.02,
          "atm_support": 0.04,
          "pending_cash_withdrawal": 0.03,
          "order_physical_card": 0.02,
          "contactless_not_working": 0.01,
          "lost_or_stolen_phone": 0.02,
          "transfer_fee_charged": 0.01,
          "declined_cash_withdrawal": 0.01,
          "extra_charge_on_statement": 0.01
        },
        "highConfidenceShare": 0.78,
        "highConfidenceAccuracy": 0.41025641025641024,
        "latencyP50Ms": 198.6,
        "latencyP95Ms": 230.2,
        "meanInputTokens": null,
        "usdPer1000Calls": null,
        "reliability": [
          {
            "from": 0.1,
            "to": 0.2,
            "n": 5,
            "accuracy": 0,
            "confidence": 0.136
          },
          {
            "from": 0.4,
            "to": 0.5,
            "n": 3,
            "accuracy": 0.333,
            "confidence": 0.472
          },
          {
            "from": 0.5,
            "to": 0.6,
            "n": 2,
            "accuracy": 0,
            "confidence": 0.57
          },
          {
            "from": 0.6,
            "to": 0.7,
            "n": 3,
            "accuracy": 0,
            "confidence": 0.636
          },
          {
            "from": 0.7,
            "to": 0.8,
            "n": 1,
            "accuracy": 1,
            "confidence": 0.718
          },
          {
            "from": 0.8,
            "to": 0.9,
            "n": 8,
            "accuracy": 0.125,
            "confidence": 0.865
          },
          {
            "from": 0.9,
            "to": 1,
            "n": 78,
            "accuracy": 0.41,
            "confidence": 0.992
          }
        ]
      }
    },
    "ag_news": {
      "name": "AG News topic (4 options)",
      "source": "https://huggingface.co/datasets/fancyzhx/ag_news",
      "license": "Non-commercial research use (see dataset card)",
      "split": "test",
      "question": {
        "type": "choice",
        "options": 4
      },
      "jev": {
        "n": 32,
        "failed": 1,
        "accuracy": 0.78125,
        "accuracy95": [
          0.6124466920955516,
          0.8897630306846815
        ],
        "meanConfidence": 0.9731249999999999,
        "ece": 0.20999999999999994,
        "predictionShare": {
          "World": 0.375,
          "Sci/Tech": 0.188,
          "Business": 0.281,
          "Sports": 0.156
        },
        "highConfidenceShare": 0.90625,
        "highConfidenceAccuracy": 0.8275862068965517,
        "latencyP50Ms": 1760,
        "latencyP95Ms": 3839,
        "meanInputTokens": 427.3125,
        "usdPer1000Calls": 0.017947125,
        "reliability": [
          {
            "from": 0.7,
            "to": 0.8,
            "n": 1,
            "accuracy": 1,
            "confidence": 0.71
          },
          {
            "from": 0.8,
            "to": 0.9,
            "n": 2,
            "accuracy": 0,
            "confidence": 0.85
          },
          {
            "from": 0.9,
            "to": 1,
            "n": 29,
            "accuracy": 0.828,
            "confidence": 0.991
          }
        ]
      },
      "laya": {
        "n": 33,
        "failed": 0,
        "accuracy": 0.9696969696969697,
        "accuracy95": [
          0.8468093832683193,
          0.9946307279103482
        ],
        "meanConfidence": 0.8924636363636365,
        "ece": 0.07723333333333339,
        "predictionShare": {
          "World": 0.333,
          "Sci/Tech": 0.333,
          "Business": 0.182,
          "Sports": 0.152
        },
        "highConfidenceShare": 0.6666666666666666,
        "highConfidenceAccuracy": 0.9545454545454546,
        "latencyP50Ms": 79.5,
        "latencyP95Ms": 182.6,
        "meanInputTokens": null,
        "usdPer1000Calls": null,
        "reliability": [
          {
            "from": 0.5,
            "to": 0.6,
            "n": 1,
            "accuracy": 1,
            "confidence": 0.546
          },
          {
            "from": 0.6,
            "to": 0.7,
            "n": 1,
            "accuracy": 1,
            "confidence": 0.602
          },
          {
            "from": 0.7,
            "to": 0.8,
            "n": 3,
            "accuracy": 1,
            "confidence": 0.742
          },
          {
            "from": 0.8,
            "to": 0.9,
            "n": 6,
            "accuracy": 1,
            "confidence": 0.865
          },
          {
            "from": 0.9,
            "to": 1,
            "n": 22,
            "accuracy": 0.955,
            "confidence": 0.949
          }
        ]
      }
    },
    "emotion": {
      "name": "Emotion (6 options)",
      "source": "https://huggingface.co/datasets/dair-ai/emotion",
      "license": "Educational and research use (see dataset card)",
      "split": "test",
      "question": {
        "type": "choice",
        "options": 6
      },
      "laya": {
        "n": 200,
        "failed": 0,
        "accuracy": 0.555,
        "accuracy95": [
          0.4857317563236392,
          0.6221951830253456
        ],
        "meanConfidence": 0.8204804999999996,
        "ece": 0.2654804999999999,
        "predictionShare": {
          "sadness": 0.41,
          "joy": 0.305,
          "anger": 0.085,
          "fear": 0.125,
          "love": 0.065,
          "surprise": 0.01
        },
        "highConfidenceShare": 0.5,
        "highConfidenceAccuracy": 0.7,
        "latencyP50Ms": 89.4,
        "latencyP95Ms": 129.7,
        "meanInputTokens": null,
        "usdPer1000Calls": null,
        "reliability": [
          {
            "from": 0.2,
            "to": 0.3,
            "n": 2,
            "accuracy": 0,
            "confidence": 0.278
          },
          {
            "from": 0.3,
            "to": 0.4,
            "n": 7,
            "accuracy": 0.143,
            "confidence": 0.373
          },
          {
            "from": 0.4,
            "to": 0.5,
            "n": 9,
            "accuracy": 0.222,
            "confidence": 0.469
          },
          {
            "from": 0.5,
            "to": 0.6,
            "n": 17,
            "accuracy": 0.471,
            "confidence": 0.54
          },
          {
            "from": 0.6,
            "to": 0.7,
            "n": 15,
            "accuracy": 0.6,
            "confidence": 0.643
          },
          {
            "from": 0.7,
            "to": 0.8,
            "n": 28,
            "accuracy": 0.5,
            "confidence": 0.748
          },
          {
            "from": 0.8,
            "to": 0.9,
            "n": 22,
            "accuracy": 0.318,
            "confidence": 0.865
          },
          {
            "from": 0.9,
            "to": 1,
            "n": 100,
            "accuracy": 0.7,
            "confidence": 0.979
          }
        ]
      }
    },
    "sst2": {
      "name": "SST-2 sentiment (yes/no)",
      "source": "https://huggingface.co/datasets/stanfordnlp/sst2",
      "license": "See dataset card",
      "split": "validation",
      "question": {
        "type": "boolean"
      },
      "laya": {
        "n": 200,
        "failed": 0,
        "accuracy": 0.48,
        "accuracy95": [
          0.4117904407948983,
          0.5489633994418346
        ],
        "meanConfidence": 0.9725279999999994,
        "ece": 0.5013979999999996,
        "predictionShare": {
          "negative": 0.975,
          "positive": 0.025
        },
        "highConfidenceShare": 0.92,
        "highConfidenceAccuracy": 0.4945652173913043,
        "latencyP50Ms": 116.8,
        "latencyP95Ms": 600.7,
        "meanInputTokens": null,
        "usdPer1000Calls": null,
        "reliability": [
          {
            "from": 0.5,
            "to": 0.6,
            "n": 2,
            "accuracy": 1,
            "confidence": 0.557
          },
          {
            "from": 0.6,
            "to": 0.7,
            "n": 4,
            "accuracy": 0.25,
            "confidence": 0.65
          },
          {
            "from": 0.7,
            "to": 0.8,
            "n": 4,
            "accuracy": 0,
            "confidence": 0.736
          },
          {
            "from": 0.8,
            "to": 0.9,
            "n": 6,
            "accuracy": 0.333,
            "confidence": 0.849
          },
          {
            "from": 0.9,
            "to": 1,
            "n": 184,
            "accuracy": 0.495,
            "confidence": 0.993
          }
        ]
      }
    },
    "sst2_choice": {
      "name": "SST-2 sentiment as a choice (2 options)",
      "source": "https://huggingface.co/datasets/stanfordnlp/sst2",
      "license": "See dataset card",
      "split": "validation",
      "question": {
        "type": "choice",
        "options": 2
      },
      "laya": {
        "n": 200,
        "failed": 0,
        "accuracy": 0.93,
        "accuracy95": [
          0.8859442178331628,
          0.9578482170770813
        ],
        "meanConfidence": 0.8872254999999997,
        "ece": 0.08814550000000002,
        "predictionShare": {
          "negative": 0.435,
          "positive": 0.565
        },
        "highConfidenceShare": 0.67,
        "highConfidenceAccuracy": 0.9850746268656716,
        "latencyP50Ms": 55.6,
        "latencyP95Ms": 69.5,
        "meanInputTokens": null,
        "usdPer1000Calls": null,
        "reliability": [
          {
            "from": 0.5,
            "to": 0.6,
            "n": 12,
            "accuracy": 0.167,
            "confidence": 0.545
          },
          {
            "from": 0.6,
            "to": 0.7,
            "n": 12,
            "accuracy": 0.917,
            "confidence": 0.652
          },
          {
            "from": 0.7,
            "to": 0.8,
            "n": 13,
            "accuracy": 1,
            "confidence": 0.751
          },
          {
            "from": 0.8,
            "to": 0.9,
            "n": 29,
            "accuracy": 0.966,
            "confidence": 0.859
          },
          {
            "from": 0.9,
            "to": 1,
            "n": 134,
            "accuracy": 0.985,
            "confidence": 0.958
          }
        ]
      }
    }
  }
}