Compare a Claim with Supplied Evidence

{
  "origin": "fixture",
  "answers": {
    "relation_e1": {
      "type": "choice",
      "choice": "supports",
      "probabilities": {
        "supports": 0.76,
        "contradicts": 0.07,
        "irrelevant": 0.05,
        "insufficient": 0.12
      }
    }
  },
  "providerConfidence": {
    "relation_e1": 0.55
  },
  "policyDecision": {
    "action": "supported",
    "label": "Supported by supplied text",
    "reasonCodes": [
      "support-above-threshold"
    ]
  },
  "run": null
}

import { experimental_evaluate as evaluate } from 'ai';

const state = {
  "claim": "The customer was refunded.",
  "evidence": [
    {
      "id": "e1",
      "text": "Synthetic support transcript: We returned the full payment to your original payment method today."
    }
  ]
};
const questions = {
  "relation_e1": {
    "type": "choice",
    "instructions": "What is the relationship between supplied evidence e1 and the claim? Judge only the supplied texts, not outside-world truth.",
    "criteria": {
      "supports": "The supplied text directly supports the claim.",
      "contradicts": "The supplied text directly contradicts the claim.",
      "irrelevant": "The supplied text is unrelated to the claim.",
      "insufficient": "The relationship or support is ambiguous or incomplete."
    }
  }
};
const policy = {
  "minRelationProbability": 0.7
};

const result = await evaluate({
  model: 'typesafe-ai/jev',
  state,
  questions,
  maxRetries: 0,
  providerOptions: { gateway: { zeroDataRetention: true } },
});

// Validate result.answers before applying your deterministic policy.
console.log({ answers: result.answers, policy });