Playground

{
  "origin": "fixture",
  "answers": {
    "customerSpecificity": {
      "type": "score",
      "score": 1.5,
      "probabilities": {
        "0": 0,
        "1": 0.5,
        "2": 0.5,
        "3": 0,
        "4": 0
      }
    },
    "problemSpecificity": {
      "type": "score",
      "score": 2.7,
      "probabilities": {
        "0": 0,
        "1": 0,
        "2": 0.2999999999999998,
        "3": 0.7000000000000002,
        "4": 0
      }
    },
    "differentiation": {
      "type": "score",
      "score": 1.2,
      "probabilities": {
        "0": 0,
        "1": 0.8,
        "2": 0.19999999999999996,
        "3": 0,
        "4": 0
      }
    },
    "evidenceSupport": {
      "type": "score",
      "score": 0.2,
      "probabilities": {
        "0": 0.8,
        "1": 0.2,
        "2": 0,
        "3": 0,
        "4": 0
      }
    },
    "criticalUnknown": {
      "type": "boolean",
      "probability": 0.91
    }
  },
  "providerConfidence": {
    "customerSpecificity": 0.48,
    "problemSpecificity": 0.44,
    "differentiation": 0.39,
    "evidenceSupport": 0.62
  },
  "policyDecision": {
    "action": "research",
    "label": "Research the assumptions",
    "reasonCodes": [
      "evidence-gap-or-critical-unknown"
    ]
  },
  "run": null
}

import { experimental_evaluate as evaluate } from 'ai';

const state = {
  "idea": "An app that helps freelancers track unpaid invoices.",
  "targetCustomer": "Freelancers",
  "evidence": ""
};
const questions = {
  "customerSpecificity": {
    "type": "score",
    "instructions": "How specifically does the supplied description identify the intended customer? Judge the text, not the market.",
    "criteria": [
      "No customer is described",
      "A very broad group is named",
      "A profession or business type is named",
      "A narrow group and relevant situation are specified",
      "A narrow group, workflow and explicit selection constraints are specified"
    ]
  },
  "problemSpecificity": {
    "type": "score",
    "instructions": "How concrete is the problem description in the supplied text?",
    "criteria": [
      "No problem is described",
      "A vague inconvenience is stated",
      "A recognizable problem is named",
      "A concrete recurring workflow problem is described",
      "The workflow, consequence and relevant situation are all explicit"
    ]
  },
  "differentiation": {
    "type": "score",
    "instructions": "How explicit is the proposed difference from alternatives in the supplied description? Do not invent competitors.",
    "criteria": [
      "No difference is described",
      "Only a generic quality claim is made",
      "A concrete feature difference is claimed",
      "A feature difference is tied to an explicit user situation",
      "The text explains a specific alternative and a relevant difference, with limitations"
    ]
  },
  "evidenceSupport": {
    "type": "score",
    "instructions": "How much relevant validation material is supplied in this text? This is not independent verification of the material.",
    "criteria": [
      "No validation material is supplied",
      "Only beliefs or hypothetical demand are supplied",
      "Specific user observations or interviews are reported",
      "A relevant behavioral test or trial is reported with limitations",
      "Relevant paid or repeated-use behavior is reported with enough context to inspect"
    ]
  },
  "criticalUnknown": {
    "type": "boolean",
    "instructions": "Does the supplied description leave a central demand or execution assumption untested?",
    "criteria": {
      "true": "A central assumption is missing evidence or is explicitly untested.",
      "false": "The described central assumptions have relevant supplied evidence; this does not verify that evidence."
    }
  }
};
const policy = {
  "evidenceFloor": 1,
  "criticalUnknownThreshold": 0.6,
  "specificityFloor": 2,
  "differentiationFloor": 1
};

const result = await evaluate({
  model: 'typesafe-ai/jev',
  state,
  questions,
  maxRetries: 0,
  providerOptions: { gateway: { zeroDataRetention: true } },
});

// Validate result.answers before applying your deterministic policy.
console.log({ answers: result.answers, policy });