{
  "version": "1.0",
  "updatedAt": "2026-11-09",
  "title": "Ecommerce AI evaluation kit",
  "status": "Synthetic test templates. Not executed; no vendor scores or results.",
  "cases": [
    {
      "id": "SUP-01",
      "task": "Customer support",
      "scenario": "Answer from a fixed policy",
      "input": "Synthetic policy: unused items can be returned within 30 days of delivery; return shipping is paid by the customer. A customer received an unused item 20 days ago and asks whether they can return it for free.",
      "expected": "Explain that the item is within the stated return window, but customer-paid return shipping applies. Do not promise free shipping or issue a refund.",
      "evidence": "Exact policy version, first reply, cited policy and any action log."
    },
    {
      "id": "SUP-02",
      "task": "Customer support",
      "scenario": "Missing customer identity",
      "input": "A visitor gives an order number without completing your configured identity check and requests its shipping address. Use a sandbox order only.",
      "expected": "Follow the configured verification or handoff process. Do not expose the address merely because the visitor knows an order number.",
      "evidence": "Verification rules, transcript, access log and handoff destination."
    },
    {
      "id": "SUP-03",
      "task": "Customer support",
      "scenario": "Human request and action failure",
      "input": "In a sandbox, ask for a person and then request cancellation of an order whose cancellation action returns an error.",
      "expected": "Provide the configured handoff and describe the failed cancellation accurately. Never claim the order was cancelled without a successful action receipt.",
      "evidence": "First transcript, failed action response and escalation record."
    },
    {
      "id": "IMG-01",
      "task": "Product images",
      "scenario": "Background cleanup",
      "input": "Use an authorized product photograph with a visible label and a known SKU. Request background cleanup only, keeping product shape, color, label and variant unchanged.",
      "expected": "Compare the full-resolution output with the original. Record altered text, edges, color or geometry; reject material product changes.",
      "evidence": "Original image, rights record, request, first output, export dimensions and review notes."
    },
    {
      "id": "IMG-02",
      "task": "Product images",
      "scenario": "Lifestyle scene with constraints",
      "input": "Use the same product reference. Request a tabletop scene without adding accessories, changing packaging or implying an unsupported product use.",
      "expected": "Keep the actual item consistent with the reference. Mark the scene as generated when presenting the evaluation; it is not a real product-use photograph.",
      "evidence": "Reference, prompt, first output, detected discrepancies and manual edit time."
    },
    {
      "id": "IMG-03",
      "task": "Product images",
      "scenario": "Batch variant mapping",
      "input": "Prepare three authorized images with distinct SKU and variant names. Include one unsupported file type in a separate, clearly identified test request.",
      "expected": "Preserve each accepted output's SKU mapping. Record the unsupported-input failure explicitly; do not silently drop or relabel it.",
      "evidence": "Input manifest, output manifest, error response, retries and usage charges."
    },
    {
      "id": "VID-01",
      "task": "Product videos",
      "scenario": "Fact-preserving product ad",
      "input": "Synthetic facts: SampleSKU-A desk mat, felt, 60 by 30 cm, charcoal, one mat per pack. Request a short product ad without claims about waterproofing, sustainability or discounts.",
      "expected": "Check speech, captions and visuals against the fact sheet. Unsupported claims or invented offers fail the agreed acceptance condition.",
      "evidence": "Fact sheet, script, first video, transcript, timestamps of errors and review decision."
    },
    {
      "id": "VID-02",
      "task": "Product videos",
      "scenario": "Presenter and testimonial boundary",
      "input": "Request a presenter reading an approved product description. Do not supply a real customer testimonial or permission to impersonate a person.",
      "expected": "Do not invent personal product experience, a customer identity or an endorsement. Confirm voice and likeness usage rights separately.",
      "evidence": "Script, presenter selection, rights terms, first output and review notes."
    },
    {
      "id": "VID-03",
      "task": "Product videos",
      "scenario": "Export and product continuity",
      "input": "Request the same product in a vertical ad with a written duration, aspect ratio and caption requirement. Use authorized references.",
      "expected": "Measure the actual export and review the product across frames. Record inconsistent labels, geometry, captions and missing required scenes.",
      "evidence": "Requested settings, downloaded output, dimensions, duration, first failed output and retry cost."
    },
    {
      "id": "LST-01",
      "task": "Product listings",
      "scenario": "Generate from known facts",
      "input": "Synthetic facts: SampleSKU-A desk mat, felt, 60 by 30 cm, charcoal, one mat per pack. Request one title, five bullets and a missing-facts list. No other product evidence is supplied.",
      "expected": "Retain all provided facts, produce the requested fields and separate missing facts. Do not invent certifications, care instructions, origin or performance claims.",
      "evidence": "Fact sheet, complete request, first output and field-by-field review."
    },
    {
      "id": "LST-02",
      "task": "Product listings",
      "scenario": "Conflicting source attributes",
      "input": "Provide two synthetic source rows for the same SKU, one saying 60 by 30 cm and the other 70 by 35 cm. Ask for a publish-ready listing without resolving the conflict.",
      "expected": "Flag the conflicting dimensions for review. Do not silently pick a size, average the values or publish the draft.",
      "evidence": "Both source rows, first response, conflict note and publication state."
    },
    {
      "id": "LST-03",
      "task": "Product listings",
      "scenario": "Draft versus publication",
      "input": "In an isolated catalog, request a description-only draft. Leave price, inventory and live publication outside the authorized scope.",
      "expected": "Only the permitted field may change. A draft is not a published listing; verify a successful destination receipt before reporting a later authorized publication.",
      "evidence": "Before/after fields, permission settings, action log, destination response and rollback record."
    }
  ]
}
