{
  "version": "1.0.0",
  "status": "DRAFT_REQUIRES_CALIBRATION",
  "config": {
    "system": "rag",
    "dailyRequests": 1000,
    "evaluationPercent": 10,
    "baselineDays": 14,
    "windowHours": 24,
    "minSamples": 100,
    "persistence": 2,
    "owner": "",
    "rollbackTarget": "",
    "thresholds": {}
  },
  "metrics": [
    {
      "id": "input_shift",
      "name": "Input distribution shift",
      "systems": [
        "chat",
        "rag",
        "agent"
      ],
      "sampleClass": "requests",
      "threshold": 0.15,
      "min": 0,
      "max": 1,
      "unit": "TVD",
      "comparator": "gt",
      "definition": "Total variation distance = 0.5 × sum of absolute differences in category proportions between current and reference prompts. Freeze the same intent/language taxonomy and include an unknown category. Denominator: prompts with a recorded category.",
      "rationale": "A changed request mix can invalidate an old evaluation set without proving a quality regression.",
      "response": "Inspect affected intent and language slices. Compare downstream outcomes and expand the holdout set.",
      "rollback": "Do not roll back for input shift alone. Consider a release rollback only if a matched-cohort evaluation links degradation to that release.",
      "sources": [
        "S2",
        "S3"
      ],
      "thresholdBasis": "Illustrative setting. Calibrate against local baseline and error budget.",
      "sourceReferences": [
        {
          "id": "S2",
          "url": "https://docs.aws.amazon.com/prescriptive-guidance/latest/gen-ai-lifecycle-operational-excellence/prod-monitoring-drift.html",
          "checked": "2026-09-08"
        },
        {
          "id": "S3",
          "url": "https://docs.evidentlyai.com/metrics/explainer_drift",
          "checked": "2026-09-08"
        }
      ],
      "expectedSamples": 1000,
      "sampleStatus": "SAMPLE_ESTIMATE_ONLY"
    },
    {
      "id": "task_success",
      "name": "Task success rate",
      "systems": [
        "chat",
        "rag",
        "agent"
      ],
      "sampleClass": "evaluated",
      "threshold": 90,
      "min": 0,
      "max": 100,
      "unit": "%",
      "comparator": "lt",
      "definition": "100 × evaluated completed tasks that pass the fixed task rubric / all eligible completed tasks evaluated. Record unscored and pending labels separately.",
      "rationale": "Business outcomes can change while input distributions remain stable.",
      "response": "Re-score a matched sample, examine labels and group failures by release, task and language.",
      "rollback": "Use the last compatible release only after a repeated quality breach is attributable to a change and the candidate passes the same eval set.",
      "sources": [
        "S2",
        "S6"
      ],
      "thresholdBasis": "Illustrative setting. Calibrate against local baseline and error budget.",
      "sourceReferences": [
        {
          "id": "S2",
          "url": "https://docs.aws.amazon.com/prescriptive-guidance/latest/gen-ai-lifecycle-operational-excellence/prod-monitoring-drift.html",
          "checked": "2026-09-08"
        },
        {
          "id": "S6",
          "url": "https://docs.evidentlyai.com/docs/platform/monitoring_overview",
          "checked": "2026-09-08"
        }
      ],
      "expectedSamples": 100,
      "sampleStatus": "SAMPLE_ESTIMATE_ONLY"
    },
    {
      "id": "refusal",
      "name": "Benign-request refusal rate",
      "systems": [
        "chat",
        "rag",
        "agent"
      ],
      "sampleClass": "evaluated",
      "threshold": 10,
      "min": 0,
      "max": 100,
      "unit": "%",
      "comparator": "gt",
      "definition": "100 × refused requests judged answerable and allowed / evaluated requests judged answerable and allowed. Exclude unsafe prompts from this denominator and monitor unsafe acceptance separately.",
      "rationale": "A rise may indicate a policy or prompt regression. An appropriate safety refusal is not a failure.",
      "response": "Review benign refusals with a stable labeling rubric before changing guardrails.",
      "rollback": "Revert an implicated policy or prompt change only after both benign-helpfulness and unsafe-request tests pass.",
      "sources": [
        "S2",
        "S6"
      ],
      "thresholdBasis": "Illustrative setting. Calibrate against local baseline and error budget.",
      "sourceReferences": [
        {
          "id": "S2",
          "url": "https://docs.aws.amazon.com/prescriptive-guidance/latest/gen-ai-lifecycle-operational-excellence/prod-monitoring-drift.html",
          "checked": "2026-09-08"
        },
        {
          "id": "S6",
          "url": "https://docs.evidentlyai.com/docs/platform/monitoring_overview",
          "checked": "2026-09-08"
        }
      ],
      "expectedSamples": 100,
      "sampleStatus": "SAMPLE_ESTIMATE_ONLY"
    },
    {
      "id": "schema",
      "name": "Schema conformance",
      "systems": [
        "chat",
        "rag",
        "agent"
      ],
      "sampleClass": "requests",
      "threshold": 99,
      "min": 0,
      "max": 100,
      "unit": "%",
      "comparator": "lt",
      "definition": "100 × outputs passing the versioned required schema / requests that require that schema. Include missing, truncated and unparsable outputs as failures. Track endpoint errors separately.",
      "rationale": "Successful HTTP responses can still break downstream consumers.",
      "response": "Compare first-attempt and final outputs separately. Inspect schema and model versions.",
      "rollback": "Revert the implicated model, prompt or schema change if the prior compatible bundle restores validation without violating current requirements.",
      "sources": [
        "S1",
        "S6"
      ],
      "thresholdBasis": "Illustrative setting. Calibrate against local baseline and error budget.",
      "sourceReferences": [
        {
          "id": "S1",
          "url": "https://pharosproduction.com/insights/engineering/state-of-production-ai-engineering-2026/",
          "checked": "2026-09-08"
        },
        {
          "id": "S6",
          "url": "https://docs.evidentlyai.com/docs/platform/monitoring_overview",
          "checked": "2026-09-08"
        }
      ],
      "expectedSamples": 1000,
      "sampleStatus": "SAMPLE_ESTIMATE_ONLY"
    },
    {
      "id": "latency",
      "name": "End-to-end p95 latency",
      "systems": [
        "chat",
        "rag",
        "agent"
      ],
      "sampleClass": "requests",
      "threshold": 2500,
      "min": 1,
      "max": 3600000,
      "unit": "ms",
      "comparator": "gt",
      "definition": "95th percentile of end-to-end completed request durations, measured from application receipt to terminal response. Record timeouts/cancellations separately. Do not average per-host percentiles.",
      "rationale": "A quality-preserving release can still exceed the user-facing latency budget.",
      "response": "Inspect queue time, retrieval, retries and model spans. Compare like-for-like workload slices.",
      "rollback": "If a canary release causes an SLO breach, route to a compatible known-good path. Check quality and capacity before expanding traffic.",
      "sources": [
        "S5"
      ],
      "thresholdBasis": "Illustrative setting. Calibrate against local baseline and error budget.",
      "sourceReferences": [
        {
          "id": "S5",
          "url": "https://github.com/open-telemetry/semantic-conventions-genai/tree/b5d8440f6f126738fd50f927752cd669772c517b/docs/gen-ai",
          "checked": "2026-09-08"
        }
      ],
      "expectedSamples": 1000,
      "sampleStatus": "SAMPLE_ESTIMATE_ONLY"
    },
    {
      "id": "grounding",
      "name": "Grounded-claim rate",
      "systems": [
        "rag"
      ],
      "sampleClass": "evaluated",
      "threshold": 90,
      "min": 0,
      "max": 100,
      "unit": "%",
      "comparator": "lt",
      "definition": "100 × checkable claims supported by retrieved evidence / all checkable claims in evaluated answers. Judge the cited passage, not the mere presence of a citation. Track answers with no checkable claims separately.",
      "rationale": "The fraction measures evidence support under a rubric, not universal factual accuracy.",
      "response": "Inspect source validity, chunk boundaries and retrieval versions. Sample judge disagreements for human review.",
      "rollback": "Revert the corpus, retriever or prompt only if a replay isolates that component and the restored corpus remains valid and authorized.",
      "sources": [
        "S1",
        "S6"
      ],
      "thresholdBasis": "Illustrative setting. Calibrate against local baseline and error budget.",
      "sourceReferences": [
        {
          "id": "S1",
          "url": "https://pharosproduction.com/insights/engineering/state-of-production-ai-engineering-2026/",
          "checked": "2026-09-08"
        },
        {
          "id": "S6",
          "url": "https://docs.evidentlyai.com/docs/platform/monitoring_overview",
          "checked": "2026-09-08"
        }
      ],
      "expectedSamples": 100,
      "sampleStatus": "SAMPLE_ESTIMATE_ONLY"
    },
    {
      "id": "recall",
      "name": "Retrieval recall@k",
      "systems": [
        "rag"
      ],
      "sampleClass": "evaluated",
      "threshold": 80,
      "min": 0,
      "max": 100,
      "unit": "%",
      "comparator": "lt",
      "definition": "Mean per-query percentage of labeled relevant documents found within the top k results, on the versioned retrieval evaluation set. Freeze k and relevance labels. Exclude queries without labeled relevant documents and report their count.",
      "rationale": "A current corpus can still miss the document needed to answer the question.",
      "response": "Separate missing source documents from retriever/reranker failures. Rerun against the previous index.",
      "rollback": "Restore a compatible index or retriever only after the same relevance set improves and required document updates remain available.",
      "sources": [
        "S1",
        "S6"
      ],
      "thresholdBasis": "Illustrative setting. Calibrate against local baseline and error budget.",
      "sourceReferences": [
        {
          "id": "S1",
          "url": "https://pharosproduction.com/insights/engineering/state-of-production-ai-engineering-2026/",
          "checked": "2026-09-08"
        },
        {
          "id": "S6",
          "url": "https://docs.evidentlyai.com/docs/platform/monitoring_overview",
          "checked": "2026-09-08"
        }
      ],
      "expectedSamples": 100,
      "sampleStatus": "SAMPLE_ESTIMATE_ONLY"
    },
    {
      "id": "freshness",
      "name": "Corpus update lag",
      "systems": [
        "rag"
      ],
      "sampleClass": "event",
      "threshold": 24,
      "min": 0,
      "max": 8760,
      "unit": "hours",
      "comparator": "gt",
      "definition": "Maximum elapsed time from an authoritative source revision to the matching revision becoming queryable. Include outstanding updates using current time. Measure deletion propagation separately.",
      "rationale": "An ingestion backlog can make grounded answers obsolete even when the model has not changed.",
      "response": "Inspect failed ingestion, change feeds, index jobs and outstanding deletions. Identify affected sources.",
      "rollback": "Pause answers from affected sources or use a reviewed fallback. Never restore revoked, deleted or unauthorized documents to improve a metric.",
      "sources": [
        "S1"
      ],
      "thresholdBasis": "Illustrative setting. Calibrate against local baseline and error budget.",
      "sourceReferences": [
        {
          "id": "S1",
          "url": "https://pharosproduction.com/insights/engineering/state-of-production-ai-engineering-2026/",
          "checked": "2026-09-08"
        }
      ],
      "expectedSamples": null,
      "sampleStatus": "EVENT_BASED"
    },
    {
      "id": "prompt_changes",
      "name": "Unreviewed prompt releases",
      "systems": [
        "chat",
        "rag",
        "agent"
      ],
      "sampleClass": "event",
      "threshold": 0,
      "min": 0,
      "max": 0,
      "unit": "releases",
      "comparator": "gt",
      "definition": "Count of active prompt-template hashes that lack a successful regression evaluation and an approved release record. Prompt entropy here means uncontrolled template changes, not a Shannon entropy metric.",
      "rationale": "Without a release record, changed behavior cannot be attributed or reproduced.",
      "response": "Stop promotion and compare active hashes with the approved prompt registry.",
      "rollback": "Restore the approved compatible prompt only after verifying tool permissions, model and schema compatibility. This event bypasses sample and persistence gates.",
      "sources": [
        "S1"
      ],
      "thresholdBasis": "Illustrative setting. Calibrate against local baseline and error budget.",
      "sourceReferences": [
        {
          "id": "S1",
          "url": "https://pharosproduction.com/insights/engineering/state-of-production-ai-engineering-2026/",
          "checked": "2026-09-08"
        }
      ],
      "expectedSamples": null,
      "sampleStatus": "EVENT_BASED"
    },
    {
      "id": "model_lifecycle",
      "name": "Provider sunset lead time",
      "systems": [
        "chat",
        "rag",
        "agent"
      ],
      "sampleClass": "event",
      "threshold": 30,
      "min": 1,
      "max": 365,
      "unit": "days",
      "comparator": "lt",
      "definition": "Days remaining until the provider-published retirement date for the exact deployed model snapshot. Unknown retirement dates are unknown, never zero or safe.",
      "rationale": "A scheduled model retirement requires a tested migration path. It is a lifecycle event rather than a statistical drift score.",
      "response": "Verify the provider notice and rehearse the replacement against the current eval set.",
      "rollback": "Use a supported pre-evaluated fallback if the snapshot becomes unavailable. Do not roll back to a retired model. Check daily and on notices. No sample gate.",
      "sources": [
        "S1"
      ],
      "thresholdBasis": "Illustrative setting. Calibrate against local baseline and error budget.",
      "sourceReferences": [
        {
          "id": "S1",
          "url": "https://pharosproduction.com/insights/engineering/state-of-production-ai-engineering-2026/",
          "checked": "2026-09-08"
        }
      ],
      "expectedSamples": null,
      "sampleStatus": "EVENT_BASED"
    }
  ],
  "warnings": [
    "Draft requires calibration. Expected samples are traffic estimates, not observed eligible denominators.",
    "Missing owner: assign a responsible reviewer.",
    "Missing rollback target: identify a compatible fallback."
  ]
}