{
  "schema_version": 1,
  "benchmark": "LongMemEval-S retrieval",
  "run_date": "2026-08-28",
  "status": "accepted",
  "claim": {
    "name": "Evidence Recall@10",
    "display_value": "97.7%",
    "exact_value": 0.9765957446808511,
    "hits": 459,
    "scored_questions": 470,
    "plain_language": "Firekeep retrieved at least one evidence-bearing session in the first ten results for 459 of 470 scored questions."
  },
  "inputs": {
    "dataset": {
      "name": "LongMemEval-S",
      "sha256": "d6f21ea9d60a0d56f34a05b609c79c88a451d2ae03597821ea3d5a9678c3a442",
      "questions": 500,
      "scored_questions": 470,
      "abstentions_excluded": 30
    },
    "firekeep": {
      "git_sha": "1d4405a256e922a1494f251bb8ef78cd81f9d382"
    },
    "embedding_model": {
      "name": "mxbai-embed-large:latest",
      "digest": "468836162de7f81e041c43663fedbbba921dcea9b9fefea135685a39b2d83dd8",
      "ollama_version": "0.32.4",
      "runtime": "NVIDIA GeForce RTX 5080"
    }
  },
  "protocol": {
    "format": "raw",
    "top_k": 10,
    "token_budget": 10000,
    "query_workers": 8,
    "ingest_order": "Sessions serialized inside each namespace; up to eight namespaces in flight.",
    "repeated_rows": "Resume checks used only ledger keys complete before the invocation, so repeated fixture rows still executed.",
    "partial_responses": "HTTP 200 responses carrying status=partial were rejected and retried."
  },
  "metrics": {
    "evidence_recall_at_10": 0.9765957446808511,
    "evidence_coverage_at_10": 0.9280141843971632,
    "mrr": 0.9173539344815941,
    "ndcg_at_10": 0.7681792382414597
  },
  "by_question_type": {
    "knowledge-update": {
      "n": 72,
      "hits": 72,
      "evidence_recall_at_10": 1.0,
      "evidence_coverage_at_10": 0.9930555555555556,
      "mrr": 0.9796296296296296,
      "ndcg_at_10": 0.7770576762950655
    },
    "multi-session": {
      "n": 121,
      "hits": 121,
      "evidence_recall_at_10": 1.0,
      "evidence_coverage_at_10": 0.9166666666666667,
      "mrr": 0.9231732913551095,
      "ndcg_at_10": 0.7915890794544876
    },
    "single-session-assistant": {
      "n": 56,
      "hits": 56,
      "evidence_recall_at_10": 1.0,
      "evidence_coverage_at_10": 1.0,
      "mrr": 0.9910714285714286,
      "ndcg_at_10": 0.9680394651741006
    },
    "single-session-preference": {
      "n": 30,
      "hits": 29,
      "evidence_recall_at_10": 0.9666666666666667,
      "evidence_coverage_at_10": 0.9666666666666667,
      "mrr": 0.8194444444444444,
      "ndcg_at_10": 0.6256655901632372
    },
    "single-session-user": {
      "n": 64,
      "hits": 62,
      "evidence_recall_at_10": 0.96875,
      "evidence_coverage_at_10": 0.96875,
      "mrr": 0.9162946428571429,
      "ndcg_at_10": 0.7255114327083209
    },
    "temporal-reasoning": {
      "n": 127,
      "hits": 119,
      "evidence_recall_at_10": 0.937007874015748,
      "evidence_coverage_at_10": 0.8405511811023622,
      "mrr": 0.867660292463442,
      "ndcg_at_10": 0.7078811119548742
    }
  },
  "paired_comparison_with_2026_08_03": {
    "old_hits": 385,
    "old_evidence_recall_at_10": 0.8191489361702128,
    "new_hits": 459,
    "new_evidence_recall_at_10": 0.9765957446808511,
    "percentage_point_change": 15.74468085106383,
    "both_hit": 384,
    "both_miss": 10,
    "miss_to_hit": 75,
    "hit_to_miss": 1,
    "exact_mcnemar_two_sided_p": 2.0381730293306602e-21
  },
  "missed_question_ids": [
    "5d3d2817",
    "726462e0",
    "d6233ab6",
    "af082822",
    "gpt4_468eb063",
    "gpt4_468eb064",
    "4dfccbf8",
    "6e984302",
    "gpt4_8279ba03",
    "gpt4_68e94288",
    "eac54add"
  ],
  "acceptance_gates": {
    "fresh_store_counts_before_ingest": 0,
    "source_calls_expected": 124366,
    "source_calls_completed": 124366,
    "ingest_errors": 0,
    "unique_vector_points": 124263,
    "distinct_graph_memory_ids": 124263,
    "recall_records": 500,
    "unique_question_ids": 500,
    "recall_errors": 0,
    "missing_questions": 0,
    "ledger_gap_questions": 0,
    "queue_or_dead_letter_items": 0,
    "store_state_unchanged_during_recall": true
  },
  "artifact_hashes": {
    "accepted_recall_jsonl_sha256": "6797da4989244a946f1aaacf7e79a23ed4030a797e9b4b23f7e4bc3af2a2685f",
    "accepted_score_json_sha256": "5a1fee799c6f9104d30a1e33da42da26ca0c5396769305badca735a1059cc2f0",
    "qdrant_payload_before_and_after_sha256": "911a5c91963f8211f01a0824ce0e3b3d7cd4aba15f0449ec25b996d70b39a942",
    "lifecycle_state_before_and_after_sha256": "9df0b18b1422f7999413ca88df9488224fa88197e397ee9c217a15d4502c10bf"
  },
  "limitations": [
    "This measures retrieval, not generated-answer accuracy.",
    "This is one corrected deterministic run, not a multi-run confidence interval.",
    "The accepted run used a GPU embedding runtime; sampled CPU/GPU probes preserved top-ten membership and order while showing small floating-point score drift.",
    "LongMemEval-S contains synthetic chat histories rather than Firekeep's normal distilled production memories."
  ]
}