feat(benchmarks): promote vector_search to stable with integrity spec

benchbox/core/benchmark_registry.yaml

@@ -586,7 +586,7 @@ benchmark_metadata:
     display_name: Vector Search
     description: Vector similarity search benchmark (kNN, ANN, filtered)
     category: AI/ML
-    support_status: beta
+    support_status: stable
     num_queries: 6
     query_description: 6 queries (kNN cosine/L2, filtered, recall@k, ANN, multi-category)
     supports_streams: false

benchbox/core/results/benchmark_specs.yaml

@@ -835,3 +835,20 @@ benchmark_specs:
     requires_tables_object: false
     sf1_row_counts: null
     sf1_power_at_size_range: null
+  vector_search:
+    benchmark_id: vector_search
+    unique_query_ids:
+    - '1'
+    - '2'
+    - '3'
+    - '4'
+    - '5'
+    - '6'
+    min_unique_queries: 0
+    min_success_rate: 1.0
+    high_failure_expected: false
+    requires_tables_object: true
+    sf1_row_counts:
+      vectors: 1000000
+      vector_queries: 100
+    sf1_power_at_size_range: null

benchbox/core/results/integrity_validator.py

@@ -264,16 +264,22 @@ def _check_query_count_math(self, raw: dict[str, Any], checks: list[CheckResult]
         total = sq.get("total", 0)
         passed = sq.get("passed", 0)
         failed = sq.get("failed", 0)
-        if passed + failed != total:
+        skipped = sq.get("skipped", 0)
+        if passed + failed + skipped != total:
             _fail(
                 checks,
                 _STRUCTURAL,
                 "query_count_math",
-                f"{passed} passed + {failed} failed != {total} total",
-                {"passed": passed, "failed": failed, "total": total},
+                f"{passed} passed + {failed} failed + {skipped} skipped != {total} total",
+                {"passed": passed, "failed": failed, "skipped": skipped, "total": total},
             )
         else:
-            _pass(checks, _STRUCTURAL, "query_count_math", f"{total} = {passed} passed + {failed} failed")
+            _pass(
+                checks,
+                _STRUCTURAL,
+                "query_count_math",
+                f"{total} = {passed} passed + {failed} failed + {skipped} skipped",
+            )
 
     def _check_timing_non_negative(self, raw: dict[str, Any], checks: list[CheckResult]) -> None:
         timing = raw.get("summary", {}).get("timing", {})
@@ -500,11 +506,19 @@ def _check_success_rate(self, raw: dict[str, Any], spec: Any | None, checks: lis
         sq = raw.get("summary", {}).get("queries", {})
         total = sq.get("total", 0)
         passed = sq.get("passed", 0)
+        skipped = sq.get("skipped", 0)
         if total == 0:
             _fail(checks, _BELIEVABILITY, "success_rate", "No queries recorded")
             return
 
-        rate = passed / total
+        # Compatibility-skipped queries (version-gated rules such as the
+        # StarRocks Q2 l2_distance skip) are not failures: discount them
+        # from both sides before applying the spec floor.
+        billable = total - skipped
+        if billable == 0:
+            _pass(checks, _BELIEVABILITY, "success_rate", "All queries skipped by compatibility rules")
+            return
+        rate = passed / billable
         if spec is not None and spec.high_failure_expected:
             _pass(
                 checks,
@@ -521,10 +535,15 @@ def _check_success_rate(self, raw: dict[str, Any], spec: Any | None, checks: lis
                 _BELIEVABILITY,
                 "success_rate",
                 f"{rate:.1%} success rate below {floor:.0%} floor",
-                {"rate": rate, "floor": floor, "passed": passed, "total": total},
+                {"rate": rate, "floor": floor, "passed": passed, "total": billable, "skipped": skipped},
             )
         else:
-            _pass(checks, _BELIEVABILITY, "success_rate", f"{rate:.1%} >= {floor:.0%} floor")
+            _pass(
+                checks,
+                _BELIEVABILITY,
+                "success_rate",
+                f"{rate:.1%} >= {floor:.0%} floor ({skipped} skipped)",
+            )
 
     def _check_sf1_row_counts(
         self,

docs/benchmarks/support-status.md

@@ -73,7 +73,7 @@ integrity spec, query count, dataset/source, scales, and DataFrame capability.
 | `tsbs_devops` | `beta` | 18-query time-series DevOps workload; generated; integrity spec; docs; DataFrame-capable. | Time-series engine coverage breadth still expanding. |
 | `nyctaxi` | `beta` | 25-query real-world OLAP; external NYC TLC data; integrity spec; docs; DataFrame-capable. | External-dataset availability and a pinned source contract before `stable`. |
 | `flightdata` | `beta` | 20-query real-world aviation OLAP; external BTS data; integrity spec; docs; DataFrame-capable. | External-dataset availability and a pinned source contract before `stable`. |
-| `vector_search` | `beta` | 6-query vector similarity; synthetic embeddings; docs; not DataFrame-capable. | **No integrity spec** and ANN-recall validation still pending; capability is SQL-only. |
+| `vector_search` | `stable` | 6-query vector similarity; synthetic embeddings; integrity spec; docs; not DataFrame-capable (SQL-only by recorded routing decision, `supports_dataframe: false`). Q5 measures the same brute-force kNN SQL as Q1: no loader or adapter builds an HNSW index today, so ANN-index acceleration is not part of the measured workload. | None — maintain. |
 | `tpcds_obt` | `experimental` | 17-query denormalized One-Big-Table; derived from `tpcds`; integrity spec; docs; SF1 only; DataFrame-capable. | Single-scale subset of TPC-DS; still on the deprecated core base. |
 | `ai_primitives` | `experimental` | 16-query SQL AI functions; derived from TPC-H text; docs; core-only ID; not DataFrame-capable. | Cloud-only AI functions, cost-gated, no integrity spec; intentionally research-only. |
 | `tpchavoc` | `experimental` | 220-variant optimizer stress (22 queries × 10 syntax variants); generated; integrity spec; docs; DataFrame-capable. | Optimizer-stress research tool, not a standard comparison workload. |
@@ -86,10 +86,6 @@ integrity spec, query count, dataset/source, scales, and DataFrame capability.
 Recorded as explicit follow-up work rather than reviewer memory. None of these
 are applied here: this matrix documents status, it does not change it.
 
-- **`vector_search` (beta → stable candidate, blocked):** the only missing piece
-  for the integrity dimension is a `BenchmarkSpec`. Adding one (and a DataFrame
-  routing decision) is the concrete blocker. Tracked as a deferred follow-up on
-  `benchmark-support-status-criteria-matrix`.
 - **External-dataset betas (`nyctaxi`, `flightdata`):** promotion is gated on a
   complete set of reviewed SHA-256 pins for the default external corpus, not
   on query coverage. Until those maps are populated, generated corpora record

docs/development/result-integrity-validation.md

@@ -119,7 +119,7 @@ class BenchmarkSpec:
 
 ### Coverage
 
-Specs exist for 20 of 22 registered benchmarks: `tpch`, `tpcds`, `tpchavoc`, `tpch_skew`, `ssb`, `clickbench`, `nyctaxi`, `h2odb`, `amplab`, `joinorder`, `flightdata`, `datavault`, `coffeeshop`, `tpcdi`, `tsbs_devops`, `tpcds_obt`, `read_primitives`, `write_primitives`, `metadata_primitives`, `transaction_primitives`. `ai_primitives` and `vector_search` do not yet have integrity specs.
+Specs exist for 21 of 22 registered benchmarks: `tpch`, `tpcds`, `tpchavoc`, `tpch_skew`, `ssb`, `clickbench`, `nyctaxi`, `h2odb`, `amplab`, `joinorder`, `flightdata`, `datavault`, `coffeeshop`, `tpcdi`, `tsbs_devops`, `tpcds_obt`, `read_primitives`, `write_primitives`, `metadata_primitives`, `transaction_primitives`, `vector_search`. `ai_primitives` does not yet have an integrity spec.
 
 8 legacy aliases are mapped automatically (e.g., `star_schema` -> `ssb`, `amplab_big_data` -> `amplab`).
 

docs/reference/cli/utilities.md

@@ -162,7 +162,7 @@ The validator runs 20 checks in three tiers. See the [Result Integrity Validatio
 
 **Notes:**
 - Automatically excludes `.plans.json` and `.tuning.json` companion files
-- Specs cover 20 of 22 benchmarks with 8 legacy alias mappings (e.g., `star_schema` → `ssb`); `ai_primitives` and `vector_search` are not yet covered
+- Specs cover 21 of 22 benchmarks with 8 legacy alias mappings (e.g., `star_schema` → `ssb`); `ai_primitives` is not yet covered
 
 ## Related
 

docs/reference/public-contracts.md

@@ -142,13 +142,13 @@ separate `surface`/removal decision.
 
 ## Count and Drift Policy
 
-Benchmark API snapshot: **23** registry entries; **23** loader-resolved core families; **22** public discovery entries; **21** top-level Python benchmark facades; **15** lazy facades; **6** eager facades; **2** core-only benchmark IDs. Benchmark support status: **5** stable, **12** beta, **5** experimental, **1** repo-only, **0** deprecated, **0** document-only.
+Benchmark API snapshot: **23** registry entries; **23** loader-resolved core families; **22** public discovery entries; **21** top-level Python benchmark facades; **15** lazy facades; **6** eager facades; **2** core-only benchmark IDs. Benchmark support status: **6** stable, **11** beta, **5** experimental, **1** repo-only, **0** deprecated, **0** document-only.
 
 Evidence snapshot updated by `benchmark-support-status-and-discovery-policy`:
 
 | Source | Current evidence | Contract implication |
 |---|---|---|
-| `benchbox.core.benchmark_registry` | 23 benchmark metadata entries and 23 loader-resolved IDs; support status counts are stable=5, beta=12, experimental=5, repo_only=1, deprecated=0, document_only=0. | Benchmark count and support claims must derive from registry metadata or avoid exact counts. |
+| `benchbox.core.benchmark_registry` | 23 benchmark metadata entries and 23 loader-resolved IDs; support status counts are stable=6, beta=11, experimental=5, repo_only=1, deprecated=0, document_only=0. | Benchmark count and support claims must derive from registry metadata or avoid exact counts. |
 | `benchbox.core.platform_registry.PlatformRegistry.get_all_platform_metadata()` | 52 platform metadata entries: 48 SQL-capable, 18 DataFrame-capable, 14 dual-mode. | README and platform docs must not carry unqualified hand-maintained platform counts. |
 | `benchbox.core.results.schema_policy` | Current result schema version: `2.2`; runtime/explorer accepted versions: `2.0`, `2.1`, `2.2`; public submission accepts numeric `2.x`. | Result schema version claims must update with the named consumer policy or defer to this policy module. |
 | `README.md` before this TODO | Landing-page bullets claimed 22 benchmarks, 42 SQL platforms, and 9 DataFrame platforms. | Exact counts were stale relative to registry metadata; README now links to this policy instead of being authoritative. |

tests/unit/core/results/test_integrity_validator.py

@@ -6,6 +6,7 @@
 import glob
 import json
 import os
+from datetime import datetime
 from pathlib import Path
 from typing import Any
 
@@ -20,6 +21,8 @@
     validate_directory,
     validate_file,
 )
+from benchbox.core.results.models import BenchmarkResults
+from benchbox.core.results.schema import build_result_payload
 
 pytestmark = [
     pytest.mark.unit,
@@ -453,3 +456,80 @@ def test_validate_file_bad_json(self, tmp_path: Path) -> None:
         report = validate_file(bad_file)
         assert report.overall_status == CheckStatus.FAIL
         assert report.checks[0].name == "file_readable"
+
+
+# ---------------------------------------------------------------------------
+# TestVectorSearchExportIntegrity
+# ---------------------------------------------------------------------------
+
+
+def _make_vector_search_result() -> BenchmarkResults:
+    """Build a complete vector_search run using producer-style Q1-Q6 IDs."""
+    query_results = [
+        {
+            "query_id": f"Q{i}",
+            "execution_time_ms": 100.0 + i * 10,
+            "rows_returned": 10,
+            "status": "SUCCESS",
+            "iteration": 1,
+            "stream_id": 0,
+            "run_type": "measurement",
+        }
+        for i in range(1, 7)
+    ]
+    return BenchmarkResults(
+        benchmark_name="vector_search",
+        platform="duckdb",
+        scale_factor=1.0,
+        execution_id="vector-001",
+        timestamp=datetime(2026, 1, 1),
+        duration_seconds=5.0,
+        total_queries=6,
+        successful_queries=6,
+        failed_queries=0,
+        query_results=query_results,
+        table_statistics={"vectors": 1000000, "vector_queries": 100},
+        execution_metadata={
+            "mode": "standard",
+            "phase_status": {"power_test": {"status": "COMPLETED"}},
+        },
+    )
+
+
+class TestVectorSearchExportIntegrity:
+    """A complete vector_search export must pass the expected-query-IDs check.
+
+    Schema-v2 export normalizes producer IDs Q1-Q6 to 1-6, so the spec must
+    record the normalized form or every complete run reports 0/6 and fails.
+    """
+
+    def test_export_normalizes_query_ids(self) -> None:
+        """Export converts Q1-Q6 producer IDs to normalized 1-6 IDs."""
+        payload = build_result_payload(_make_vector_search_result())
+        assert [q["id"] for q in payload["queries"]] == ["1", "2", "3", "4", "5", "6"]
+
+    def test_complete_export_passes_expected_query_ids(self) -> None:
+        """All six exported queries match the spec set (6/6, no FAIL)."""
+        payload = build_result_payload(_make_vector_search_result())
+        payload["export"] = {"format": "json"}
+        report = ResultIntegrityValidator().validate(payload)
+        check = next(c for c in report.checks if c.name == "expected_query_ids")
+        assert check.status == CheckStatus.PASS
+        assert check.message.startswith("6/6 expected query IDs found")
+        assert report.overall_status == CheckStatus.PASS
+
+    def test_skipped_queries_satisfy_count_math(self) -> None:
+        """A version-gated skip (StarRocks Q2) must not fail count arithmetic."""
+        data = _make_valid_tpch_result()
+        data["summary"]["queries"] = {"total": 6, "passed": 5, "failed": 0, "skipped": 1}
+        report = ResultIntegrityValidator().validate(data)
+        check = next(c for c in report.checks if c.name == "query_count_math")
+        assert check.status == CheckStatus.PASS
+
+    def test_skipped_queries_discounted_from_success_rate(self) -> None:
+        """5/5 billable with 1 compat skip passes a 1.0 floor."""
+        data = _make_valid_tpch_result()
+        data["summary"]["queries"] = {"total": 6, "passed": 5, "failed": 0, "skipped": 1}
+        report = ResultIntegrityValidator().validate(data)
+        check = next(c for c in report.checks if c.name == "success_rate")
+        assert check.status == CheckStatus.PASS

tests/unit/core/ssb/test_ssb_family_plugin.py

@@ -91,4 +91,4 @@ def test_registry_wrapper_and_loader_counts_are_unchanged() -> None:
     assert summary["total"] == 23
     assert summary["loader"] == len(list_loader_benchmark_ids()) == 23
     assert summary["public"] == len(list_public_benchmark_ids()) == 22
-    assert summary["support_status"]["stable"] == 5
+    assert summary["support_status"]["stable"] == 6

tests/unit/core/test_benchmark_api_contract.py

@@ -51,12 +51,12 @@
     "Benchmark API snapshot: **23** registry entries; **23** loader-resolved core families; "
     "**22** public discovery entries; **21** top-level Python benchmark facades; "
     "**15** lazy facades; **6** eager facades; **2** core-only benchmark IDs. "
-    "Benchmark support status: **5** stable, **12** beta, **5** experimental, **1** repo-only, "
+    "Benchmark support status: **6** stable, **11** beta, **5** experimental, **1** repo-only, "
     "**0** deprecated, **0** document-only."
 )
 BENCHMARK_SUPPORT_STATUS_COUNTS = {
-    "stable": 5,
-    "beta": 12,
+    "stable": 6,
+    "beta": 11,
     "experimental": 5,
     "repo_only": 1,
     "deprecated": 0,
@@ -241,8 +241,8 @@ def test_benchmark_support_status_metadata_matches_contract_map() -> None:
         get_benchmarks_by_support_status("unknown")  # type: ignore[arg-type]
 
     contract_doc = PUBLIC_CONTRACTS_DOC.read_text()
-    assert "Benchmark support status: **5** stable, **12** beta, **5** experimental" in contract_doc
-    assert "support status counts are stable=5, beta=12, experimental=5" in contract_doc
+    assert "Benchmark support status: **6** stable, **11** beta, **5** experimental" in contract_doc
+    assert "support status counts are stable=6, beta=11, experimental=5" in contract_doc
 
 
 def test_external_dataset_stable_promotion_requires_complete_content_pins(tmp_path: Path) -> None: