diff --git a/benchbox/core/benchmark_registry.yaml b/benchbox/core/benchmark_registry.yaml index c10a3db75d..ab34551fcd 100644 --- a/benchbox/core/benchmark_registry.yaml +++ b/benchbox/core/benchmark_registry.yaml @@ -586,7 +586,7 @@ benchmark_metadata: display_name: Vector Search description: Vector similarity search benchmark (kNN, ANN, filtered) category: AI/ML - support_status: beta + support_status: stable num_queries: 6 query_description: 6 queries (kNN cosine/L2, filtered, recall@k, ANN, multi-category) supports_streams: false diff --git a/benchbox/core/results/benchmark_specs.yaml b/benchbox/core/results/benchmark_specs.yaml index 283e50db7f..624f232b9e 100644 --- a/benchbox/core/results/benchmark_specs.yaml +++ b/benchbox/core/results/benchmark_specs.yaml @@ -835,3 +835,20 @@ benchmark_specs: requires_tables_object: false sf1_row_counts: null sf1_power_at_size_range: null + vector_search: + benchmark_id: vector_search + unique_query_ids: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + min_unique_queries: 0 + min_success_rate: 1.0 + high_failure_expected: false + requires_tables_object: true + sf1_row_counts: + vectors: 1000000 + vector_queries: 100 + sf1_power_at_size_range: null diff --git a/benchbox/core/results/integrity_validator.py b/benchbox/core/results/integrity_validator.py index d147b66bab..f7e1fc1dc6 100644 --- a/benchbox/core/results/integrity_validator.py +++ b/benchbox/core/results/integrity_validator.py @@ -264,16 +264,22 @@ def _check_query_count_math(self, raw: dict[str, Any], checks: list[CheckResult] total = sq.get("total", 0) passed = sq.get("passed", 0) failed = sq.get("failed", 0) - if passed + failed != total: + skipped = sq.get("skipped", 0) + if passed + failed + skipped != total: _fail( checks, _STRUCTURAL, "query_count_math", - f"{passed} passed + {failed} failed != {total} total", - {"passed": passed, "failed": failed, "total": total}, + f"{passed} passed + {failed} failed + {skipped} skipped != {total} total", + {"passed": passed, "failed": failed, "skipped": skipped, "total": total}, ) else: - _pass(checks, _STRUCTURAL, "query_count_math", f"{total} = {passed} passed + {failed} failed") + _pass( + checks, + _STRUCTURAL, + "query_count_math", + f"{total} = {passed} passed + {failed} failed + {skipped} skipped", + ) def _check_timing_non_negative(self, raw: dict[str, Any], checks: list[CheckResult]) -> None: timing = raw.get("summary", {}).get("timing", {}) @@ -500,11 +506,19 @@ def _check_success_rate(self, raw: dict[str, Any], spec: Any | None, checks: lis sq = raw.get("summary", {}).get("queries", {}) total = sq.get("total", 0) passed = sq.get("passed", 0) + skipped = sq.get("skipped", 0) if total == 0: _fail(checks, _BELIEVABILITY, "success_rate", "No queries recorded") return - rate = passed / total + # Compatibility-skipped queries (version-gated rules such as the + # StarRocks Q2 l2_distance skip) are not failures: discount them + # from both sides before applying the spec floor. + billable = total - skipped + if billable == 0: + _pass(checks, _BELIEVABILITY, "success_rate", "All queries skipped by compatibility rules") + return + rate = passed / billable if spec is not None and spec.high_failure_expected: _pass( checks, @@ -521,10 +535,15 @@ def _check_success_rate(self, raw: dict[str, Any], spec: Any | None, checks: lis _BELIEVABILITY, "success_rate", f"{rate:.1%} success rate below {floor:.0%} floor", - {"rate": rate, "floor": floor, "passed": passed, "total": total}, + {"rate": rate, "floor": floor, "passed": passed, "total": billable, "skipped": skipped}, ) else: - _pass(checks, _BELIEVABILITY, "success_rate", f"{rate:.1%} >= {floor:.0%} floor") + _pass( + checks, + _BELIEVABILITY, + "success_rate", + f"{rate:.1%} >= {floor:.0%} floor ({skipped} skipped)", + ) def _check_sf1_row_counts( self, diff --git a/docs/benchmarks/support-status.md b/docs/benchmarks/support-status.md index 8203fd870f..bdd1d53dbd 100644 --- a/docs/benchmarks/support-status.md +++ b/docs/benchmarks/support-status.md @@ -73,7 +73,7 @@ integrity spec, query count, dataset/source, scales, and DataFrame capability. | `tsbs_devops` | `beta` | 18-query time-series DevOps workload; generated; integrity spec; docs; DataFrame-capable. | Time-series engine coverage breadth still expanding. | | `nyctaxi` | `beta` | 25-query real-world OLAP; external NYC TLC data; integrity spec; docs; DataFrame-capable. | External-dataset availability and a pinned source contract before `stable`. | | `flightdata` | `beta` | 20-query real-world aviation OLAP; external BTS data; integrity spec; docs; DataFrame-capable. | External-dataset availability and a pinned source contract before `stable`. | -| `vector_search` | `beta` | 6-query vector similarity; synthetic embeddings; docs; not DataFrame-capable. | **No integrity spec** and ANN-recall validation still pending; capability is SQL-only. | +| `vector_search` | `stable` | 6-query vector similarity; synthetic embeddings; integrity spec; docs; not DataFrame-capable (SQL-only by recorded routing decision, `supports_dataframe: false`). Q5 measures the same brute-force kNN SQL as Q1: no loader or adapter builds an HNSW index today, so ANN-index acceleration is not part of the measured workload. | None — maintain. | | `tpcds_obt` | `experimental` | 17-query denormalized One-Big-Table; derived from `tpcds`; integrity spec; docs; SF1 only; DataFrame-capable. | Single-scale subset of TPC-DS; still on the deprecated core base. | | `ai_primitives` | `experimental` | 16-query SQL AI functions; derived from TPC-H text; docs; core-only ID; not DataFrame-capable. | Cloud-only AI functions, cost-gated, no integrity spec; intentionally research-only. | | `tpchavoc` | `experimental` | 220-variant optimizer stress (22 queries × 10 syntax variants); generated; integrity spec; docs; DataFrame-capable. | Optimizer-stress research tool, not a standard comparison workload. | @@ -86,10 +86,6 @@ integrity spec, query count, dataset/source, scales, and DataFrame capability. Recorded as explicit follow-up work rather than reviewer memory. None of these are applied here: this matrix documents status, it does not change it. -- **`vector_search` (beta → stable candidate, blocked):** the only missing piece - for the integrity dimension is a `BenchmarkSpec`. Adding one (and a DataFrame - routing decision) is the concrete blocker. Tracked as a deferred follow-up on - `benchmark-support-status-criteria-matrix`. - **External-dataset betas (`nyctaxi`, `flightdata`):** promotion is gated on a complete set of reviewed SHA-256 pins for the default external corpus, not on query coverage. Until those maps are populated, generated corpora record diff --git a/docs/development/result-integrity-validation.md b/docs/development/result-integrity-validation.md index 4882dd5d57..cd5a89bfbc 100644 --- a/docs/development/result-integrity-validation.md +++ b/docs/development/result-integrity-validation.md @@ -119,7 +119,7 @@ class BenchmarkSpec: ### Coverage -Specs exist for 20 of 22 registered benchmarks: `tpch`, `tpcds`, `tpchavoc`, `tpch_skew`, `ssb`, `clickbench`, `nyctaxi`, `h2odb`, `amplab`, `joinorder`, `flightdata`, `datavault`, `coffeeshop`, `tpcdi`, `tsbs_devops`, `tpcds_obt`, `read_primitives`, `write_primitives`, `metadata_primitives`, `transaction_primitives`. `ai_primitives` and `vector_search` do not yet have integrity specs. +Specs exist for 21 of 22 registered benchmarks: `tpch`, `tpcds`, `tpchavoc`, `tpch_skew`, `ssb`, `clickbench`, `nyctaxi`, `h2odb`, `amplab`, `joinorder`, `flightdata`, `datavault`, `coffeeshop`, `tpcdi`, `tsbs_devops`, `tpcds_obt`, `read_primitives`, `write_primitives`, `metadata_primitives`, `transaction_primitives`, `vector_search`. `ai_primitives` does not yet have an integrity spec. 8 legacy aliases are mapped automatically (e.g., `star_schema` -> `ssb`, `amplab_big_data` -> `amplab`). diff --git a/docs/reference/cli/utilities.md b/docs/reference/cli/utilities.md index bdd50bde4f..a32a70b414 100644 --- a/docs/reference/cli/utilities.md +++ b/docs/reference/cli/utilities.md @@ -162,7 +162,7 @@ The validator runs 20 checks in three tiers. See the [Result Integrity Validatio **Notes:** - Automatically excludes `.plans.json` and `.tuning.json` companion files -- Specs cover 20 of 22 benchmarks with 8 legacy alias mappings (e.g., `star_schema` → `ssb`); `ai_primitives` and `vector_search` are not yet covered +- Specs cover 21 of 22 benchmarks with 8 legacy alias mappings (e.g., `star_schema` → `ssb`); `ai_primitives` is not yet covered ## Related diff --git a/docs/reference/public-contracts.md b/docs/reference/public-contracts.md index 97787c5060..09e2a9bb40 100644 --- a/docs/reference/public-contracts.md +++ b/docs/reference/public-contracts.md @@ -142,13 +142,13 @@ separate `surface`/removal decision. ## Count and Drift Policy -Benchmark API snapshot: **23** registry entries; **23** loader-resolved core families; **22** public discovery entries; **21** top-level Python benchmark facades; **15** lazy facades; **6** eager facades; **2** core-only benchmark IDs. Benchmark support status: **5** stable, **12** beta, **5** experimental, **1** repo-only, **0** deprecated, **0** document-only. +Benchmark API snapshot: **23** registry entries; **23** loader-resolved core families; **22** public discovery entries; **21** top-level Python benchmark facades; **15** lazy facades; **6** eager facades; **2** core-only benchmark IDs. Benchmark support status: **6** stable, **11** beta, **5** experimental, **1** repo-only, **0** deprecated, **0** document-only. Evidence snapshot updated by `benchmark-support-status-and-discovery-policy`: | Source | Current evidence | Contract implication | |---|---|---| -| `benchbox.core.benchmark_registry` | 23 benchmark metadata entries and 23 loader-resolved IDs; support status counts are stable=5, beta=12, experimental=5, repo_only=1, deprecated=0, document_only=0. | Benchmark count and support claims must derive from registry metadata or avoid exact counts. | +| `benchbox.core.benchmark_registry` | 23 benchmark metadata entries and 23 loader-resolved IDs; support status counts are stable=6, beta=11, experimental=5, repo_only=1, deprecated=0, document_only=0. | Benchmark count and support claims must derive from registry metadata or avoid exact counts. | | `benchbox.core.platform_registry.PlatformRegistry.get_all_platform_metadata()` | 52 platform metadata entries: 48 SQL-capable, 18 DataFrame-capable, 14 dual-mode. | README and platform docs must not carry unqualified hand-maintained platform counts. | | `benchbox.core.results.schema_policy` | Current result schema version: `2.2`; runtime/explorer accepted versions: `2.0`, `2.1`, `2.2`; public submission accepts numeric `2.x`. | Result schema version claims must update with the named consumer policy or defer to this policy module. | | `README.md` before this TODO | Landing-page bullets claimed 22 benchmarks, 42 SQL platforms, and 9 DataFrame platforms. | Exact counts were stale relative to registry metadata; README now links to this policy instead of being authoritative. | diff --git a/tests/unit/core/results/test_integrity_validator.py b/tests/unit/core/results/test_integrity_validator.py index c8ae379a2c..d0b5ab0cda 100644 --- a/tests/unit/core/results/test_integrity_validator.py +++ b/tests/unit/core/results/test_integrity_validator.py @@ -6,6 +6,7 @@ import glob import json import os +from datetime import datetime from pathlib import Path from typing import Any @@ -20,6 +21,8 @@ validate_directory, validate_file, ) +from benchbox.core.results.models import BenchmarkResults +from benchbox.core.results.schema import build_result_payload pytestmark = [ pytest.mark.unit, @@ -453,3 +456,80 @@ def test_validate_file_bad_json(self, tmp_path: Path) -> None: report = validate_file(bad_file) assert report.overall_status == CheckStatus.FAIL assert report.checks[0].name == "file_readable" + + +# --------------------------------------------------------------------------- +# TestVectorSearchExportIntegrity +# --------------------------------------------------------------------------- + + +def _make_vector_search_result() -> BenchmarkResults: + """Build a complete vector_search run using producer-style Q1-Q6 IDs.""" + query_results = [ + { + "query_id": f"Q{i}", + "execution_time_ms": 100.0 + i * 10, + "rows_returned": 10, + "status": "SUCCESS", + "iteration": 1, + "stream_id": 0, + "run_type": "measurement", + } + for i in range(1, 7) + ] + return BenchmarkResults( + benchmark_name="vector_search", + platform="duckdb", + scale_factor=1.0, + execution_id="vector-001", + timestamp=datetime(2026, 1, 1), + duration_seconds=5.0, + total_queries=6, + successful_queries=6, + failed_queries=0, + query_results=query_results, + table_statistics={"vectors": 1000000, "vector_queries": 100}, + execution_metadata={ + "mode": "standard", + "phase_status": {"power_test": {"status": "COMPLETED"}}, + }, + ) + + +class TestVectorSearchExportIntegrity: + """A complete vector_search export must pass the expected-query-IDs check. + + Schema-v2 export normalizes producer IDs Q1-Q6 to 1-6, so the spec must + record the normalized form or every complete run reports 0/6 and fails. + """ + + def test_export_normalizes_query_ids(self) -> None: + """Export converts Q1-Q6 producer IDs to normalized 1-6 IDs.""" + payload = build_result_payload(_make_vector_search_result()) + assert [q["id"] for q in payload["queries"]] == ["1", "2", "3", "4", "5", "6"] + + def test_complete_export_passes_expected_query_ids(self) -> None: + """All six exported queries match the spec set (6/6, no FAIL).""" + payload = build_result_payload(_make_vector_search_result()) + payload["export"] = {"format": "json"} + report = ResultIntegrityValidator().validate(payload) + check = next(c for c in report.checks if c.name == "expected_query_ids") + assert check.status == CheckStatus.PASS + assert check.message.startswith("6/6 expected query IDs found") + assert report.overall_status == CheckStatus.PASS + + def test_skipped_queries_satisfy_count_math(self) -> None: + """A version-gated skip (StarRocks Q2) must not fail count arithmetic.""" + data = _make_valid_tpch_result() + data["summary"]["queries"] = {"total": 6, "passed": 5, "failed": 0, "skipped": 1} + report = ResultIntegrityValidator().validate(data) + check = next(c for c in report.checks if c.name == "query_count_math") + assert check.status == CheckStatus.PASS + + def test_skipped_queries_discounted_from_success_rate(self) -> None: + """5/5 billable with 1 compat skip passes a 1.0 floor.""" + data = _make_valid_tpch_result() + data["summary"]["queries"] = {"total": 6, "passed": 5, "failed": 0, "skipped": 1} + report = ResultIntegrityValidator().validate(data) + check = next(c for c in report.checks if c.name == "success_rate") + assert check.status == CheckStatus.PASS diff --git a/tests/unit/core/ssb/test_ssb_family_plugin.py b/tests/unit/core/ssb/test_ssb_family_plugin.py index b94706b820..8ecdbb6119 100644 --- a/tests/unit/core/ssb/test_ssb_family_plugin.py +++ b/tests/unit/core/ssb/test_ssb_family_plugin.py @@ -91,4 +91,4 @@ def test_registry_wrapper_and_loader_counts_are_unchanged() -> None: assert summary["total"] == 23 assert summary["loader"] == len(list_loader_benchmark_ids()) == 23 assert summary["public"] == len(list_public_benchmark_ids()) == 22 - assert summary["support_status"]["stable"] == 5 + assert summary["support_status"]["stable"] == 6 diff --git a/tests/unit/core/test_benchmark_api_contract.py b/tests/unit/core/test_benchmark_api_contract.py index c072acf349..14a347aa58 100644 --- a/tests/unit/core/test_benchmark_api_contract.py +++ b/tests/unit/core/test_benchmark_api_contract.py @@ -51,12 +51,12 @@ "Benchmark API snapshot: **23** registry entries; **23** loader-resolved core families; " "**22** public discovery entries; **21** top-level Python benchmark facades; " "**15** lazy facades; **6** eager facades; **2** core-only benchmark IDs. " - "Benchmark support status: **5** stable, **12** beta, **5** experimental, **1** repo-only, " + "Benchmark support status: **6** stable, **11** beta, **5** experimental, **1** repo-only, " "**0** deprecated, **0** document-only." ) BENCHMARK_SUPPORT_STATUS_COUNTS = { - "stable": 5, - "beta": 12, + "stable": 6, + "beta": 11, "experimental": 5, "repo_only": 1, "deprecated": 0, @@ -241,8 +241,8 @@ def test_benchmark_support_status_metadata_matches_contract_map() -> None: get_benchmarks_by_support_status("unknown") # type: ignore[arg-type] contract_doc = PUBLIC_CONTRACTS_DOC.read_text() - assert "Benchmark support status: **5** stable, **12** beta, **5** experimental" in contract_doc - assert "support status counts are stable=5, beta=12, experimental=5" in contract_doc + assert "Benchmark support status: **6** stable, **11** beta, **5** experimental" in contract_doc + assert "support status counts are stable=6, beta=11, experimental=5" in contract_doc def test_external_dataset_stable_promotion_requires_complete_content_pins(tmp_path: Path) -> None: