diff --git a/benchbox/cli/orchestrator.py b/benchbox/cli/orchestrator.py index 48dfc85ef5..7204bc195e 100644 --- a/benchbox/cli/orchestrator.py +++ b/benchbox/cli/orchestrator.py @@ -151,10 +151,14 @@ def _get_benchmark_instance(self, config: BenchmarkConfig, system_profile): # the get_data_source_benchmark() instance method (no # DATA_SOURCE_BENCHMARK class attribute) were not resolved at # construction, so redirect them to the shared root now. Class-attr - # sharers were already constructed with it. + # sharers were already constructed with it. Benchmarks that generate + # their own output (e.g. tpcds_obt, which transforms source data into + # a separate OBT table) must keep their own output_dir: redirecting + # would hide their generated files from the loader. if getattr(benchmark_class, "DATA_SOURCE_BENCHMARK", None) is None: data_source = getattr(benchmark_instance, "get_data_source_benchmark", lambda: None)() - if data_source and self.custom_output_dir is None: + generates_own_output = getattr(benchmark_instance, "GENERATES_OWN_OUTPUT", False) + if data_source and self.custom_output_dir is None and not generates_own_output: shared_path = self.directory_manager.get_datagen_path(data_source.lower(), config.scale_factor) benchmark_instance.output_dir = shared_path diff --git a/benchbox/core/tpcds_obt/benchmark.py b/benchbox/core/tpcds_obt/benchmark.py index d07aa2f795..7cb5c46f27 100644 --- a/benchbox/core/tpcds_obt/benchmark.py +++ b/benchbox/core/tpcds_obt/benchmark.py @@ -127,6 +127,11 @@ class TPCDSOBTBenchmark(BaseBenchmark): while OBT-specific transformations are stored separately. """ + # OBT generates its own output table from TPC-DS source data, so the CLI + # must not redirect its output_dir to the shared TPC-DS root: that would + # hide the generated OBT parquet from the cloud loader. + GENERATES_OWN_OUTPUT = True + def __init__( self, scale_factor: float = 1.0, diff --git a/benchbox/core/transaction_primitives/benchmark.py b/benchbox/core/transaction_primitives/benchmark.py index e1afc54c28..7529cd5d98 100644 --- a/benchbox/core/transaction_primitives/benchmark.py +++ b/benchbox/core/transaction_primitives/benchmark.py @@ -29,6 +29,10 @@ from benchbox.core.transaction_primitives.operations import TransactionOperationsManager from benchbox.core.transaction_primitives.schema import STAGING_TABLES, get_all_staging_tables_sql, get_create_table_sql from benchbox.core.transactional.benchmark_base import TransactionalBenchmarkBase +from benchbox.sql_compat.rules.execution_filter.cloud_transaction_primitives import ( + BIGQUERY_TRANSACTION_PRIMITIVES_OPERATION_SKIPS, + SNOWFLAKE_TRANSACTION_PRIMITIVES_OPERATION_SKIPS, +) from benchbox.sql_compat.rules.execution_filter.databricks_transaction_primitives import ( DATABRICKS_TRANSACTION_PRIMITIVES_OPERATION_SKIPS, ) @@ -708,8 +712,10 @@ def execute_operation( try: platform_operation_skips = { + "bigquery": BIGQUERY_TRANSACTION_PRIMITIVES_OPERATION_SKIPS, "databricks": DATABRICKS_TRANSACTION_PRIMITIVES_OPERATION_SKIPS, "duckdb": DUCKDB_TRANSACTION_PRIMITIVES_OPERATION_SKIPS, + "snowflake": SNOWFLAKE_TRANSACTION_PRIMITIVES_OPERATION_SKIPS, "pg_duckdb": PG_DUCKDB_TRANSACTION_PRIMITIVES_OPERATION_SKIPS, "timescaledb": TIMESCALEDB_TRANSACTION_PRIMITIVES_OPERATION_SKIPS, } diff --git a/benchbox/core/transaction_primitives/catalog/operations.yaml b/benchbox/core/transaction_primitives/catalog/operations.yaml index 337114e9a9..8ffa84373b 100644 --- a/benchbox/core/transaction_primitives/catalog/operations.yaml +++ b/benchbox/core/transaction_primitives/catalog/operations.yaml @@ -235,6 +235,9 @@ operations: expected_rows: 1 cleanup_sql: "DELETE FROM txn_orders WHERE o_orderkey BETWEEN 9990001 AND 9990010\n" expected_rows_affected: 10 + platform_overrides: + snowflake: "BEGIN TRANSACTION;\nINSERT INTO txn_orders\nWITH order_data AS (\n SELECT 9990000 + n AS o_orderkey,\n 1 AS o_custkey,\n 'O' AS o_orderstatus,\n 1000.0 * n AS o_totalprice,\n DATE '1998-01-01' AS o_orderdate,\n '5-LOW' AS o_orderpriority,\n 'Clerk#000000001' AS o_clerk,\n 0 AS o_shippriority,\n 'cte_test' AS o_comment\n FROM (SELECT unnest(generate_series(1, 10)) AS n) t\n)\nSELECT * FROM order_data;\nUPDATE txn_orders SET o_totalprice = o_totalprice * 2 WHERE o_orderkey BETWEEN 9990001 AND 9990010;\nCOMMIT;\n" + bigquery: "BEGIN TRANSACTION;\nINSERT INTO txn_orders\nWITH order_data AS (\n SELECT 9990000 + n AS o_orderkey,\n 1 AS o_custkey,\n 'O' AS o_orderstatus,\n 1000.0 * n AS o_totalprice,\n DATE '1998-01-01' AS o_orderdate,\n '5-LOW' AS o_orderpriority,\n 'Clerk#000000001' AS o_clerk,\n 0 AS o_shippriority,\n 'cte_test' AS o_comment\n FROM (SELECT unnest(generate_series(1, 10)) AS n) t\n)\nSELECT * FROM order_data;\nUPDATE txn_orders SET o_totalprice = o_totalprice * 2 WHERE o_orderkey BETWEEN 9990001 AND 9990010;\nCOMMIT;\n" - id: transaction_nested_subquery_updates category: advanced description: Tests UPDATE with nested subqueries within transaction diff --git a/benchbox/core/transactional/benchmark_base.py b/benchbox/core/transactional/benchmark_base.py index 9ae0a8c310..b0831eadd9 100644 --- a/benchbox/core/transactional/benchmark_base.py +++ b/benchbox/core/transactional/benchmark_base.py @@ -32,6 +32,18 @@ # SQL ROLLBACK is needed to close a transaction opened by catalog BEGIN SQL. _SQL_ROLLBACK_AFTER_ERROR_DIALECTS = frozenset({"databricks"}) _UNNEST_GENERATE_SERIES_RE = re.compile(r"unnest\(\s*generate_series\((?P[^()]*)\)\s*\)", re.IGNORECASE) +# The catalog writes integer series as ``(SELECT unnest(generate_series(a, b)) AS n) t``. +# Snowflake and BigQuery have neither function (verified live: "Unknown functions +# GENERATE_SERIES, UNNEST" and "Unexpected keyword UNNEST"); rewrite the whole +# derived-table body into each engine's series generator. +_UNNEST_SERIES_SELECT_RE = re.compile( + r"SELECT\s+unnest\(\s*generate_series\(\s*(?P[^,()]+?)\s*,\s*(?P[^,()]+?)\s*\)\s*\)\s+AS\s+(?P\w+)", + re.IGNORECASE, +) +_SERIES_SELECT_TEMPLATES = { + "snowflake": "SELECT value::INT AS {alias} FROM TABLE(FLATTEN(ARRAY_GENERATE_RANGE({lo}, {hi} + 1)))", + "bigquery": "SELECT {alias} FROM UNNEST(GENERATE_ARRAY({lo}, {hi})) AS {alias}", +} _SET_THEN_BEGIN_ISOLATION_RE = re.compile( r"SET\s+TRANSACTION\s+ISOLATION\s+LEVEL\s+(?PREPEATABLE\s+READ|SERIALIZABLE|READ\s+COMMITTED)\s*;\s*" r"BEGIN\s+TRANSACTION\s*;", @@ -337,7 +349,13 @@ def _lookup_platform_override( def _rewrite_transactional_sql_for_platform(self, sql: str, platform_key: str | None) -> str: """Apply narrow transactional catalog rewrites for the active SQL dialect.""" - if (platform_key or "").lower() not in _POSTGRES_SERIES_DIALECTS: + dialect = (platform_key or "").lower() + template = _SERIES_SELECT_TEMPLATES.get(dialect) + if template is not None: + return _UNNEST_SERIES_SELECT_RE.sub( + lambda m: template.format(lo=m.group("lo"), hi=m.group("hi"), alias=m.group("alias")), sql + ) + if dialect not in _POSTGRES_SERIES_DIALECTS: return sql sql = _UNNEST_GENERATE_SERIES_RE.sub(r"generate_series(\g)", sql) return _SET_THEN_BEGIN_ISOLATION_RE.sub(r"BEGIN TRANSACTION ISOLATION LEVEL \g;", sql) diff --git a/benchbox/platforms/bigquery.py b/benchbox/platforms/bigquery.py index 1e18f0806e..3e420bca7a 100644 --- a/benchbox/platforms/bigquery.py +++ b/benchbox/platforms/bigquery.py @@ -695,6 +695,7 @@ def preprocess_operation_sql(self, query_id: str, operation: Any) -> str | None: flags=re.IGNORECASE, ) rewritten = re.sub(r"\bINTERVAL\s+'(\d+)'\s+([A-Za-z]+)", r"INTERVAL \1 \2", rewritten) + rewritten = self._numeric_decimal_literals(rewritten) statements = split_sql_statements(rewritten) parts = [] changed = False @@ -715,6 +716,25 @@ def preprocess_operation_sql(self, query_id: str, operation: Any) -> str | None: rewritten = rewritten[: bare_insert.start()] + bare_insert.group(1) + "INSERT ROW" + bare_insert.group(2) return rewritten + @staticmethod + def _numeric_decimal_literals(sql: str) -> str: + """Type bare decimal literals in operation SQL as NUMERIC. + + BigQuery types ``1000.0`` as FLOAT64 and refuses to insert FLOAT64 into + the NUMERIC money and discount columns of the TPC-H staging tables + (verified live: "Query column 4 has type FLOAT64 which cannot be + inserted into column o_totalprice, which has type NUMERIC"). Every + decimal literal in the write catalogs targets those columns, so + ``NUMERIC '1000.0'`` keeps the arithmetic exact and the insert legal. + String literals and identifiers are left untouched. + """ + import re + + parts = re.split(r"('(?:[^'\\]|\\.|'')*')", sql) + for index in range(0, len(parts), 2): + parts[index] = re.sub(r"(? dict[str, Any]: """Get standardized connection parameters.""" return { @@ -2307,6 +2327,8 @@ def _qualify_single_statement(self, statement: str, allow_fallback: bool = True) if not allow_fallback: return statement table_names = list(self._FALLBACK_QUALIFY_TABLES) + if self._batch_temp_tables: + table_names = [name for name in table_names if name not in self._batch_temp_tables] # Blank string literals and comments length-preservingly so matches # found in the masked copy align with the original statement. @@ -2403,7 +2425,27 @@ def _qualify_table_names(self, query: str, allow_fallback: bool = True) -> str: if not statements: return query - qualified_parts = [self._qualify_single_statement(stmt, allow_fallback=allow_fallback) for stmt in statements] + # Script-scoped temporary tables live outside the dataset, so their + # names must stay bare everywhere in the batch (verified live: a + # qualified temp-table name fails "Not found: Table ...TEMP_..."). + import re + + temp_tables = { + m.group(1).upper() + for stmt in statements + for m in re.finditer( + r"(?i)\bCREATE\s+(?:OR\s+REPLACE\s+)?TEMP(?:ORARY)?\s+TABLE\s+(?:IF\s+NOT\s+EXISTS\s+)?`?(\w+)`?", + stmt, + ) + } + previous_temp_tables = self._batch_temp_tables + self._batch_temp_tables = temp_tables + try: + qualified_parts = [ + self._qualify_single_statement(stmt, allow_fallback=allow_fallback) for stmt in statements + ] + finally: + self._batch_temp_tables = previous_temp_tables result = ";\n".join(qualified_parts) if query.rstrip().endswith(";"): result += ";" @@ -2412,6 +2454,8 @@ def _qualify_table_names(self, query: str, allow_fallback: bool = True) -> str: return result _FROM_SCAN_TOKEN = None + # Temp-table names created in the batch currently being qualified. + _batch_temp_tables: frozenset[str] | set[str] = frozenset() @staticmethod def _from_scan_token(): diff --git a/benchbox/platforms/databricks/adapter.py b/benchbox/platforms/databricks/adapter.py index e07f478dae..9b1eb4e3ef 100644 --- a/benchbox/platforms/databricks/adapter.py +++ b/benchbox/platforms/databricks/adapter.py @@ -1972,7 +1972,7 @@ def load_data( existing_tables, data_source, ) - table_stats[table_name.upper()] = row_count + table_stats[table_name.lower()] = row_count load_time = elapsed_seconds(load_start) per_table_timings[table_name.upper()] = { @@ -1986,7 +1986,7 @@ def load_data( except Exception as e: self.logger.error(f"Failed to load {table_name}: {str(e)[:200]}") - table_stats[table_name.upper()] = 0 + table_stats[table_name.lower()] = 0 per_table_timings[table_name.upper()] = { "copy_into_ms": 0, "optimize_ms": 0, @@ -2529,7 +2529,7 @@ def create_external_tables( cursor.execute(f"CREATE TABLE {table_name_upper} USING PARQUET LOCATION '{location}'") cursor.execute(f"SELECT COUNT(*) FROM {table_name_upper}") result = cursor.fetchone() - table_stats[table_name_upper] = int(result[0]) if result else 0 + table_stats[table_name_lower] = int(result[0]) if result else 0 finally: cursor.close() diff --git a/benchbox/sql_compat/rules/execution_filter/cloud_transaction_primitives.py b/benchbox/sql_compat/rules/execution_filter/cloud_transaction_primitives.py new file mode 100644 index 0000000000..b50fd551bb --- /dev/null +++ b/benchbox/sql_compat/rules/execution_filter/cloud_transaction_primitives.py @@ -0,0 +1,57 @@ +"""Snowflake and BigQuery execution-filter rules for transaction_primitives operation gaps.""" + +from __future__ import annotations + +from benchbox.sql_compat.actions import CompatAction +from benchbox.sql_compat.context import Phase +from benchbox.sql_compat.decision import CompatibilityDecision, FailureMode, SkipQueryPayload, SupportLevel +from benchbox.sql_compat.registry import REGISTRY + +_SAVEPOINT = "{engine} has no SAVEPOINT statement." +_ISOLATION = "{engine} rejects SET TRANSACTION ISOLATION LEVEL." +_TEMP_TABLE = ( + "BigQuery runs each multi-statement script as chained single-statement jobs, " + "so a TEMP TABLE created in one statement is invisible to the next." +) + +_OPERATIONS = { + "transaction_savepoint_nested": _SAVEPOINT, + "transaction_savepoint_deep_nesting": _SAVEPOINT, + "transaction_isolation_read_committed": _ISOLATION, + "transaction_isolation_repeatable_read": _ISOLATION, + "transaction_isolation_serializable": _ISOLATION, +} + +_BIGQUERY_ONLY_OPERATIONS = { + "transaction_create_temp_table": _TEMP_TABLE, +} + +SNOWFLAKE_TRANSACTION_PRIMITIVES_OPERATION_SKIPS = { + query_id: reason.format(engine="Snowflake") for query_id, reason in _OPERATIONS.items() +} +BIGQUERY_TRANSACTION_PRIMITIVES_OPERATION_SKIPS = { + query_id: reason.format(engine="BigQuery") for query_id, reason in _OPERATIONS.items() +} | dict(_BIGQUERY_ONLY_OPERATIONS) + +for _platform, _skips in ( + ("snowflake", SNOWFLAKE_TRANSACTION_PRIMITIVES_OPERATION_SKIPS), + ("bigquery", BIGQUERY_TRANSACTION_PRIMITIVES_OPERATION_SKIPS), +): + for _query_id, _reason in _skips.items(): + REGISTRY.register( + CompatibilityDecision( + rule_id=f"execution_filter.{_platform}.transaction_primitives.{_query_id}", + action=CompatAction.SKIP_QUERY, + support_level=SupportLevel.SKIPPED_QUERY, + failure_mode=FailureMode.UNSUPPORTED_FEATURE, + payload=SkipQueryPayload( + reason=f"{_reason} Evidence: live {_platform} probe on 2026-09-26.", + query_id=_query_id, + ), + reason=_reason, + ), + Phase.EXECUTION_FILTER, + _platform, + benchmark="transaction_primitives", + query_id=_query_id, + ) diff --git a/docs/compat/capability-matrix.md b/docs/compat/capability-matrix.md index b529e12551..2c442454a1 100644 --- a/docs/compat/capability-matrix.md +++ b/docs/compat/capability-matrix.md @@ -4,7 +4,7 @@ Every rule registered in `benchbox.sql_compat` is listed below. The registry is the authoritative source of compatibility policy; this document is regenerated from it. See [adr-sql-compat-phase-aware-pipeline.md](../development/adr/adr-sql-compat-phase-aware-pipeline.md) for the design. -**Total registered rules:** 545 +**Total registered rules:** 556 **Platforms covered:** 35 @@ -13,7 +13,7 @@ Every rule registered in `benchbox.sql_compat` is listed below. The registry is | platform | benchmark_gate | query_source | query_adapter | schema_emit | ddl_optimize | execution_filter | total | |---|---|---|---|---|---|---|---| | athena | - | - | - | - | 1 | - | 1 | -| bigquery | - | 2 | 25 | 2 | 1 | 2 | 32 | +| bigquery | - | 2 | 25 | 2 | 1 | 8 | 38 | | clickhouse | - | 19 | 3 | 4 | 1 | - | 27 | | clickhouse-cloud | - | - | 2 | - | - | 21 | 23 | | clickhouse-local | 1 | - | 2 | - | - | 21 | 24 | @@ -37,7 +37,7 @@ Every rule registered in `benchbox.sql_compat` is listed below. The registry is | questdb | 1 | - | - | - | 1 | - | 2 | | redshift | - | - | - | 2 | 1 | - | 3 | | singlestore | - | - | - | - | 4 | - | 4 | -| snowflake | - | 7 | 2 | 2 | 1 | 2 | 14 | +| snowflake | - | 7 | 2 | 2 | 1 | 7 | 19 | | spark | - | 6 | 11 | 2 | 1 | - | 20 | | sqlite | - | 1 | - | - | - | - | 1 | | starrocks | - | 15 | 1 | 2 | 1 | - | 19 | @@ -92,6 +92,12 @@ Every rule registered in `benchbox.sql_compat` is listed below. The registry is | ddl_optimize | platform-wide | rewrite_ddl | REWRITTEN | SYNTAX_ERROR | `ddl_optimize.bigquery.all.convert_to_bigquery_table` | | execution_filter | benchmark=tpchavoc, query=1_v7 | skip_query | SKIPPED_QUERY | UNSUPPORTED_FEATURE | `execution_filter.bigquery.tpchavoc.1_v7` | | execution_filter | benchmark=tpchavoc, query=2_v2 | skip_query | SKIPPED_QUERY | UNSUPPORTED_FEATURE | `execution_filter.bigquery.tpchavoc.2_v2` | +| execution_filter | benchmark=transaction_primitives, query=transaction_create_temp_table | skip_query | SKIPPED_QUERY | UNSUPPORTED_FEATURE | `execution_filter.bigquery.transaction_primitives.transaction_create_temp_table` | +| execution_filter | benchmark=transaction_primitives, query=transaction_isolation_read_committed | skip_query | SKIPPED_QUERY | UNSUPPORTED_FEATURE | `execution_filter.bigquery.transaction_primitives.transaction_isolation_read_committed` | +| execution_filter | benchmark=transaction_primitives, query=transaction_isolation_repeatable_read | skip_query | SKIPPED_QUERY | UNSUPPORTED_FEATURE | `execution_filter.bigquery.transaction_primitives.transaction_isolation_repeatable_read` | +| execution_filter | benchmark=transaction_primitives, query=transaction_isolation_serializable | skip_query | SKIPPED_QUERY | UNSUPPORTED_FEATURE | `execution_filter.bigquery.transaction_primitives.transaction_isolation_serializable` | +| execution_filter | benchmark=transaction_primitives, query=transaction_savepoint_deep_nesting | skip_query | SKIPPED_QUERY | UNSUPPORTED_FEATURE | `execution_filter.bigquery.transaction_primitives.transaction_savepoint_deep_nesting` | +| execution_filter | benchmark=transaction_primitives, query=transaction_savepoint_nested | skip_query | SKIPPED_QUERY | UNSUPPORTED_FEATURE | `execution_filter.bigquery.transaction_primitives.transaction_savepoint_nested` | ### clickhouse @@ -641,6 +647,11 @@ Every rule registered in `benchbox.sql_compat` is listed below. The registry is | ddl_optimize | platform-wide | rewrite_ddl | REWRITTEN | SYNTAX_ERROR | `ddl_optimize.snowflake.all.optimize_table_definition` | | execution_filter | benchmark=tpchavoc, query=1_v7 | skip_query | SKIPPED_QUERY | UNSUPPORTED_FEATURE | `execution_filter.snowflake.tpchavoc.1_v7` | | execution_filter | benchmark=tpchavoc, query=2_v2 | skip_query | SKIPPED_QUERY | UNSUPPORTED_FEATURE | `execution_filter.snowflake.tpchavoc.2_v2` | +| execution_filter | benchmark=transaction_primitives, query=transaction_isolation_read_committed | skip_query | SKIPPED_QUERY | UNSUPPORTED_FEATURE | `execution_filter.snowflake.transaction_primitives.transaction_isolation_read_committed` | +| execution_filter | benchmark=transaction_primitives, query=transaction_isolation_repeatable_read | skip_query | SKIPPED_QUERY | UNSUPPORTED_FEATURE | `execution_filter.snowflake.transaction_primitives.transaction_isolation_repeatable_read` | +| execution_filter | benchmark=transaction_primitives, query=transaction_isolation_serializable | skip_query | SKIPPED_QUERY | UNSUPPORTED_FEATURE | `execution_filter.snowflake.transaction_primitives.transaction_isolation_serializable` | +| execution_filter | benchmark=transaction_primitives, query=transaction_savepoint_deep_nesting | skip_query | SKIPPED_QUERY | UNSUPPORTED_FEATURE | `execution_filter.snowflake.transaction_primitives.transaction_savepoint_deep_nesting` | +| execution_filter | benchmark=transaction_primitives, query=transaction_savepoint_nested | skip_query | SKIPPED_QUERY | UNSUPPORTED_FEATURE | `execution_filter.snowflake.transaction_primitives.transaction_savepoint_nested` | ### spark diff --git a/docs/compat/skip-reference.md b/docs/compat/skip-reference.md index eee323855d..9ae1b85fc4 100644 --- a/docs/compat/skip-reference.md +++ b/docs/compat/skip-reference.md @@ -4,7 +4,7 @@ Rules the registry applies to queries, benchmarks, and DDL statements. Split into two sections based on whether the outcome is user-visible in result counts. Each entry names the platform, the scope the rule applies to, the registered reason, and the rule_id you can grep for in `benchbox/sql_compat/rules/`. -**Total rules:** 326 +**Total rules:** 337 **Platforms with rules:** 22 @@ -18,6 +18,12 @@ Queries or benchmarks that are **omitted from the result set** - either because |---|---|---|---|---| | SKIPPED_QUERY | benchmark=tpchavoc, query=1_v7 | execution_filter | BigQuery does not support DuckDB LIST aggregates and LIST lambda functions in this variant. | `execution_filter.bigquery.tpchavoc.1_v7` | | SKIPPED_QUERY | benchmark=tpchavoc, query=2_v2 | execution_filter | BigQuery rejects the correlated scalar subquery in this grouped variant. | `execution_filter.bigquery.tpchavoc.2_v2` | +| SKIPPED_QUERY | benchmark=transaction_primitives, query=transaction_create_temp_table | execution_filter | BigQuery runs each multi-statement script as chained single-statement jobs, so a TEMP TABLE created in one statement is invisible to the next. | `execution_filter.bigquery.transaction_primitives.transaction_create_temp_table` | +| SKIPPED_QUERY | benchmark=transaction_primitives, query=transaction_isolation_read_committed | execution_filter | BigQuery rejects SET TRANSACTION ISOLATION LEVEL. | `execution_filter.bigquery.transaction_primitives.transaction_isolation_read_committed` | +| SKIPPED_QUERY | benchmark=transaction_primitives, query=transaction_isolation_repeatable_read | execution_filter | BigQuery rejects SET TRANSACTION ISOLATION LEVEL. | `execution_filter.bigquery.transaction_primitives.transaction_isolation_repeatable_read` | +| SKIPPED_QUERY | benchmark=transaction_primitives, query=transaction_isolation_serializable | execution_filter | BigQuery rejects SET TRANSACTION ISOLATION LEVEL. | `execution_filter.bigquery.transaction_primitives.transaction_isolation_serializable` | +| SKIPPED_QUERY | benchmark=transaction_primitives, query=transaction_savepoint_deep_nesting | execution_filter | BigQuery has no SAVEPOINT statement. | `execution_filter.bigquery.transaction_primitives.transaction_savepoint_deep_nesting` | +| SKIPPED_QUERY | benchmark=transaction_primitives, query=transaction_savepoint_nested | execution_filter | BigQuery has no SAVEPOINT statement. | `execution_filter.bigquery.transaction_primitives.transaction_savepoint_nested` | ### clickhouse-cloud @@ -365,6 +371,11 @@ Queries or benchmarks that are **omitted from the result set** - either because |---|---|---|---|---| | SKIPPED_QUERY | benchmark=tpchavoc, query=1_v7 | execution_filter | Snowflake does not support DuckDB LIST aggregates and LIST lambda functions in this variant. | `execution_filter.snowflake.tpchavoc.1_v7` | | SKIPPED_QUERY | benchmark=tpchavoc, query=2_v2 | execution_filter | Snowflake rejects the correlated scalar subquery in this grouped variant. | `execution_filter.snowflake.tpchavoc.2_v2` | +| SKIPPED_QUERY | benchmark=transaction_primitives, query=transaction_isolation_read_committed | execution_filter | Snowflake rejects SET TRANSACTION ISOLATION LEVEL. | `execution_filter.snowflake.transaction_primitives.transaction_isolation_read_committed` | +| SKIPPED_QUERY | benchmark=transaction_primitives, query=transaction_isolation_repeatable_read | execution_filter | Snowflake rejects SET TRANSACTION ISOLATION LEVEL. | `execution_filter.snowflake.transaction_primitives.transaction_isolation_repeatable_read` | +| SKIPPED_QUERY | benchmark=transaction_primitives, query=transaction_isolation_serializable | execution_filter | Snowflake rejects SET TRANSACTION ISOLATION LEVEL. | `execution_filter.snowflake.transaction_primitives.transaction_isolation_serializable` | +| SKIPPED_QUERY | benchmark=transaction_primitives, query=transaction_savepoint_deep_nesting | execution_filter | Snowflake has no SAVEPOINT statement. | `execution_filter.snowflake.transaction_primitives.transaction_savepoint_deep_nesting` | +| SKIPPED_QUERY | benchmark=transaction_primitives, query=transaction_savepoint_nested | execution_filter | Snowflake has no SAVEPOINT statement. | `execution_filter.snowflake.transaction_primitives.transaction_savepoint_nested` | ### starrocks diff --git a/tests/integration/platforms/test_databricks_smoke.py b/tests/integration/platforms/test_databricks_smoke.py index b2deca9f90..df4a7f7e33 100644 --- a/tests/integration/platforms/test_databricks_smoke.py +++ b/tests/integration/platforms/test_databricks_smoke.py @@ -30,7 +30,7 @@ def test_databricks_smoke_run(monkeypatch, tmp_path): stats, metadata, stub_state = run_smoke_benchmark(adapter, benchmark, tmp_path) - assert stats["LINEITEM"] == stub_state.row_counts["LINEITEM"] + assert stats["lineitem"] == stub_state.row_counts["LINEITEM"] assert any(stmt.startswith("COPY INTO LINEITEM") for stmt in stub_state.copy_statements) assert metadata["platform_name"] == "Databricks" diff --git a/tests/unit/cli/test_orchestrator_data_sharing.py b/tests/unit/cli/test_orchestrator_data_sharing.py index 09be9f0077..5df3652b6c 100644 --- a/tests/unit/cli/test_orchestrator_data_sharing.py +++ b/tests/unit/cli/test_orchestrator_data_sharing.py @@ -261,6 +261,21 @@ def test_method_only_data_sharing_falls_back_to_redirect(self): expected = orchestrator.directory_manager.get_datagen_path("tpch", 1.0) assert Path(str(benchmark.output_dir)) == Path(str(expected)) + def test_own_output_benchmark_keeps_constructed_output_dir(self): + """Benchmarks generating their own output (tpcds_obt) keep it.""" + + class _OwnOutputBenchmark(_MethodOnlySharingBenchmark): + GENERATES_OWN_OUTPUT = True + + orchestrator = BenchmarkOrchestrator() + config = BenchmarkConfig(name="tpcds_obt", display_name="Stub", scale_factor=1.0, compress_data=False) + + with patch.object(orchestrator, "_get_benchmark_class", return_value=_OwnOutputBenchmark): + benchmark = orchestrator._get_benchmark_instance(config, None) + + expected = orchestrator.directory_manager.get_datagen_path("tpcds_obt", 1.0) + assert Path(str(benchmark.output_dir)) == Path(str(expected)) + def test_cloud_custom_output_defers_resolution(self): orchestrator = BenchmarkOrchestrator() orchestrator.set_custom_output_dir("s3://bucket/prefix") diff --git a/tests/unit/core/transaction_primitives/test_execute_operation.py b/tests/unit/core/transaction_primitives/test_execute_operation.py index 2a5a3aabd1..b01dcf8ef3 100644 --- a/tests/unit/core/transaction_primitives/test_execute_operation.py +++ b/tests/unit/core/transaction_primitives/test_execute_operation.py @@ -238,6 +238,33 @@ def test_rewrites_generate_series_for_postgres(self, tmp_path: Path): assert "FROM (SELECT generate_series(1, 100) AS n) t" in executed_sql assert "unnest(generate_series" not in executed_sql + @pytest.mark.parametrize( + ("platform_key", "expected"), + [ + ("snowflake", "FROM (SELECT value::INT AS n FROM TABLE(FLATTEN(ARRAY_GENERATE_RANGE(1, 100 + 1)))) t"), + ("bigquery", "FROM (SELECT n FROM UNNEST(GENERATE_ARRAY(1, 100)) AS n) t"), + ], + ) + def test_rewrites_generate_series_for_cloud_dialects(self, tmp_path: Path, platform_key: str, expected: str): + """Snowflake and BigQuery have neither unnest nor generate_series (verified live).""" + bench = _make_benchmark(tmp_path) + bench.operations_manager.get_operation.return_value = _make_operation( + write_sql=""" + BEGIN TRANSACTION; + INSERT INTO txn_orders + SELECT 9200000 + n, 1 + FROM (SELECT unnest(generate_series(1, 100)) AS n) t; + COMMIT; + """, + ) + conn = _make_connection() + + bench.execute_operation("op1", conn, platform_key=platform_key) + + executed_sql = conn.execute.call_args.args[0] + assert expected in executed_sql + assert "generate_series" not in executed_sql + def test_rewrites_isolation_level_begin_for_postgres(self, tmp_path: Path): bench = _make_benchmark(tmp_path) bench.operations_manager.get_operation.return_value = _make_operation( diff --git a/tests/unit/platforms/test_bigquery_adapter.py b/tests/unit/platforms/test_bigquery_adapter.py index 8fddf23aba..a8ab43604f 100644 --- a/tests/unit/platforms/test_bigquery_adapter.py +++ b/tests/unit/platforms/test_bigquery_adapter.py @@ -2241,6 +2241,34 @@ def test_inline_pk_ignores_parens_in_comments(self, mock_bigquery): assert "/* note ) */" in result assert result.rstrip().endswith("PRIMARY KEY (id) NOT ENFORCED)") + @patch("benchbox.platforms.bigquery.bigquery") + def test_numeric_decimal_literals_skip_strings_and_identifiers(self, mock_bigquery): + """FLOAT64 literals cannot be inserted into NUMERIC columns (verified live).""" + sql = "UPDATE t SET p = p * 1.05, c = 'v1.5', q = 1000.0 * n WHERE k2 = 3 AND x = 0.02" + result = BigQueryAdapter._numeric_decimal_literals(sql) + assert result == ( + "UPDATE t SET p = p * NUMERIC '1.05', c = 'v1.5', q = NUMERIC '1000.0' * n " + "WHERE k2 = 3 AND x = NUMERIC '0.02'" + ) + + @patch("benchbox.platforms.bigquery.bigquery") + def test_qualify_leaves_script_temp_tables_bare(self, mock_bigquery): + """Script temp tables live outside the dataset; qualifying them fails with Not found.""" + adapter = BigQueryAdapter(project_id="proj", dataset_id="ds") + sql = ( + "BEGIN TRANSACTION;\n" + "CREATE TEMP TABLE IF NOT EXISTS temp_orders_1 AS SELECT * FROM orders WHERE 1=0;\n" + "INSERT INTO temp_orders_1 VALUES (1);\n" + "INSERT INTO txn_orders SELECT * FROM temp_orders_1;\n" + "DROP TABLE IF EXISTS temp_orders_1;\n" + "COMMIT;\n" + ) + result = adapter._qualify_table_names(sql) + assert "`proj.ds.ORDERS`" in result + assert "INSERT INTO `proj.ds.TXN_ORDERS` SELECT * FROM temp_orders_1" in result + assert "TEMP_ORDERS_1" not in result + assert adapter._batch_temp_tables == frozenset() + @patch("benchbox.platforms.bigquery.bigquery") def test_preprocess_operation_sql_rewrites_bq_gaps(self, mock_bigquery): """CAST AS VARCHAR and quoted INTERVAL literals fail server-side.""" diff --git a/tests/unit/platforms/test_databricks_adapter.py b/tests/unit/platforms/test_databricks_adapter.py index 8063912da0..698dc7e44e 100644 --- a/tests/unit/platforms/test_databricks_adapter.py +++ b/tests/unit/platforms/test_databricks_adapter.py @@ -548,8 +548,8 @@ def test_load_data_with_delta_tables(self, mock_databricks_sql): assert isinstance(table_stats, dict) assert isinstance(load_time, float) assert load_time >= 0 - assert "TEST_TABLE" in table_stats - assert table_stats["TEST_TABLE"] == 100 + assert "test_table" in table_stats + assert table_stats["test_table"] == 100 # Should execute COPY INTO statements without temporary views or insert-select execute_calls = [str(call) for call in mock_cursor.execute.call_args_list] @@ -590,7 +590,7 @@ def test_create_external_tables_uses_parquet_location(self, mock_databricks_sql) stats, load_time, _ = adapter.create_external_tables(benchmark, mock_connection, Path("dbfs:/tmp/data")) - assert stats["ORDERS"] == 123 + assert stats["orders"] == 123 assert load_time >= 0 execute_calls = [str(call) for call in mock_cursor.execute.call_args_list] diff --git a/tests/unit/platforms/test_databricks_adapter_coverage.py b/tests/unit/platforms/test_databricks_adapter_coverage.py index f72a66dece..a95bca6598 100644 --- a/tests/unit/platforms/test_databricks_adapter_coverage.py +++ b/tests/unit/platforms/test_databricks_adapter_coverage.py @@ -736,7 +736,7 @@ def test_happy_path_issues_create_table_sql(self): create_sql = next((s for s in executed_sqls if "CREATE TABLE" in s and "PARQUET" in s), None) assert create_sql is not None, f"No CREATE TABLE USING PARQUET found in: {executed_sqls}" assert "ORDERS" in create_sql - assert "ORDERS" in table_stats + assert "orders" in table_stats # --------------------------------------------------------------------------- @@ -2711,9 +2711,9 @@ def test_loads_all_tables_and_returns_stats(self): data_dir=Path("/data"), ) - assert "ORDERS" in table_stats - assert "LINEITEM" in table_stats - assert table_stats["ORDERS"] == 1500 + assert "orders" in table_stats + assert "lineitem" in table_stats + assert table_stats["orders"] == 1500 assert total_time == 1.0 def test_handles_individual_table_failure_gracefully(self): @@ -2746,7 +2746,7 @@ def test_handles_individual_table_failure_gracefully(self): ) # Table should be recorded with 0 rows on failure - assert table_stats["ORDERS"] == 0 + assert table_stats["orders"] == 0 # --------------------------------------------------------------------------- diff --git a/tests/unit/platforms/test_databricks_copy_into.py b/tests/unit/platforms/test_databricks_copy_into.py index a3d73ef0e3..0442643900 100644 --- a/tests/unit/platforms/test_databricks_copy_into.py +++ b/tests/unit/platforms/test_databricks_copy_into.py @@ -120,8 +120,8 @@ class B: assert "COPY INTO ORDERS FROM 'dbfs:/Volumes/workspace/raw/source/tpch_sf01/orders.csv'" in sqls assert "CREATE OR REPLACE TEMPORARY VIEW" not in sqls assert "INSERT INTO" not in sqls - assert stats["LINEITEM"] == 42 - assert stats["ORDERS"] == 42 + assert stats["lineitem"] == 42 + assert stats["orders"] == 42 def test_copy_into_requires_staging(adapter):