From e70e87d36123d40c19d284fac5c683f0eb6ac0aa Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Tue, 27 Jan 2026 15:09:24 +0100 Subject: [PATCH 1/8] iceberg_partition_timezone setting --- src/Core/Settings.cpp | 9 ++++++++ src/Core/SettingsChangesHistory.cpp | 1 + .../DataLakes/Iceberg/ManifestFile.cpp | 9 +++++++- .../Iceberg/ManifestFilesPruning.cpp | 11 +++++++-- .../DataLakes/Iceberg/ManifestFilesPruning.h | 2 +- .../ObjectStorage/DataLakes/Iceberg/Utils.cpp | 23 +++++++++++-------- .../ObjectStorage/DataLakes/Iceberg/Utils.h | 3 ++- 7 files changed, 44 insertions(+), 14 deletions(-) diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index e72653747d54..34878852b7bb 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -7315,6 +7315,15 @@ Allows creation of [QBit](../../sql-reference/data-types/qbit.md) data type. )", BETA, allow_experimental_qbit_type) \ DECLARE(UInt64, archive_adaptive_buffer_max_size_bytes, 8 * DBMS_DEFAULT_BUFFER_SIZE, R"( Limits the maximum size of the adaptive buffer used when writing to archive files (for example, tar archives)", 0) \ + DECLARE(Timezone, iceberg_partition_timezone, "", R"( +Time zone by which partitioning of Iceberg tables was performed. +Possible values: + +- Any valid timezone, e.g. `Europe/Berlin`, `UTC` or `Zulu` +- `` (empty value) - use server or session timezone + +Default value is empty. +)", 0) \ \ /* ####################################################### */ \ /* ########### START OF EXPERIMENTAL FEATURES ############ */ \ diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index e8686e2a43ea..36d2121b3dc4 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -220,6 +220,7 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() {"os_threads_nice_value_query", 0, 0, "New setting."}, {"os_threads_nice_value_materialized_view", 0, 0, "New setting."}, {"os_thread_priority", 0, 0, "Alias for os_threads_nice_value_query."}, + {"iceberg_partition_timezone", "", "", "New setting."}, }); addSettingsChanges(settings_changes_history, "25.8", { diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.cpp index 550e18c5c96d..0a473671b7c5 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.cpp @@ -6,6 +6,7 @@ #include #include +#include #include #include @@ -14,6 +15,7 @@ #include #include +#include #include #include #include @@ -34,6 +36,11 @@ namespace DB::ErrorCodes extern const int BAD_ARGUMENTS; } +namespace DB::Setting +{ + extern const SettingsTimezone iceberg_partition_timezone; +} + namespace DB::Iceberg { @@ -219,7 +226,7 @@ ManifestFileContent::ManifestFileContent( auto transform_name = partition_specification_field->getValue(f_partition_transform); auto partition_name = partition_specification_field->getValue(f_partition_name); common_partition_specification.emplace_back(source_id, transform_name, partition_name); - auto partition_ast = getASTFromTransform(transform_name, numeric_column_name); + auto partition_ast = getASTFromTransform(transform_name, numeric_column_name, context->getSettingsRef()[Setting::iceberg_partition_timezone]); /// Unsupported partition key expression if (partition_ast == nullptr) continue; diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFilesPruning.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFilesPruning.cpp index a5ce96e37031..cfeefebf7d27 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFilesPruning.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFilesPruning.cpp @@ -26,9 +26,9 @@ using namespace DB; namespace DB::Iceberg { -DB::ASTPtr getASTFromTransform(const String & transform_name_src, const String & column_name) +DB::ASTPtr getASTFromTransform(const String & transform_name_src, const String & column_name, const String & time_zone) { - auto transform_and_argument = parseTransformAndArgument(transform_name_src); + auto transform_and_argument = parseTransformAndArgument(transform_name_src, time_zone); if (!transform_and_argument) { LOG_WARNING(&Poco::Logger::get("Iceberg Partition Pruning"), "Cannot parse iceberg transform name: {}.", transform_name_src); @@ -47,6 +47,13 @@ DB::ASTPtr getASTFromTransform(const String & transform_name_src, const String & return makeASTFunction( transform_and_argument->transform_name, make_intrusive(*transform_and_argument->argument), make_intrusive(column_name)); } + if (transform_and_argument->time_zone) + { + return makeASTFunction( + transform_and_argument->transform_name, + make_intrusive(column_name), + make_intrusive(*transform_and_argument->time_zone)); + } return makeASTFunction(transform_and_argument->transform_name, make_intrusive(column_name)); } diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFilesPruning.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFilesPruning.h index 1aaf5407f4f1..9b3a3f660b1c 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFilesPruning.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFilesPruning.h @@ -31,7 +31,7 @@ namespace DB::Iceberg struct ManifestFileEntry; class ManifestFileContent; -DB::ASTPtr getASTFromTransform(const String & transform_name_src, const String & column_name); +DB::ASTPtr getASTFromTransform(const String & transform_name_src, const String & column_name, const String & time_zone); /// Prune specific data files based on manifest content class ManifestFilesPruner diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp index 2a1dff4caa26..f9ae07d427ef 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp @@ -81,6 +81,7 @@ namespace ProfileEvents namespace DB::Setting { extern const SettingsUInt64 output_format_compression_level; + extern const SettingsTimezone iceberg_partition_timezone; } /// Hard to imagine a hint file larger than 10 MB @@ -240,27 +241,31 @@ bool writeMetadataFileAndVersionHint( } -std::optional parseTransformAndArgument(const String & transform_name_src) +std::optional parseTransformAndArgument(const String & transform_name_src, const String & time_zone) { std::string transform_name = Poco::toLower(transform_name_src); + std::optional time_zone_opt; + if (!time_zone.empty()) + time_zone_opt = time_zone; + if (transform_name == "year" || transform_name == "years") - return TransformAndArgument{"toYearNumSinceEpoch", std::nullopt}; + return TransformAndArgument{"toYearNumSinceEpoch", std::nullopt, time_zone_opt}; if (transform_name == "month" || transform_name == "months") - return TransformAndArgument{"toMonthNumSinceEpoch", std::nullopt}; + return TransformAndArgument{"toMonthNumSinceEpoch", std::nullopt, time_zone_opt}; if (transform_name == "day" || transform_name == "date" || transform_name == "days" || transform_name == "dates") - return TransformAndArgument{"toRelativeDayNum", std::nullopt}; + return TransformAndArgument{"toRelativeDayNum", std::nullopt, time_zone_opt}; if (transform_name == "hour" || transform_name == "hours") - return TransformAndArgument{"toRelativeHourNum", std::nullopt}; + return TransformAndArgument{"toRelativeHourNum", std::nullopt, time_zone_opt}; if (transform_name == "identity") - return TransformAndArgument{"identity", std::nullopt}; + return TransformAndArgument{"identity", std::nullopt, std::nullopt}; if (transform_name == "void") - return TransformAndArgument{"tuple", std::nullopt}; + return TransformAndArgument{"tuple", std::nullopt, std::nullopt}; if (transform_name.starts_with("truncate") || transform_name.starts_with("bucket")) { @@ -284,11 +289,11 @@ std::optional parseTransformAndArgument(const String & tra if (transform_name.starts_with("truncate")) { - return TransformAndArgument{"icebergTruncate", argument}; + return TransformAndArgument{"icebergTruncate", argument, std::nullopt}; } else if (transform_name.starts_with("bucket")) { - return TransformAndArgument{"icebergBucket", argument}; + return TransformAndArgument{"icebergBucket", argument, std::nullopt}; } } return std::nullopt; diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.h index 484e01cfe869..b08cff19109b 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.h @@ -54,9 +54,10 @@ struct TransformAndArgument { String transform_name; std::optional argument; + std::optional time_zone; }; -std::optional parseTransformAndArgument(const String & transform_name_src); +std::optional parseTransformAndArgument(const String & transform_name_src, const String & time_zone); Poco::JSON::Object::Ptr getMetadataJSONObject( const String & metadata_file_path, From 1343f096529929f5c317bbf4ff30b0548da61d18 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Tue, 27 Jan 2026 18:49:46 +0100 Subject: [PATCH 2/8] Test --- .../configs/iceberg_partition_timezone.xml | 7 + .../configs/timezone.xml | 3 + .../test_partition_timezone.py | 178 ++++++++++++++++++ 3 files changed, 188 insertions(+) create mode 100644 tests/integration/test_database_iceberg/configs/iceberg_partition_timezone.xml create mode 100644 tests/integration/test_database_iceberg/configs/timezone.xml create mode 100644 tests/integration/test_database_iceberg/test_partition_timezone.py diff --git a/tests/integration/test_database_iceberg/configs/iceberg_partition_timezone.xml b/tests/integration/test_database_iceberg/configs/iceberg_partition_timezone.xml new file mode 100644 index 000000000000..40aebd33c515 --- /dev/null +++ b/tests/integration/test_database_iceberg/configs/iceberg_partition_timezone.xml @@ -0,0 +1,7 @@ + + + + UTC + + + diff --git a/tests/integration/test_database_iceberg/configs/timezone.xml b/tests/integration/test_database_iceberg/configs/timezone.xml new file mode 100644 index 000000000000..269e52ef2247 --- /dev/null +++ b/tests/integration/test_database_iceberg/configs/timezone.xml @@ -0,0 +1,3 @@ + + Asia/Istanbul + \ No newline at end of file diff --git a/tests/integration/test_database_iceberg/test_partition_timezone.py b/tests/integration/test_database_iceberg/test_partition_timezone.py new file mode 100644 index 000000000000..55bb525cb0ca --- /dev/null +++ b/tests/integration/test_database_iceberg/test_partition_timezone.py @@ -0,0 +1,178 @@ +import glob +import json +import logging +import os +import random +import time +import uuid +from datetime import datetime, timedelta + +import pyarrow as pa +import pytest +import requests +import urllib3 +import pytz +from minio import Minio +from pyiceberg.catalog import load_catalog +from pyiceberg.partitioning import PartitionField, PartitionSpec, UNPARTITIONED_PARTITION_SPEC +from pyiceberg.schema import Schema +from pyiceberg.table.sorting import SortField, SortOrder +from pyiceberg.transforms import DayTransform, IdentityTransform +from pyiceberg.types import ( + DoubleType, + LongType, + FloatType, + NestedField, + StringType, + StructType, + TimestampType, + TimestamptzType +) +from pyiceberg.table.sorting import UNSORTED_SORT_ORDER + +from helpers.cluster import ClickHouseCluster, ClickHouseInstance, is_arm +from helpers.config_cluster import minio_secret_key, minio_access_key +from helpers.s3_tools import get_file_contents, list_s3_objects, prepare_s3_bucket +from helpers.test_tools import TSV, csv_compare +from helpers.config_cluster import minio_secret_key + +BASE_URL = "http://rest:8181/v1" +BASE_URL_LOCAL = "http://localhost:8182/v1" +BASE_URL_LOCAL_RAW = "http://localhost:8182" + +CATALOG_NAME = "demo" + +DEFAULT_PARTITION_SPEC = PartitionSpec( + PartitionField( + source_id=1, field_id=1000, transform=DayTransform(), name="datetime_day" + ) +) +DEFAULT_SORT_ORDER = SortOrder(SortField(source_id=2, transform=IdentityTransform())) +DEFAULT_SCHEMA = Schema( + NestedField(field_id=1, name="datetime", field_type=TimestampType(), required=False), + NestedField(field_id=2, name="value", field_type=LongType(), required=False), +) + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster = ClickHouseCluster(__file__) + cluster.add_instance( + "node1", + main_configs=["configs/timezone.xml", "configs/cluster.xml"], + user_configs=["configs/iceberg_partition_timezone.xml"], + stay_alive=True, + with_iceberg_catalog=True, + with_zookeeper=True, + ) + + logging.info("Starting cluster...") + cluster.start() + + # TODO: properly wait for container + time.sleep(10) + + yield cluster + + finally: + cluster.shutdown() + + +def load_catalog_impl(started_cluster): + return load_catalog( + CATALOG_NAME, + **{ + "uri": BASE_URL_LOCAL_RAW, + "type": "rest", + "s3.endpoint": f"http://{started_cluster.get_instance_ip('minio')}:9000", + "s3.access-key-id": minio_access_key, + "s3.secret-access-key": minio_secret_key, + }, + ) + + +def create_table( + catalog, + namespace, + table, + schema=DEFAULT_SCHEMA, + partition_spec=DEFAULT_PARTITION_SPEC, + sort_order=DEFAULT_SORT_ORDER, +): + return catalog.create_table( + identifier=f"{namespace}.{table}", + schema=schema, + location=f"s3://warehouse-rest/data", + partition_spec=partition_spec, + sort_order=sort_order, + ) + + +def create_clickhouse_iceberg_database( + node, name, additional_settings={}, engine='DataLakeCatalog' +): + settings = { + "catalog_type": "rest", + "warehouse": "demo", + "storage_endpoint": "http://minio:9000/warehouse-rest", + } + + settings.update(additional_settings) + + node.query( + f""" +DROP DATABASE IF EXISTS {name}; +SET allow_database_iceberg=true; +SET write_full_path_in_iceberg_metadata=1; +CREATE DATABASE {name} ENGINE = {engine}('{BASE_URL}', 'minio', '{minio_secret_key}') +SETTINGS {",".join((k+"="+repr(v) for k, v in settings.items()))} + """ + ) + show_result = node.query(f"SHOW DATABASE {name}") + assert minio_secret_key not in show_result + assert "HIDDEN" in show_result + + +def test_partition_timezone(started_cluster): + catalog = load_catalog_impl(started_cluster) + catalog.create_namespace("timezone_ns") + table = create_table( + catalog, + "timezone_ns", + "tz_table", + ) + + # catalog accept data in UTC + data = [{"datetime": datetime(2024, 1, 1, 20, 0), "value": 1}, # partition 20240101 + {"datetime": datetime(2024, 1, 1, 23, 0), "value": 2}, # partition 20240101 + {"datetime": datetime(2024, 1, 2, 2, 0), "value": 3}] # partition 20240102 + df = pa.Table.from_pylist(data) + table.append(df) + + node = started_cluster.instances["node1"] + create_clickhouse_iceberg_database(node, CATALOG_NAME) + + # server timezone is Asia/Istanbul (UTC+3) + assert node.query(f""" + SELECT datetime, value + FROM {CATALOG_NAME}.`timezone_ns.tz_table` + ORDER BY datetime + """, timeout=10) == TSV( + [ + ["2024-01-01 23:00:00.000000", 1], + ["2024-01-02 02:00:00.000000", 2], + ["2024-01-02 05:00:00.000000", 3], + ]) + + # partitioning works correctly + assert node.query(f""" + SELECT datetime, value + FROM {CATALOG_NAME}.`timezone_ns.tz_table` + WHERE datetime >= '2024-01-02 00:00:00' + ORDER BY datetime + """, timeout=10) == TSV( + [ + ["2024-01-02 02:00:00.000000", 2], + ["2024-01-02 05:00:00.000000", 3], + ]) From 97abe9f276db646fcf876ec55bc037790dc80d89 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Fri, 30 Jan 2026 11:57:09 +0100 Subject: [PATCH 3/8] Comment about setting --- src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.h index b08cff19109b..844ebc72f015 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.h @@ -54,6 +54,10 @@ struct TransformAndArgument { String transform_name; std::optional argument; + /// When Iceberg table is partitioned by time, splitting by partitions can be made using different timezone + /// (UTC in most cases). This timezone can be set with setting `iceberg_partition_timezone`, value is in this member. + /// When Iceberg partition condition converted to ClickHouse function in `parseTransformAndArgument` method + /// `time_zone` added as second argument to functions like `toRelativeDayNum`, `toYearNumSinceEpoch`, etc. std::optional time_zone; }; From 841ec9f1f593474a321f83ca174f77ba7e8c133c Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Fri, 30 Jan 2026 15:48:46 +0100 Subject: [PATCH 4/8] Add missing parts --- .../DataLakes/Iceberg/ChunkPartitioner.cpp | 12 +++++++++++- .../DataLakes/Iceberg/ChunkPartitioner.h | 1 + .../ObjectStorage/DataLakes/Iceberg/Utils.cpp | 8 ++++++-- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/ChunkPartitioner.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/ChunkPartitioner.cpp index 606879f99dd7..f5930b5742e9 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/ChunkPartitioner.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/ChunkPartitioner.cpp @@ -18,6 +18,7 @@ namespace DB namespace Setting { extern const SettingsUInt64 iceberg_insert_max_partitions; + extern const SettingsTimezone iceberg_partition_timezone; } namespace ErrorCodes @@ -53,7 +54,7 @@ ChunkPartitioner::ChunkPartitioner( auto & factory = FunctionFactory::instance(); - auto transform_and_argument = Iceberg::parseTransformAndArgument(transform_name); + auto transform_and_argument = Iceberg::parseTransformAndArgument(transform_name, context->getSettingsRef()[Setting::iceberg_partition_timezone]); if (!transform_and_argument) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unknown transform {}", transform_name); @@ -67,6 +68,7 @@ ChunkPartitioner::ChunkPartitioner( result_data_types.push_back(function->getReturnType(columns_for_function)); functions.push_back(function); function_params.push_back(transform_and_argument->argument); + function_time_zones.push_back(transform_and_argument->time_zone); columns_to_apply.push_back(column_name); } } @@ -103,6 +105,14 @@ ChunkPartitioner::partitionChunk(const Chunk & chunk) arguments.push_back(ColumnWithTypeAndName(const_column->clone(), type, "#")); } arguments.push_back(name_to_column[columns_to_apply[transform_ind]]); + if (function_time_zones[transform_ind].has_value()) + { + auto type = std::make_shared(); + auto column_value = ColumnString::create(); + column_value->insert(*function_time_zones[transform_ind]); + auto const_column = ColumnConst::create(std::move(column_value), chunk.getNumRows()); + arguments.push_back(ColumnWithTypeAndName(const_column->clone(), type, "PartitioningTimezone")); + } auto result = functions[transform_ind]->build(arguments)->execute(arguments, std::make_shared(), chunk.getNumRows(), false); for (size_t i = 0; i < chunk.getNumRows(); ++i) diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/ChunkPartitioner.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/ChunkPartitioner.h index 4c42e037174a..f77b27a1b15e 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/ChunkPartitioner.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/ChunkPartitioner.h @@ -39,6 +39,7 @@ class ChunkPartitioner std::vector functions; std::vector> function_params; + std::vector> function_time_zones; std::vector columns_to_apply; std::vector result_data_types; diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp index f9ae07d427ef..f067fc0e2b32 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp @@ -1143,7 +1143,8 @@ KeyDescription getSortingKeyDescriptionFromMetadata(Poco::JSON::Object::Ptr meta auto column_name = source_id_to_column_name[source_id]; int direction = field->getValue(f_direction) == "asc" ? 1 : -1; auto iceberg_transform_name = field->getValue(f_transform); - auto clickhouse_transform_name = parseTransformAndArgument(iceberg_transform_name); + auto clickhouse_transform_name = parseTransformAndArgument(iceberg_transform_name, + local_context->getSettingsRef()[Setting::iceberg_partition_timezone]); String full_argument; if (clickhouse_transform_name->transform_name != "identity") { @@ -1152,7 +1153,10 @@ KeyDescription getSortingKeyDescriptionFromMetadata(Poco::JSON::Object::Ptr meta { full_argument += std::to_string(*clickhouse_transform_name->argument) + ", "; } - full_argument += column_name + ")"; + full_argument += column_name; + if (clickhouse_transform_name->time_zone) + full_argument += ", " + *clickhouse_transform_name->time_zone; + full_argument += ")"; } else { From 33e479da8d04ea9bd9167537408a8911e120f388 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Wed, 4 Feb 2026 11:12:48 +0100 Subject: [PATCH 5/8] Fix settings history, fix NamespaceAlreadyExistsError in test --- src/Core/SettingsChangesHistory.cpp | 6 +++++- .../test_database_iceberg/test_partition_timezone.py | 7 ++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index 36d2121b3dc4..ddf7e8466191 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -39,6 +39,11 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() /// controls new feature and it's 'true' by default, use 'false' as previous_value). /// It's used to implement `compatibility` setting (see https://github.com/ClickHouse/ClickHouse/issues/35972) /// Note: please check if the key already exists to prevent duplicate entries. + addSettingsChanges(settings_changes_history, "26.2", + { + {"default_dictionary_database", "", "", "New setting"}, + {"iceberg_partition_timezone", "", "", "New setting."}, + }); addSettingsChanges(settings_changes_history, "26.1", { {"parallel_replicas_filter_pushdown", false, false, "New setting"}, @@ -220,7 +225,6 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() {"os_threads_nice_value_query", 0, 0, "New setting."}, {"os_threads_nice_value_materialized_view", 0, 0, "New setting."}, {"os_thread_priority", 0, 0, "Alias for os_threads_nice_value_query."}, - {"iceberg_partition_timezone", "", "", "New setting."}, }); addSettingsChanges(settings_changes_history, "25.8", { diff --git a/tests/integration/test_database_iceberg/test_partition_timezone.py b/tests/integration/test_database_iceberg/test_partition_timezone.py index 55bb525cb0ca..77f76c9ad859 100644 --- a/tests/integration/test_database_iceberg/test_partition_timezone.py +++ b/tests/integration/test_database_iceberg/test_partition_timezone.py @@ -136,7 +136,8 @@ def create_clickhouse_iceberg_database( def test_partition_timezone(started_cluster): catalog = load_catalog_impl(started_cluster) - catalog.create_namespace("timezone_ns") + namespace = f"timezone_ns_{uuid.uuid4()}" + catalog.create_namespace(namespace) table = create_table( catalog, "timezone_ns", @@ -156,7 +157,7 @@ def test_partition_timezone(started_cluster): # server timezone is Asia/Istanbul (UTC+3) assert node.query(f""" SELECT datetime, value - FROM {CATALOG_NAME}.`timezone_ns.tz_table` + FROM {CATALOG_NAME}.`{namespace}}.tz_table` ORDER BY datetime """, timeout=10) == TSV( [ @@ -168,7 +169,7 @@ def test_partition_timezone(started_cluster): # partitioning works correctly assert node.query(f""" SELECT datetime, value - FROM {CATALOG_NAME}.`timezone_ns.tz_table` + FROM {CATALOG_NAME}.`{namespace}}.tz_table` WHERE datetime >= '2024-01-02 00:00:00' ORDER BY datetime """, timeout=10) == TSV( From 7088a59f1011630095081f3f65201056bf838c00 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Wed, 4 Feb 2026 12:30:27 +0100 Subject: [PATCH 6/8] Fix test --- .../test_database_iceberg/test_partition_timezone.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_database_iceberg/test_partition_timezone.py b/tests/integration/test_database_iceberg/test_partition_timezone.py index 77f76c9ad859..35f1c97d0b42 100644 --- a/tests/integration/test_database_iceberg/test_partition_timezone.py +++ b/tests/integration/test_database_iceberg/test_partition_timezone.py @@ -157,7 +157,7 @@ def test_partition_timezone(started_cluster): # server timezone is Asia/Istanbul (UTC+3) assert node.query(f""" SELECT datetime, value - FROM {CATALOG_NAME}.`{namespace}}.tz_table` + FROM {CATALOG_NAME}.`{namespace}.tz_table` ORDER BY datetime """, timeout=10) == TSV( [ @@ -169,7 +169,7 @@ def test_partition_timezone(started_cluster): # partitioning works correctly assert node.query(f""" SELECT datetime, value - FROM {CATALOG_NAME}.`{namespace}}.tz_table` + FROM {CATALOG_NAME}.`{namespace}.tz_table` WHERE datetime >= '2024-01-02 00:00:00' ORDER BY datetime """, timeout=10) == TSV( From 61c377670717b0041e0fdc1f4667916aea2b3c09 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Thu, 12 Feb 2026 22:36:48 +0100 Subject: [PATCH 7/8] Fix test --- .../test_database_iceberg/test_partition_timezone.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/integration/test_database_iceberg/test_partition_timezone.py b/tests/integration/test_database_iceberg/test_partition_timezone.py index 35f1c97d0b42..e6f4e3a3c520 100644 --- a/tests/integration/test_database_iceberg/test_partition_timezone.py +++ b/tests/integration/test_database_iceberg/test_partition_timezone.py @@ -137,11 +137,12 @@ def create_clickhouse_iceberg_database( def test_partition_timezone(started_cluster): catalog = load_catalog_impl(started_cluster) namespace = f"timezone_ns_{uuid.uuid4()}" + table_name = f"tz_table__{uuid.uuid4()}" catalog.create_namespace(namespace) table = create_table( catalog, - "timezone_ns", - "tz_table", + namespace, + table_name, ) # catalog accept data in UTC @@ -157,7 +158,7 @@ def test_partition_timezone(started_cluster): # server timezone is Asia/Istanbul (UTC+3) assert node.query(f""" SELECT datetime, value - FROM {CATALOG_NAME}.`{namespace}.tz_table` + FROM {CATALOG_NAME}.`{namespace}.{table_name}` ORDER BY datetime """, timeout=10) == TSV( [ @@ -169,7 +170,7 @@ def test_partition_timezone(started_cluster): # partitioning works correctly assert node.query(f""" SELECT datetime, value - FROM {CATALOG_NAME}.`{namespace}.tz_table` + FROM {CATALOG_NAME}.`{namespace}.{table_name}` WHERE datetime >= '2024-01-02 00:00:00' ORDER BY datetime """, timeout=10) == TSV( From c4c3b4e930f222ddc1088fede708cc8a27dd68fb Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Thu, 26 Feb 2026 13:52:46 +0100 Subject: [PATCH 8/8] Fix tests --- src/Core/SettingsChangesHistory.cpp | 3 +-- .../compose/docker_compose_iceberg_rest_catalog.yml | 8 ++++---- tests/integration/helpers/cluster.py | 9 +++++++++ .../test_database_iceberg/test_partition_timezone.py | 10 ++++++++-- 4 files changed, 22 insertions(+), 8 deletions(-) diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index ddf7e8466191..87b0c41c3153 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -39,9 +39,8 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() /// controls new feature and it's 'true' by default, use 'false' as previous_value). /// It's used to implement `compatibility` setting (see https://github.com/ClickHouse/ClickHouse/issues/35972) /// Note: please check if the key already exists to prevent duplicate entries. - addSettingsChanges(settings_changes_history, "26.2", + addSettingsChanges(settings_changes_history, "26.1.1.20001", { - {"default_dictionary_database", "", "", "New setting"}, {"iceberg_partition_timezone", "", "", "New setting."}, }); addSettingsChanges(settings_changes_history, "26.1", diff --git a/tests/integration/compose/docker_compose_iceberg_rest_catalog.yml b/tests/integration/compose/docker_compose_iceberg_rest_catalog.yml index c69a89f6fa58..34de4ffed21b 100644 --- a/tests/integration/compose/docker_compose_iceberg_rest_catalog.yml +++ b/tests/integration/compose/docker_compose_iceberg_rest_catalog.yml @@ -12,15 +12,15 @@ services: - AWS_SECRET_ACCESS_KEY=password - AWS_REGION=us-east-1 ports: - - 8080:8080 - - 10002:10000 - - 10003:10001 + - ${SPARK_ICEBERG_EXTERNAL_PORT:-8080}:8080 + - ${SPARK_ICEBERG_EXTERNAL_PORT_2:-10002}:10000 + - ${SPARK_ICEBERG_EXTERNAL_PORT_3:-10003}:10001 stop_grace_period: 5s cpus: 3 rest: image: tabulario/iceberg-rest:1.6.0 ports: - - 8182:8181 + - ${ICEBERG_REST_EXTERNAL_PORT:-8182}:8181 environment: - AWS_ACCESS_KEY_ID=minio - AWS_SECRET_ACCESS_KEY=ClickHouse_Minio_P@ssw0rd diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index 1c173980d7a6..9d8065b12d7c 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -674,6 +674,9 @@ def __init__( self.minio_secret_key = minio_secret_key self.spark_session = None + self.spark_iceberg_external_port = 8080 + self.spark_iceberg_external_port_2 = 10002 + self.spark_iceberg_external_port_3 = 10003 self.with_iceberg_catalog = False self.with_glue_catalog = False self.with_hms_catalog = False @@ -885,6 +888,8 @@ def __init__( self._letsencrypt_pebble_api_port = 14000 self._letsencrypt_pebble_management_port = 15000 + self.iceberg_rest_external_port = 8182 + self.docker_client: docker.DockerClient = None self.is_up = False self.env = os.environ.copy() @@ -1702,6 +1707,10 @@ def setup_hms_catalog_cmd(self, instance, env_variables, docker_compose_yml_dir) def setup_iceberg_catalog_cmd( self, instance, env_variables, docker_compose_yml_dir, extra_parameters=None ): + env_variables["ICEBERG_REST_EXTERNAL_PORT"] = str(self.iceberg_rest_external_port) + env_variables["SPARK_ICEBERG_EXTERNAL_PORT"] = str(self.spark_iceberg_external_port) + env_variables["SPARK_ICEBERG_EXTERNAL_PORT_2"] = str(self.spark_iceberg_external_port_2) + env_variables["SPARK_ICEBERG_EXTERNAL_PORT_3"] = str(self.spark_iceberg_external_port_3) self.with_iceberg_catalog = True file_name = "docker_compose_iceberg_rest_catalog.yml" if extra_parameters is not None and extra_parameters["docker_compose_file_name"] != "": diff --git a/tests/integration/test_database_iceberg/test_partition_timezone.py b/tests/integration/test_database_iceberg/test_partition_timezone.py index e6f4e3a3c520..a6bfe97a111e 100644 --- a/tests/integration/test_database_iceberg/test_partition_timezone.py +++ b/tests/integration/test_database_iceberg/test_partition_timezone.py @@ -36,9 +36,11 @@ from helpers.test_tools import TSV, csv_compare from helpers.config_cluster import minio_secret_key +ICEBERG_PORT = 8183 + BASE_URL = "http://rest:8181/v1" -BASE_URL_LOCAL = "http://localhost:8182/v1" -BASE_URL_LOCAL_RAW = "http://localhost:8182" +BASE_URL_LOCAL = f"http://localhost:{ICEBERG_PORT}/v1" +BASE_URL_LOCAL_RAW = f"http://localhost:{ICEBERG_PORT}" CATALOG_NAME = "demo" @@ -58,6 +60,10 @@ def started_cluster(): try: cluster = ClickHouseCluster(__file__) + cluster.iceberg_rest_external_port = ICEBERG_PORT + cluster.spark_iceberg_external_port = 10004 + cluster.spark_iceberg_external_port_2 = 10005 + cluster.spark_iceberg_external_port_3 = 10006 cluster.add_instance( "node1", main_configs=["configs/timezone.xml", "configs/cluster.xml"],