From db2d39c4cd6fa70914058376eb525f5467cd78e7 Mon Sep 17 00:00:00 2001 From: kanthi subramanian Date: Wed, 11 Feb 2026 22:26:42 -0600 Subject: [PATCH 1/9] Add Docker integration tests. --- .bin/pre-release-docker | 5 + .github/workflows/pre-release.yaml | 25 ++- ice-rest-catalog/pom.xml | 1 - .../rest/catalog/DockerScenarioBasedIT.java | 183 ++++++++++++++++++ .../ice/rest/catalog/RESTCatalogTestBase.java | 97 ++++++++++ .../ice/rest/catalog/ScenarioBasedIT.java | 111 +---------- .../ice/rest/catalog/ScenarioConfig.java | 13 +- .../ice/rest/catalog/ScenarioTestRunner.java | 4 +- .../src/test/resources/scenarios/README.md | 20 -- .../scenarios/basic-operations/input.parquet | Bin 0 -> 2446 bytes .../scenarios/basic-operations/run.sh.tmpl | 8 - .../scenarios/basic-operations/scenario.yaml | 19 -- .../insert-partitioned/scenario.yaml | 19 -- .../scenarios/insert-scan/scenario.yaml | 17 -- 14 files changed, 315 insertions(+), 207 deletions(-) create mode 100644 ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/DockerScenarioBasedIT.java create mode 100644 ice-rest-catalog/src/test/resources/scenarios/basic-operations/input.parquet diff --git a/.bin/pre-release-docker b/.bin/pre-release-docker index 2671de32..990a5597 100755 --- a/.bin/pre-release-docker +++ b/.bin/pre-release-docker @@ -8,6 +8,11 @@ export SKIP_VERIFY=1 export PATH="$(pwd)/.bin:$PATH" +echo >&2 'Building ice Docker image' +docker-build-ice +echo >&2 'Building ice-rest-catalog Docker image' +docker-build-ice-rest-catalog + echo >&2 'Pushing ice Docker image' docker-build-ice --push echo >&2 'Pushing ice-rest-catalog Docker image' diff --git a/.github/workflows/pre-release.yaml b/.github/workflows/pre-release.yaml index b1ec2086..fb893650 100644 --- a/.github/workflows/pre-release.yaml +++ b/.github/workflows/pre-release.yaml @@ -49,4 +49,27 @@ jobs: - uses: actions/checkout@v4 - - run: .bin/pre-release-docker + - uses: actions/setup-java@v4 + with: + java-version: '21' + distribution: 'graalvm' + cache: maven + + - name: Build Docker images + run: | + export VERSION=0.0.0-latest-master+$(git rev-parse --short HEAD) + export IMAGE_TAG="latest-master" + export SKIP_VERIFY=1 + export PATH="$(pwd)/.bin:$PATH" + docker-build-ice + docker-build-ice-rest-catalog + + - name: Run Docker integration tests + run: > + ./mvnw -pl ice-rest-catalog install -Dmaven.test.skip=true -Pno-check && + ./mvnw -pl ice-rest-catalog failsafe:integration-test failsafe:verify + -Dit.test=DockerScenarioBasedIT + -Ddocker.image=altinity/ice-rest-catalog:debug-with-ice-latest-master-amd64 + + - name: Push Docker images + run: .bin/pre-release-docker diff --git a/ice-rest-catalog/pom.xml b/ice-rest-catalog/pom.xml index 43a8121e..fda0550b 100644 --- a/ice-rest-catalog/pom.xml +++ b/ice-rest-catalog/pom.xml @@ -393,7 +393,6 @@ com.fasterxml.jackson.dataformat jackson-dataformat-yaml ${jackson.version} - test io.etcd diff --git a/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/DockerScenarioBasedIT.java b/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/DockerScenarioBasedIT.java new file mode 100644 index 00000000..2916589a --- /dev/null +++ b/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/DockerScenarioBasedIT.java @@ -0,0 +1,183 @@ +/* + * Copyright (c) 2025 Altinity Inc and/or its affiliates. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ +package com.altinity.ice.rest.catalog; + +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.Network; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.utility.MountableFile; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; + +/** + * Docker-based integration tests for ICE REST Catalog. + * + *

Runs the ice-rest-catalog Docker image (specified via system property {@code docker.image}) + * alongside a MinIO container, then executes scenario-based tests against it. + */ +public class DockerScenarioBasedIT extends RESTCatalogTestBase { + + private Network network; + + private GenericContainer minio; + + private GenericContainer catalog; + + @Override + @BeforeClass + public void setUp() throws Exception { + String dockerImage = + System.getProperty("docker.image", "altinity/ice-rest-catalog:debug-with-ice-0.12.0"); + logger.info("Using Docker image: {}", dockerImage); + + network = Network.newNetwork(); + + // Start MinIO + minio = + new GenericContainer<>("minio/minio:latest") + .withNetwork(network) + .withNetworkAliases("minio") + .withExposedPorts(9000) + .withEnv("MINIO_ACCESS_KEY", "minioadmin") + .withEnv("MINIO_SECRET_KEY", "minioadmin") + .withCommand("server", "/data") + .waitingFor(Wait.forHttp("/minio/health/live").forPort(9000)); + minio.start(); + + // Create test bucket via MinIO's host-mapped port + String minioHostEndpoint = "http://" + minio.getHost() + ":" + minio.getMappedPort(9000); + try (var s3Client = + software.amazon.awssdk.services.s3.S3Client.builder() + .endpointOverride(java.net.URI.create(minioHostEndpoint)) + .region(software.amazon.awssdk.regions.Region.US_EAST_1) + .credentialsProvider( + software.amazon.awssdk.auth.credentials.StaticCredentialsProvider.create( + software.amazon.awssdk.auth.credentials.AwsBasicCredentials.create( + "minioadmin", "minioadmin"))) + .forcePathStyle(true) + .build()) { + s3Client.createBucket( + software.amazon.awssdk.services.s3.model.CreateBucketRequest.builder() + .bucket("test-bucket") + .build()); + logger.info("Created test-bucket in MinIO"); + } + + // Build YAML config for the catalog container (using the Docker network alias for MinIO) + String catalogConfig = + String.join( + "\n", + "uri: \"jdbc:sqlite::memory:\"", + "warehouse: \"s3://test-bucket/warehouse\"", + "s3:", + " endpoint: \"http://minio:9000\"", + " pathStyleAccess: true", + " accessKeyID: \"minioadmin\"", + " secretAccessKey: \"minioadmin\"", + " region: \"us-east-1\"", + "anonymousAccess:", + " enabled: true", + " accessConfig:", + " readOnly: false", + ""); + + Path scenariosDir = getScenariosDirectory().toAbsolutePath(); + if (!Files.exists(scenariosDir) || !Files.isDirectory(scenariosDir)) { + throw new IllegalStateException( + "Scenarios directory must exist at " + + scenariosDir + + ". Run 'mvn test-compile' or run the test from Maven (e.g. mvn failsafe:integration-test)."); + } + Path insertScanInput = scenariosDir.resolve("insert-scan").resolve("input.parquet"); + if (!Files.exists(insertScanInput)) { + throw new IllegalStateException( + "Scenario input not found at " + + insertScanInput + + ". Ensure test resources are on the classpath and scenarios/insert-scan/input.parquet exists."); + } + + // Start the ice-rest-catalog container (debug-with-ice has ice CLI at /usr/local/bin/ice) + GenericContainer catalogContainer = + new GenericContainer<>(dockerImage) + .withNetwork(network) + .withExposedPorts(5000) + .withEnv("ICE_REST_CATALOG_CONFIG", "") + .withEnv("ICE_REST_CATALOG_CONFIG_YAML", catalogConfig) + .withFileSystemBind(scenariosDir.toString(), "/scenarios") + .waitingFor(Wait.forHttp("/v1/config").forPort(5000).forStatusCode(200)); + + catalog = catalogContainer; + try { + catalog.start(); + } catch (Exception e) { + if (catalog != null) { + logger.error("Catalog container logs (stdout): {}", catalog.getLogs()); + } + throw e; + } + + // Copy CLI config into container so ice CLI can talk to co-located REST server + File cliConfigHost = File.createTempFile("ice-docker-cli-", ".yaml"); + try { + Files.write(cliConfigHost.toPath(), "uri: http://localhost:5000\n".getBytes()); + catalog.copyFileToContainer( + MountableFile.forHostPath(cliConfigHost.toPath()), "/tmp/ice-cli.yaml"); + } finally { + cliConfigHost.delete(); + } + + logger.info( + "Catalog container started at {}:{}", catalog.getHost(), catalog.getMappedPort(5000)); + } + + @Override + @AfterClass + public void tearDown() { + if (catalog != null && catalog.isRunning()) { + catalog.stop(); + } + if (minio != null && minio.isRunning()) { + minio.stop(); + } + if (network != null) { + network.close(); + } + } + + @Override + protected ScenarioTestRunner createScenarioRunner(String scenarioName) throws Exception { + Path scenariosDir = getScenariosDirectory(); + + String containerId = catalog.getContainerId(); + + // Wrapper script on host: docker exec ice "$@" (CLI runs inside container) + File wrapperScript = File.createTempFile("ice-docker-exec-", ".sh"); + wrapperScript.deleteOnExit(); + String wrapperContent = "#!/bin/sh\n" + "exec docker exec " + containerId + " ice \"$@\"\n"; + Files.write(wrapperScript.toPath(), wrapperContent.getBytes()); + if (!wrapperScript.setExecutable(true)) { + throw new IllegalStateException("Could not set wrapper script executable: " + wrapperScript); + } + + Map templateVars = new HashMap<>(); + templateVars.put("ICE_CLI", wrapperScript.getAbsolutePath()); + templateVars.put("CLI_CONFIG", "/tmp/ice-cli.yaml"); + templateVars.put("SCENARIO_DIR", "/scenarios/" + scenarioName); + templateVars.put("MINIO_ENDPOINT", ""); + templateVars.put("CATALOG_URI", "http://localhost:5000"); + + return new ScenarioTestRunner(scenariosDir, templateVars); + } +} diff --git a/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/RESTCatalogTestBase.java b/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/RESTCatalogTestBase.java index d7ffcf85..54c94d40 100644 --- a/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/RESTCatalogTestBase.java +++ b/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/RESTCatalogTestBase.java @@ -14,7 +14,12 @@ import com.altinity.ice.rest.catalog.internal.config.Config; import java.io.File; import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.List; import java.util.Map; import org.apache.iceberg.catalog.Catalog; import org.eclipse.jetty.server.Server; @@ -23,6 +28,8 @@ import org.testcontainers.containers.GenericContainer; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import software.amazon.awssdk.regions.Region; @@ -152,4 +159,94 @@ protected String getMinioEndpoint() { protected String getCatalogUri() { return "http://localhost:8080"; } + + /** + * Get the path to the scenarios directory. + * + * @return Path to scenarios directory + * @throws URISyntaxException If the resource URL cannot be converted to a path + */ + protected Path getScenariosDirectory() throws URISyntaxException { + URL scenariosUrl = getClass().getClassLoader().getResource("scenarios"); + if (scenariosUrl == null) { + return Paths.get("src/test/resources/scenarios"); + } + return Paths.get(scenariosUrl.toURI()); + } + + /** + * Create a ScenarioTestRunner for the given scenario. Subclasses provide host or container-based + * CLI and config. + * + * @param scenarioName Name of the scenario (e.g. for container path resolution) + * @return Configured ScenarioTestRunner + * @throws Exception If there's an error creating the runner + */ + protected abstract ScenarioTestRunner createScenarioRunner(String scenarioName) throws Exception; + + /** Data provider that discovers all test scenarios. */ + @DataProvider(name = "scenarios") + public Object[][] scenarioProvider() throws Exception { + Path scenariosDir = getScenariosDirectory(); + ScenarioTestRunner runner = new ScenarioTestRunner(scenariosDir, Map.of()); + List scenarios = runner.discoverScenarios(); + + if (scenarios.isEmpty()) { + logger.warn("No test scenarios found in: {}", scenariosDir); + return new Object[0][0]; + } + + logger.info("Discovered {} test scenario(s): {}", scenarios.size(), scenarios); + + Object[][] data = new Object[scenarios.size()][1]; + for (int i = 0; i < scenarios.size(); i++) { + data[i][0] = scenarios.get(i); + } + return data; + } + + /** Parameterized test that executes a single scenario. */ + @Test(dataProvider = "scenarios") + public void testScenario(String scenarioName) throws Exception { + logger.info("====== Starting scenario test: {} ======", scenarioName); + + ScenarioTestRunner runner = createScenarioRunner(scenarioName); + ScenarioTestRunner.ScenarioResult result = runner.executeScenario(scenarioName); + + if (result.runScriptResult() != null) { + logger.info("Run script exit code: {}", result.runScriptResult().exitCode()); + } + if (result.verifyScriptResult() != null) { + logger.info("Verify script exit code: {}", result.verifyScriptResult().exitCode()); + } + + assertScenarioSuccess(scenarioName, result); + logger.info("====== Scenario test passed: {} ======", scenarioName); + } + + /** Assert that the scenario result indicates success; otherwise throw AssertionError. */ + protected void assertScenarioSuccess( + String scenarioName, ScenarioTestRunner.ScenarioResult result) { + if (result.isSuccess()) { + return; + } + StringBuilder errorMessage = new StringBuilder(); + errorMessage.append("Scenario '").append(scenarioName).append("' failed:\n"); + + if (result.runScriptResult() != null && result.runScriptResult().exitCode() != 0) { + errorMessage.append("\nRun script failed with exit code: "); + errorMessage.append(result.runScriptResult().exitCode()); + errorMessage.append("\nStdout:\n").append(result.runScriptResult().stdout()); + errorMessage.append("\nStderr:\n").append(result.runScriptResult().stderr()); + } + + if (result.verifyScriptResult() != null && result.verifyScriptResult().exitCode() != 0) { + errorMessage.append("\nVerify script failed with exit code: "); + errorMessage.append(result.verifyScriptResult().exitCode()); + errorMessage.append("\nStdout:\n").append(result.verifyScriptResult().stdout()); + errorMessage.append("\nStderr:\n").append(result.verifyScriptResult().stderr()); + } + + throw new AssertionError(errorMessage.toString()); + } } diff --git a/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/ScenarioBasedIT.java b/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/ScenarioBasedIT.java index b4351344..24b098e7 100644 --- a/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/ScenarioBasedIT.java +++ b/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/ScenarioBasedIT.java @@ -10,15 +10,10 @@ package com.altinity.ice.rest.catalog; import java.io.File; -import java.net.URISyntaxException; -import java.net.URL; import java.nio.file.Path; import java.nio.file.Paths; import java.util.HashMap; -import java.util.List; import java.util.Map; -import org.testng.annotations.DataProvider; -import org.testng.annotations.Test; /** * Scenario-based integration tests for ICE REST Catalog. @@ -28,92 +23,8 @@ */ public class ScenarioBasedIT extends RESTCatalogTestBase { - /** - * Data provider that discovers all test scenarios. - * - * @return Array of scenario names to be used as test parameters - * @throws Exception If there's an error discovering scenarios - */ - @DataProvider(name = "scenarios") - public Object[][] scenarioProvider() throws Exception { - Path scenariosDir = getScenariosDirectory(); - ScenarioTestRunner runner = createScenarioRunner(); - - List scenarios = runner.discoverScenarios(); - - if (scenarios.isEmpty()) { - logger.warn("No test scenarios found in: {}", scenariosDir); - return new Object[0][0]; - } - - logger.info("Discovered {} test scenario(s): {}", scenarios.size(), scenarios); - - // Convert to Object[][] for TestNG data provider - Object[][] data = new Object[scenarios.size()][1]; - for (int i = 0; i < scenarios.size(); i++) { - data[i][0] = scenarios.get(i); - } - return data; - } - - /** - * Parameterized test that executes a single scenario. - * - * @param scenarioName Name of the scenario to execute - * @throws Exception If the scenario execution fails - */ - @Test(dataProvider = "scenarios") - public void testScenario(String scenarioName) throws Exception { - logger.info("====== Starting scenario test: {} ======", scenarioName); - - ScenarioTestRunner runner = createScenarioRunner(); - ScenarioTestRunner.ScenarioResult result = runner.executeScenario(scenarioName); - - // Log results - if (result.runScriptResult() != null) { - logger.info("Run script exit code: {}", result.runScriptResult().exitCode()); - } - - if (result.verifyScriptResult() != null) { - logger.info("Verify script exit code: {}", result.verifyScriptResult().exitCode()); - } - - // Assert success - if (!result.isSuccess()) { - StringBuilder errorMessage = new StringBuilder(); - errorMessage.append("Scenario '").append(scenarioName).append("' failed:\n"); - - if (result.runScriptResult() != null && result.runScriptResult().exitCode() != 0) { - errorMessage.append("\nRun script failed with exit code: "); - errorMessage.append(result.runScriptResult().exitCode()); - errorMessage.append("\nStdout:\n"); - errorMessage.append(result.runScriptResult().stdout()); - errorMessage.append("\nStderr:\n"); - errorMessage.append(result.runScriptResult().stderr()); - } - - if (result.verifyScriptResult() != null && result.verifyScriptResult().exitCode() != 0) { - errorMessage.append("\nVerify script failed with exit code: "); - errorMessage.append(result.verifyScriptResult().exitCode()); - errorMessage.append("\nStdout:\n"); - errorMessage.append(result.verifyScriptResult().stdout()); - errorMessage.append("\nStderr:\n"); - errorMessage.append(result.verifyScriptResult().stderr()); - } - - throw new AssertionError(errorMessage.toString()); - } - - logger.info("====== Scenario test passed: {} ======", scenarioName); - } - - /** - * Create a ScenarioTestRunner with the appropriate template variables. - * - * @return Configured ScenarioTestRunner - * @throws Exception If there's an error creating the runner - */ - private ScenarioTestRunner createScenarioRunner() throws Exception { + @Override + protected ScenarioTestRunner createScenarioRunner(String scenarioName) throws Exception { Path scenariosDir = getScenariosDirectory(); // Create CLI config file @@ -143,22 +54,4 @@ private ScenarioTestRunner createScenarioRunner() throws Exception { return new ScenarioTestRunner(scenariosDir, templateVars); } - - /** - * Get the path to the scenarios directory. - * - * @return Path to scenarios directory - * @throws URISyntaxException If the resource URL cannot be converted to a path - */ - private Path getScenariosDirectory() throws URISyntaxException { - // Get the scenarios directory from test resources - URL scenariosUrl = getClass().getClassLoader().getResource("scenarios"); - - if (scenariosUrl == null) { - // If not found in resources, try relative to project - return Paths.get("src/test/resources/scenarios"); - } - - return Paths.get(scenariosUrl.toURI()); - } } diff --git a/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/ScenarioConfig.java b/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/ScenarioConfig.java index 17d5b169..3d6ae098 100644 --- a/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/ScenarioConfig.java +++ b/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/ScenarioConfig.java @@ -9,7 +9,6 @@ */ package com.altinity.ice.rest.catalog; -import java.util.List; import java.util.Map; /** @@ -21,17 +20,7 @@ public record ScenarioConfig( String name, String description, CatalogConfig catalogConfig, - Map env, - CloudResources cloudResources, - List phases) { + Map env) { public record CatalogConfig(String warehouse, String name, String uri) {} - - public record CloudResources(S3Resources s3, SqsResources sqs) {} - - public record S3Resources(List buckets) {} - - public record SqsResources(List queues) {} - - public record Phase(String name, String description) {} } diff --git a/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/ScenarioTestRunner.java b/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/ScenarioTestRunner.java index 83a6b19d..c74d22b0 100644 --- a/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/ScenarioTestRunner.java +++ b/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/ScenarioTestRunner.java @@ -105,7 +105,9 @@ public ScenarioResult executeScenario(String scenarioName) throws Exception { // Build template variables map Map templateVars = new HashMap<>(globalTemplateVars); - templateVars.put("SCENARIO_DIR", scenarioDir.toAbsolutePath().toString()); + if (!templateVars.containsKey("SCENARIO_DIR")) { + templateVars.put("SCENARIO_DIR", scenarioDir.toAbsolutePath().toString()); + } // Add environment variables from scenario config if (config.env() != null) { diff --git a/ice-rest-catalog/src/test/resources/scenarios/README.md b/ice-rest-catalog/src/test/resources/scenarios/README.md index 0b0ba237..feb6de3e 100644 --- a/ice-rest-catalog/src/test/resources/scenarios/README.md +++ b/ice-rest-catalog/src/test/resources/scenarios/README.md @@ -32,26 +32,6 @@ env: NAMESPACE_NAME: "test_ns" TABLE_NAME: "test_ns.table1" INPUT_FILE: "input.parquet" - -# Optional: Cloud resources needed (for future provisioning) -cloudResources: - s3: - buckets: - - "test-bucket" - sqs: - queues: - - "test-queue" - -# Optional: Test execution phases -phases: - - name: "setup" - description: "Initialize resources" - - name: "run" - description: "Execute main test logic" - - name: "verify" - description: "Verify results" - - name: "cleanup" - description: "Clean up resources" ``` ## Script Templates diff --git a/ice-rest-catalog/src/test/resources/scenarios/basic-operations/input.parquet b/ice-rest-catalog/src/test/resources/scenarios/basic-operations/input.parquet new file mode 100644 index 0000000000000000000000000000000000000000..028c64cf9f7d9ea46bfeeaf940482aaee90a1b54 GIT binary patch literal 2446 zcmZ`*L1-Ii7JgDj^8efVlXj z+h~)tReRFWhaP+iY7a$+Qgkq^*-PEa;-d=<2zpU(g6+Yl#l^)qB!vBb$#tBx9gt_{ z{qO(gz4v|Z{Vu=u=TMXW->5#yzNoj-c+To9(buwYj)=C>skKZhHy(ZdSt^wqi|&7y zN?D^(=3;JbG@?w3J<{(uM64qj3mr{oQZ#l|z@TipsH&9LfvnNEcG%`J+fs0g##Ncd zbUX7exWy$nkLrLEd4Po?Zml3GL2Z)^amx`evS2yK8Lb3r>3ybuE+Fm-Q=hc3= zNd<)4#fLqv(M9da4o6rocpN%8X>h^1rgkZHk(+Rj{l-b2p`%XBV1fC_E;2_-tzKG(RB zxnWH|)LsdNuF!!t4fhz?K0wudcULwRJSzEwmaEg@e2aLSeYTC_Yuo}$M_1AGM>@{zU?#KLo&SQ`ZEz>>IwEsSoA>Df=};m6B7Ba zYPh`*Hwha|&mO5BWFjo18v1$OCRwHj$n`+`nLnXJ50r-ichGNo)XJZ@VbL)d_zA^& zgo(2W(^`t(p^AoQy!rplOZJpnJOxz=gF0Hp^ypT|-Ly+QQoK(kQ1y)Z&I9Jjbf985Lr!D=ae)}z)e`vso zO&o&sm({zs?ko@E>94D|@2p?Hb@SG3Y2Mav-&lWt{rWOyZ9#kegVP=yoag^|!e@Gg zR#Gczopx`&49To~aQ*Kq?_Ildy}J63Wj#!Kcp~lO?yT09Z@zSM_5B-ns{_->!1U4j z$}g5&ZT0T6P6N}4Qw~T&A1>csU%mVB1W|q~?H1g>P2%Vob~O&ur@o@oSkK=Z%YQL6 zDPDEN;MBMrIbynhag-}9|3F6rxPXRw2n*3_$6p?|5J?1)tc-D%QCx}lac?$W} z(B!|*pF&9H>%5SA6e=JPiD-;}gstG4MkWHTQ+21e&5i48iNkM-0b*SR?d74NQmGPh0{{e!n B Date: Mon, 23 Feb 2026 17:36:25 -0600 Subject: [PATCH 2/9] feat: Add list partitions command. --- .../java/com/altinity/ice/cli/internal/cmd/ListPartitions.java | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 ice/src/main/java/com/altinity/ice/cli/internal/cmd/ListPartitions.java diff --git a/ice/src/main/java/com/altinity/ice/cli/internal/cmd/ListPartitions.java b/ice/src/main/java/com/altinity/ice/cli/internal/cmd/ListPartitions.java new file mode 100644 index 00000000..e69de29b From e8807cd128d6118a5bfa635756b80c48b28b3eca Mon Sep 17 00:00:00 2001 From: kanthi subramanian Date: Mon, 23 Feb 2026 17:38:33 -0600 Subject: [PATCH 3/9] feat: Implement list partitions command functionality --- .idea/.gitignore | 5 +++++ .idea/codeStyles/codeStyleConfig.xml | 5 +++++ .idea/compiler.xml | 23 +++++++++++++++++++++++ .idea/encodings.xml | 11 +++++++++++ .idea/jarRepositories.xml | 20 ++++++++++++++++++++ .idea/misc.xml | 13 +++++++++++++ .idea/vcs.xml | 6 ++++++ 7 files changed, 83 insertions(+) create mode 100644 .idea/.gitignore create mode 100644 .idea/codeStyles/codeStyleConfig.xml create mode 100644 .idea/compiler.xml create mode 100644 .idea/encodings.xml create mode 100644 .idea/jarRepositories.xml create mode 100644 .idea/misc.xml create mode 100644 .idea/vcs.xml diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 00000000..b58b603f --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,5 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Editor-based HTTP Client requests +/httpRequests/ diff --git a/.idea/codeStyles/codeStyleConfig.xml b/.idea/codeStyles/codeStyleConfig.xml new file mode 100644 index 00000000..a55e7a17 --- /dev/null +++ b/.idea/codeStyles/codeStyleConfig.xml @@ -0,0 +1,5 @@ + + + + \ No newline at end of file diff --git a/.idea/compiler.xml b/.idea/compiler.xml new file mode 100644 index 00000000..591b37b6 --- /dev/null +++ b/.idea/compiler.xml @@ -0,0 +1,23 @@ + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/encodings.xml b/.idea/encodings.xml new file mode 100644 index 00000000..2e150ec1 --- /dev/null +++ b/.idea/encodings.xml @@ -0,0 +1,11 @@ + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/jarRepositories.xml b/.idea/jarRepositories.xml new file mode 100644 index 00000000..712ab9d9 --- /dev/null +++ b/.idea/jarRepositories.xml @@ -0,0 +1,20 @@ + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 00000000..f1a132c6 --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,13 @@ + + + + {} + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 00000000..35eb1ddf --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file From cf998c020e8020aac93e51f6eca2cb6b125def95 Mon Sep 17 00:00:00 2001 From: kanthi subramanian Date: Mon, 23 Feb 2026 17:47:09 -0600 Subject: [PATCH 4/9] Added command to list partitions --- .../main/java/com/altinity/ice/cli/Main.java | 18 +++++ .../ice/cli/internal/cmd/ListPartitions.java | 76 +++++++++++++++++++ 2 files changed, 94 insertions(+) diff --git a/ice/src/main/java/com/altinity/ice/cli/Main.java b/ice/src/main/java/com/altinity/ice/cli/Main.java index a10e1da0..b89050cb 100644 --- a/ice/src/main/java/com/altinity/ice/cli/Main.java +++ b/ice/src/main/java/com/altinity/ice/cli/Main.java @@ -21,6 +21,7 @@ import com.altinity.ice.cli.internal.cmd.DescribeParquet; import com.altinity.ice.cli.internal.cmd.Insert; import com.altinity.ice.cli.internal.cmd.InsertWatch; +import com.altinity.ice.cli.internal.cmd.ListPartitions; import com.altinity.ice.cli.internal.cmd.Scan; import com.altinity.ice.cli.internal.config.Config; import com.altinity.ice.cli.internal.iceberg.rest.RESTCatalogFactory; @@ -594,6 +595,23 @@ void scanTable( } } + @CommandLine.Command(name = "list-partitions", description = "List partitions in a table.") + void listPartitions( + @CommandLine.Parameters( + arity = "1", + paramLabel = "", + description = "Table name (e.g. ns1.table1)") + String name, + @CommandLine.Option( + names = {"--json"}, + description = "Output JSON instead of YAML") + boolean json) + throws IOException { + try (RESTCatalog catalog = loadCatalog()) { + ListPartitions.run(catalog, TableIdentifier.parse(name), json); + } + } + @CommandLine.Command(name = "delete-table", description = "Delete table.") void deleteTable( @CommandLine.Parameters( diff --git a/ice/src/main/java/com/altinity/ice/cli/internal/cmd/ListPartitions.java b/ice/src/main/java/com/altinity/ice/cli/internal/cmd/ListPartitions.java index e69de29b..0fcad946 100644 --- a/ice/src/main/java/com/altinity/ice/cli/internal/cmd/ListPartitions.java +++ b/ice/src/main/java/com/altinity/ice/cli/internal/cmd/ListPartitions.java @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2025 Altinity Inc and/or its affiliates. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ +package com.altinity.ice.cli.internal.cmd; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.TreeSet; +import org.apache.iceberg.FileScanTask; +import org.apache.iceberg.PartitionField; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.io.CloseableIterable; +import org.apache.iceberg.rest.RESTCatalog; + +public final class ListPartitions { + + private ListPartitions() {} + + public static void run(RESTCatalog catalog, TableIdentifier tableId, boolean json) + throws IOException { + org.apache.iceberg.Table table = catalog.loadTable(tableId); + PartitionSpec spec = table.spec(); + + if (!spec.isPartitioned()) { + var result = new Result(tableId.toString(), List.of(), List.of()); + output(result, json); + return; + } + + List partitionSpec = new ArrayList<>(); + for (PartitionField field : spec.fields()) { + String sourceColumn = table.schema().findField(field.sourceId()).name(); + partitionSpec.add( + new PartitionFieldInfo(sourceColumn, field.name(), field.transform().toString())); + } + + TreeSet partitionPaths = new TreeSet<>(); + try (CloseableIterable tasks = table.newScan().planFiles()) { + for (FileScanTask task : tasks) { + String path = spec.partitionToPath(task.file().partition()); + partitionPaths.add(path); + } + } + + var result = + new Result(tableId.toString(), partitionSpec, new ArrayList<>(partitionPaths)); + output(result, json); + } + + private static void output(Result result, boolean json) throws IOException { + ObjectMapper mapper = + json + ? new ObjectMapper() + : new ObjectMapper(new YAMLFactory().enable(YAMLGenerator.Feature.MINIMIZE_QUOTES)); + mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); + System.out.println(mapper.writeValueAsString(result)); + } + + @JsonInclude(JsonInclude.Include.NON_NULL) + record Result(String table, List partitionSpec, List partitions) {} + + record PartitionFieldInfo(String sourceColumn, String name, String transform) {} + +} From f8a0577f82f9cface8d15ae236d7e78927d7c93b Mon Sep 17 00:00:00 2001 From: kanthi subramanian Date: Mon, 23 Feb 2026 19:21:33 -0600 Subject: [PATCH 5/9] Added idea directory to gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 52ecf198..4ae9503a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +.idea/ target /demo/data /demo/clickhouse-udfs.xml From 8bdc585668f6150c5879f160ed693a3db2d27af0 Mon Sep 17 00:00:00 2001 From: kanthi subramanian Date: Mon, 23 Feb 2026 19:22:24 -0600 Subject: [PATCH 6/9] Removed .idea --- .idea/.gitignore | 5 ----- .idea/codeStyles/codeStyleConfig.xml | 5 ----- .idea/compiler.xml | 23 ----------------------- .idea/encodings.xml | 11 ----------- .idea/jarRepositories.xml | 20 -------------------- .idea/misc.xml | 13 ------------- .idea/vcs.xml | 6 ------ 7 files changed, 83 deletions(-) delete mode 100644 .idea/.gitignore delete mode 100644 .idea/codeStyles/codeStyleConfig.xml delete mode 100644 .idea/compiler.xml delete mode 100644 .idea/encodings.xml delete mode 100644 .idea/jarRepositories.xml delete mode 100644 .idea/misc.xml delete mode 100644 .idea/vcs.xml diff --git a/.idea/.gitignore b/.idea/.gitignore deleted file mode 100644 index b58b603f..00000000 --- a/.idea/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -# Default ignored files -/shelf/ -/workspace.xml -# Editor-based HTTP Client requests -/httpRequests/ diff --git a/.idea/codeStyles/codeStyleConfig.xml b/.idea/codeStyles/codeStyleConfig.xml deleted file mode 100644 index a55e7a17..00000000 --- a/.idea/codeStyles/codeStyleConfig.xml +++ /dev/null @@ -1,5 +0,0 @@ - - - - \ No newline at end of file diff --git a/.idea/compiler.xml b/.idea/compiler.xml deleted file mode 100644 index 591b37b6..00000000 --- a/.idea/compiler.xml +++ /dev/null @@ -1,23 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/encodings.xml b/.idea/encodings.xml deleted file mode 100644 index 2e150ec1..00000000 --- a/.idea/encodings.xml +++ /dev/null @@ -1,11 +0,0 @@ - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/jarRepositories.xml b/.idea/jarRepositories.xml deleted file mode 100644 index 712ab9d9..00000000 --- a/.idea/jarRepositories.xml +++ /dev/null @@ -1,20 +0,0 @@ - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml deleted file mode 100644 index f1a132c6..00000000 --- a/.idea/misc.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - {} - - - - - \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml deleted file mode 100644 index 35eb1ddf..00000000 --- a/.idea/vcs.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file From 6b22bdbe8989875417b913b42d18d6baf89f77a5 Mon Sep 17 00:00:00 2001 From: kanthi subramanian Date: Mon, 23 Feb 2026 19:23:32 -0600 Subject: [PATCH 7/9] rolled back changes --- .bin/pre-release-docker | 5 ----- .../java/com/altinity/ice/rest/catalog/ScenarioConfig.java | 1 - 2 files changed, 6 deletions(-) diff --git a/.bin/pre-release-docker b/.bin/pre-release-docker index 990a5597..2671de32 100755 --- a/.bin/pre-release-docker +++ b/.bin/pre-release-docker @@ -8,11 +8,6 @@ export SKIP_VERIFY=1 export PATH="$(pwd)/.bin:$PATH" -echo >&2 'Building ice Docker image' -docker-build-ice -echo >&2 'Building ice-rest-catalog Docker image' -docker-build-ice-rest-catalog - echo >&2 'Pushing ice Docker image' docker-build-ice --push echo >&2 'Pushing ice-rest-catalog Docker image' diff --git a/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/ScenarioConfig.java b/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/ScenarioConfig.java index dbf3b97f..e7163456 100644 --- a/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/ScenarioConfig.java +++ b/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/ScenarioConfig.java @@ -9,7 +9,6 @@ */ package com.altinity.ice.rest.catalog; -import java.util.List; import java.util.Map; /** From 4c2bb4b1ac0ad656a21da54b9933d6590879dd35 Mon Sep 17 00:00:00 2001 From: kanthi subramanian Date: Mon, 23 Feb 2026 19:33:22 -0600 Subject: [PATCH 8/9] Fixed lint errors --- .../ice/cli/internal/cmd/ListPartitions.java | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/ice/src/main/java/com/altinity/ice/cli/internal/cmd/ListPartitions.java b/ice/src/main/java/com/altinity/ice/cli/internal/cmd/ListPartitions.java index 0fcad946..e19dc5fc 100644 --- a/ice/src/main/java/com/altinity/ice/cli/internal/cmd/ListPartitions.java +++ b/ice/src/main/java/com/altinity/ice/cli/internal/cmd/ListPartitions.java @@ -29,7 +29,7 @@ public final class ListPartitions { private ListPartitions() {} public static void run(RESTCatalog catalog, TableIdentifier tableId, boolean json) - throws IOException { + throws IOException { org.apache.iceberg.Table table = catalog.loadTable(tableId); PartitionSpec spec = table.spec(); @@ -43,7 +43,7 @@ public static void run(RESTCatalog catalog, TableIdentifier tableId, boolean jso for (PartitionField field : spec.fields()) { String sourceColumn = table.schema().findField(field.sourceId()).name(); partitionSpec.add( - new PartitionFieldInfo(sourceColumn, field.name(), field.transform().toString())); + new PartitionFieldInfo(sourceColumn, field.name(), field.transform().toString())); } TreeSet partitionPaths = new TreeSet<>(); @@ -54,16 +54,15 @@ public static void run(RESTCatalog catalog, TableIdentifier tableId, boolean jso } } - var result = - new Result(tableId.toString(), partitionSpec, new ArrayList<>(partitionPaths)); + var result = new Result(tableId.toString(), partitionSpec, new ArrayList<>(partitionPaths)); output(result, json); } private static void output(Result result, boolean json) throws IOException { ObjectMapper mapper = - json - ? new ObjectMapper() - : new ObjectMapper(new YAMLFactory().enable(YAMLGenerator.Feature.MINIMIZE_QUOTES)); + json + ? new ObjectMapper() + : new ObjectMapper(new YAMLFactory().enable(YAMLGenerator.Feature.MINIMIZE_QUOTES)); mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); System.out.println(mapper.writeValueAsString(result)); } @@ -72,5 +71,4 @@ private static void output(Result result, boolean json) throws IOException { record Result(String table, List partitionSpec, List partitions) {} record PartitionFieldInfo(String sourceColumn, String name, String transform) {} - } From 4b1b9ec1f5adcdf1741138eeec09e980d465a14c Mon Sep 17 00:00:00 2001 From: kanthi subramanian Date: Thu, 5 Mar 2026 20:28:29 -0600 Subject: [PATCH 9/9] Added test for list-partitions --- .../scenarios/insert-partitioned/run.sh.tmpl | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/ice-rest-catalog/src/test/resources/scenarios/insert-partitioned/run.sh.tmpl b/ice-rest-catalog/src/test/resources/scenarios/insert-partitioned/run.sh.tmpl index c6bdf2a6..0e6b07f7 100644 --- a/ice-rest-catalog/src/test/resources/scenarios/insert-partitioned/run.sh.tmpl +++ b/ice-rest-catalog/src/test/resources/scenarios/insert-partitioned/run.sh.tmpl @@ -15,8 +15,21 @@ INPUT_PATH="${SCENARIO_DIR}/${INPUT_FILE}" {{ICE_CLI}} --config {{CLI_CONFIG}} insert --create-table ${TABLE_NAME} ${INPUT_PATH} --partition="${PARTITION_SPEC}" echo "OK Inserted data with partitioning into table ${TABLE_NAME}" -# Describe the table to verify partitioning (if describe command exists) -# {{ICE_CLI}} --config {{CLI_CONFIG}} describe-table ${TABLE_NAME} +# List partitions and validate output +LIST_PARTITIONS_OUT=$(mktemp) +trap "rm -f '${LIST_PARTITIONS_OUT}'" EXIT +{{ICE_CLI}} --config {{CLI_CONFIG}} list-partitions ${TABLE_NAME} > "${LIST_PARTITIONS_OUT}" +if ! grep -q "partitions:" "${LIST_PARTITIONS_OUT}"; then + echo "FAIL: list-partitions output missing 'partitions:' section" + cat "${LIST_PARTITIONS_OUT}" + exit 1 +fi +if ! grep -qE -- "- *[^=]+=" "${LIST_PARTITIONS_OUT}"; then + echo "FAIL: list-partitions output has no partition entries (expected at least one key=value)" + cat "${LIST_PARTITIONS_OUT}" + exit 1 +fi +echo "OK Listed and validated partitions for ${TABLE_NAME}" # Cleanup {{ICE_CLI}} --config {{CLI_CONFIG}} delete-table ${TABLE_NAME}