diff --git a/examples/localfileio/.ice-rest-catalog.yaml b/examples/localfileio/.ice-rest-catalog.yaml index 7fd6cb39..9ce97633 100644 --- a/examples/localfileio/.ice-rest-catalog.yaml +++ b/examples/localfileio/.ice-rest-catalog.yaml @@ -1,6 +1,5 @@ uri: jdbc:sqlite:file:data/ice-rest-catalog/catalog.sqlite?journal_mode=WAL&synchronous=OFF&journal_size_limit=500 -warehouse: file://warehouse -localFileIOBaseDir: data/ice-rest-catalog +warehouse: file:///tmp/ice-example/warehouse anonymousAccess: enabled: true accessConfig: diff --git a/examples/localfileio/.ice.yaml b/examples/localfileio/.ice.yaml index 785bcec5..f65cfe11 100644 --- a/examples/localfileio/.ice.yaml +++ b/examples/localfileio/.ice.yaml @@ -1,3 +1,4 @@ +warehouse: file:///tmp/ice-example/warehouse + uri: http://localhost:5000 -localFileIOBaseDir: data/ice-rest-catalog httpCacheDir: data/ice/http/cache diff --git a/examples/localfileio/README.md b/examples/localfileio/README.md index ff06271b..3aed701a 100644 --- a/examples/localfileio/README.md +++ b/examples/localfileio/README.md @@ -1,13 +1,15 @@ # examples/localfileio -This example is primarily intended for learning and experimentation. -All data is stored in data/ directory as regular files. +This is an example setup where Iceberg table data is stored on your local disk (under /tmp/ice-example/warehouse) instead of +in cloud object storage (S3, GCS, etc.). +This example is primarily intended for learning and experimentation, and development without cloud credentials. +Table data is stored under `/tmp/ice-example/warehouse` as regular files (see `warehouse` in `.ice-rest-catalog.yaml`). The catalog metadata stays under `data/ice-rest-catalog/`. ```shell # optional: open shell containing `sqlite3` (sqlite command line client) devbox shell -# start Iceberg REST Catalog server backed by sqlite with warehouse set to file://warehouse +# start Iceberg REST Catalog server backed by sqlite with warehouse set to file:///tmp/ice-example/warehouse ice-rest-catalog # insert data into catalog @@ -17,7 +19,7 @@ ice insert flowers.iris -p file://iris.parquet ice describe # list all warehouse files -find data/ice-rest-catalog/warehouse +find /tmp/ice-example/warehouse # inspect sqlite data sqlite3 data/ice-rest-catalog/catalog.sqlite @@ -28,12 +30,18 @@ sqlite> select * from iceberg_tables; sqlite> select * from iceberg_namespace_properties; sqlite> .quit -# open ClickHouse* shell, then try SQL below -docker run -it --rm --network host -v $(pwd)/data/ice-rest-catalog/warehouse:/warehouse \ - altinity/clickhouse-server:25.3.3.20186.altinityantalya clickhouse local +# open ClickHouse* shell, then try SQL below +# IMPORTANT: make sure mount path is set to /tmp +# For Linux: +docker run -it --rm --add-host=host.docker.internal:host-gateway --network host -v /tmp/ice-example/warehouse:/tmp/ice-example/warehouse \ + altinity/clickhouse-server:25.8.16.20002.altinityantalya clickhouse local +# For Mac: +docker run -it --rm --network host -v /tmp/ice-example/warehouse:/tmp/ice-example/warehouse \ + altinity/clickhouse-server:25.8.16.20002.altinityantalya clickhouse local + ``` -> \* currently this only works with altinity/clickhouse-server:25.3+ builds. +> currently this only works with altinity/clickhouse-server:25.3+ builds. ```sql -- enable Iceberg support (required as of 25.4.1.1795) @@ -43,18 +51,50 @@ SET allow_experimental_database_iceberg = 1; DROP DATABASE IF EXISTS ice; CREATE DATABASE ice - ENGINE = DataLakeCatalog('http://localhost:5000') + ENGINE = DataLakeCatalog('http://host.docker.internal:5000') SETTINGS catalog_type = 'rest', vended_credentials = false, warehouse = 'warehouse'; -SHOW TABLES FROM ice; - -- inspect SHOW DATABASES; SHOW TABLES FROM ice; + + +Query id: 8c671684-ec90-4e18-aacd-aee5fae2aeed + + ┌─name──────────┐ +1. │ flowers.iris │ + + + SHOW CREATE TABLE ice.`flowers.iris`; select count(*) from ice.`flowers.iris`; select * from ice.`flowers.iris` limit 10 FORMAT CSVWithNamesAndTypes; + +Query id: f3a1773d-ebeb-4d26-80fb-7d9e29b5c07e + + ┌─sepal.length─┬─sepal.width─┬─petal.length─┬─petal.width─┬─variety────┐ + 1. │ 5.1 │ 3.5 │ 1.4 │ 0.2 │ Setosa │ + 2. │ 4.9 │ 3 │ 1.4 │ 0.2 │ Setosa │ + 3. │ 4.7 │ 3.2 │ 1.3 │ 0.2 │ Setosa │ + 4. │ 4.6 │ 3.1 │ 1.5 │ 0.2 │ Setosa │ + 5. │ 5 │ 3.6 │ 1.4 │ 0.2 │ Setosa │ + 6. │ 5.4 │ 3.9 │ 1.7 │ 0.4 │ Setosa │ + 7. │ 4.6 │ 3.4 │ 1.4 │ 0.3 │ Setosa │ + 8. │ 5 │ 3.4 │ 1.5 │ 0.2 │ Setosa │ + 9. │ 4.4 │ 2.9 │ 1.4 │ 0.2 │ Setosa │ + 10. │ 4.9 │ 3.1 │ 1.5 │ 0.1 │ Setosa │ + 11. │ 5.4 │ 3.7 │ 1.5 │ 0.2 │ Setosa │ + 12. │ 4.8 │ 3.4 │ 1.6 │ 0.2 │ Setosa │ + 13. │ 4.8 │ 3 │ 1.4 │ 0.1 │ Setosa │ + 14. │ 4.3 │ 3 │ 1.1 │ 0.1 │ Setosa │ + 15. │ 5.8 │ 4 │ 1.2 │ 0.2 │ Setosa │ + 16. │ 5.7 │ 4.4 │ 1.5 │ 0.4 │ Setosa │ + 17. │ 5.4 │ 3.9 │ 1.3 │ 0.4 │ Setosa │ + 18. │ 5.1 │ 3.5 │ 1.4 │ 0.3 │ Setosa │ + 19. │ 5.7 │ 3.8 │ 1.7 │ ``` + +The REST catalog `warehouse` must be an absolute `file:///` URI (e.g. `file:///tmp/ice-example/warehouse`). A relative form like `file://warehouse` is rejected by the server. diff --git a/ice/src/main/java/com/altinity/ice/internal/iceberg/io/LocalFileIO.java b/ice/src/main/java/com/altinity/ice/internal/iceberg/io/LocalFileIO.java index 7247394f..2cbd151b 100644 --- a/ice/src/main/java/com/altinity/ice/internal/iceberg/io/LocalFileIO.java +++ b/ice/src/main/java/com/altinity/ice/internal/iceberg/io/LocalFileIO.java @@ -57,6 +57,11 @@ public void initialize(Map properties) { warehouse.startsWith("file://"), "\"%s\" must start with file://", LOCALFILEIO_PROP_WAREHOUSE); + Preconditions.checkArgument( + warehouse.startsWith("file:///") || warehouse.equals("file://"), + "\"%s\" must use an absolute path (file:///abs/path), got: %s", + LOCALFILEIO_PROP_WAREHOUSE, + warehouse); this.workDir = resolveWorkdir(warehouse, properties.get(LOCALFILEIO_PROP_BASEDIR)); Path warehousePath = resolveWarehousePath(warehouse, workDir); if (!Files.isDirectory(warehousePath)) { diff --git a/ice/src/test/java/com/altinity/ice/internal/iceberg/io/LocalFileIOIT.java b/ice/src/test/java/com/altinity/ice/internal/iceberg/io/LocalFileIOIT.java index 113979fe..22e31dee 100644 --- a/ice/src/test/java/com/altinity/ice/internal/iceberg/io/LocalFileIOIT.java +++ b/ice/src/test/java/com/altinity/ice/internal/iceberg/io/LocalFileIOIT.java @@ -13,11 +13,12 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; import com.altinity.ice.internal.strings.Strings; -import java.io.File; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Comparator; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.function.Function; @@ -58,14 +59,23 @@ public void tearDown() throws IOException { @Test public void testBasicFlow() throws IOException { - for (var warehouse : new String[] {"file://.", "file://", "file://x/y/z"}) { - tempDir.toFile().mkdirs(); - new File(tempDir.toString(), Strings.removePrefix(warehouse, "file://")).mkdirs(); - Path fooFile = tempDir.resolve(Strings.removePrefix(warehouse, "file://")).resolve("foo"); - Files.writeString(fooFile, "foo_content"); - Files.writeString( - tempDir.resolve(Strings.removePrefix(warehouse, "file://")).resolve("bar"), - "bar_content"); + Path absTemp = tempDir.toAbsolutePath().normalize(); + String[] warehouses = + new String[] { + absTemp.toUri().toString(), "file://", absTemp.resolve("x/y/z").toUri().toString() + }; + + for (String warehouse : warehouses) { + Path warehousePhysical = + "file://".equals(warehouse) + ? absTemp + : Paths.get(Strings.removePrefix(warehouse, "file://")); + Files.createDirectories(warehousePhysical); + Files.writeString(warehousePhysical.resolve("foo"), "foo_content"); + Files.writeString(warehousePhysical.resolve("bar"), "bar_content"); + + Path fooFile = warehousePhysical.resolve("foo"); + try (LocalFileIO io = new LocalFileIO()) { assertThatThrownBy( () -> @@ -81,12 +91,12 @@ public void testBasicFlow() throws IOException { Function warehouseLocation = (String s) -> (warehouse.endsWith("/") ? warehouse : warehouse + "/") + s; - io.initialize( - Map.of( - LocalFileIO.LOCALFILEIO_PROP_BASEDIR, - tempDir.toString(), - LocalFileIO.LOCALFILEIO_PROP_WAREHOUSE, - warehouse)); + Map props = new HashMap<>(); + props.put(LocalFileIO.LOCALFILEIO_PROP_WAREHOUSE, warehouse); + if ("file://".equals(warehouse)) { + props.put(LocalFileIO.LOCALFILEIO_PROP_BASEDIR, absTemp.toString()); + } + io.initialize(props); InputFile inputFile = io.newInputFile("foo"); OutputFile outputFile = io.newOutputFile("foo.out"); @@ -146,4 +156,15 @@ public void testBasicFlow() throws IOException { } } } + + @Test + public void testRelativeFileUriRejected() { + try (LocalFileIO io = new LocalFileIO()) { + assertThatThrownBy( + () -> + io.initialize(Map.of(LocalFileIO.LOCALFILEIO_PROP_WAREHOUSE, "file://warehouse"))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("absolute path"); + } + } }