Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions examples/localfileio/.ice-rest-catalog.yaml
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
uri: jdbc:sqlite:file:data/ice-rest-catalog/catalog.sqlite?journal_mode=WAL&synchronous=OFF&journal_size_limit=500
warehouse: file://warehouse
localFileIOBaseDir: data/ice-rest-catalog
warehouse: file:///tmp/ice-example/warehouse
anonymousAccess:
enabled: true
accessConfig:
Expand Down
3 changes: 2 additions & 1 deletion examples/localfileio/.ice.yaml
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
warehouse: file:///tmp/ice-example/warehouse

uri: http://localhost:5000
localFileIOBaseDir: data/ice-rest-catalog
httpCacheDir: data/ice/http/cache
62 changes: 51 additions & 11 deletions examples/localfileio/README.md
Original file line number Diff line number Diff line change
@@ -1,13 +1,15 @@
# examples/localfileio

This example is primarily intended for learning and experimentation.
All data is stored in data/ directory as regular files.
This is an example setup where Iceberg table data is stored on your local disk (under /tmp/ice-example/warehouse) instead of
in cloud object storage (S3, GCS, etc.).
This example is primarily intended for learning and experimentation, and development without cloud credentials.
Table data is stored under `/tmp/ice-example/warehouse` as regular files (see `warehouse` in `.ice-rest-catalog.yaml`). The catalog metadata stays under `data/ice-rest-catalog/`.

```shell
# optional: open shell containing `sqlite3` (sqlite command line client)
devbox shell

# start Iceberg REST Catalog server backed by sqlite with warehouse set to file://warehouse
# start Iceberg REST Catalog server backed by sqlite with warehouse set to file:///tmp/ice-example/warehouse
ice-rest-catalog

# insert data into catalog
Expand All @@ -17,7 +19,7 @@ ice insert flowers.iris -p file://iris.parquet
ice describe

# list all warehouse files
find data/ice-rest-catalog/warehouse
find /tmp/ice-example/warehouse

# inspect sqlite data
sqlite3 data/ice-rest-catalog/catalog.sqlite
Expand All @@ -28,12 +30,18 @@ sqlite> select * from iceberg_tables;
sqlite> select * from iceberg_namespace_properties;
sqlite> .quit

# open ClickHouse* shell, then try SQL below
docker run -it --rm --network host -v $(pwd)/data/ice-rest-catalog/warehouse:/warehouse \
altinity/clickhouse-server:25.3.3.20186.altinityantalya clickhouse local
# open ClickHouse* shell, then try SQL below
# IMPORTANT: make sure mount path is set to /tmp
# For Linux:
docker run -it --rm --add-host=host.docker.internal:host-gateway --network host -v /tmp/ice-example/warehouse:/tmp/ice-example/warehouse \
altinity/clickhouse-server:25.8.16.20002.altinityantalya clickhouse local
# For Mac:
docker run -it --rm --network host -v /tmp/ice-example/warehouse:/tmp/ice-example/warehouse \
altinity/clickhouse-server:25.8.16.20002.altinityantalya clickhouse local

```

> \* currently this only works with altinity/clickhouse-server:25.3+ builds.
> currently this only works with altinity/clickhouse-server:25.3+ builds.

```sql
-- enable Iceberg support (required as of 25.4.1.1795)
Expand All @@ -43,18 +51,50 @@ SET allow_experimental_database_iceberg = 1;
DROP DATABASE IF EXISTS ice;

CREATE DATABASE ice
ENGINE = DataLakeCatalog('http://localhost:5000')
ENGINE = DataLakeCatalog('http://host.docker.internal:5000')
SETTINGS catalog_type = 'rest',
vended_credentials = false,
warehouse = 'warehouse';

SHOW TABLES FROM ice;

-- inspect
SHOW DATABASES;
SHOW TABLES FROM ice;


Query id: 8c671684-ec90-4e18-aacd-aee5fae2aeed

┌─name──────────┐
1. │ flowers.iris │



SHOW CREATE TABLE ice.`flowers.iris`;

select count(*) from ice.`flowers.iris`;
select * from ice.`flowers.iris` limit 10 FORMAT CSVWithNamesAndTypes;

Query id: f3a1773d-ebeb-4d26-80fb-7d9e29b5c07e

┌─sepal.length─┬─sepal.width─┬─petal.length─┬─petal.width─┬─variety────┐
1. │ 5.1 │ 3.5 │ 1.4 │ 0.2 │ Setosa │
2. │ 4.9 │ 3 │ 1.4 │ 0.2 │ Setosa │
3. │ 4.7 │ 3.2 │ 1.3 │ 0.2 │ Setosa │
4. │ 4.6 │ 3.1 │ 1.5 │ 0.2 │ Setosa │
5. │ 5 │ 3.6 │ 1.4 │ 0.2 │ Setosa │
6. │ 5.4 │ 3.9 │ 1.7 │ 0.4 │ Setosa │
7. │ 4.6 │ 3.4 │ 1.4 │ 0.3 │ Setosa │
8. │ 5 │ 3.4 │ 1.5 │ 0.2 │ Setosa │
9. │ 4.4 │ 2.9 │ 1.4 │ 0.2 │ Setosa │
10. │ 4.9 │ 3.1 │ 1.5 │ 0.1 │ Setosa │
11. │ 5.4 │ 3.7 │ 1.5 │ 0.2 │ Setosa │
12. │ 4.8 │ 3.4 │ 1.6 │ 0.2 │ Setosa │
13. │ 4.8 │ 3 │ 1.4 │ 0.1 │ Setosa │
14. │ 4.3 │ 3 │ 1.1 │ 0.1 │ Setosa │
15. │ 5.8 │ 4 │ 1.2 │ 0.2 │ Setosa │
16. │ 5.7 │ 4.4 │ 1.5 │ 0.4 │ Setosa │
17. │ 5.4 │ 3.9 │ 1.3 │ 0.4 │ Setosa │
18. │ 5.1 │ 3.5 │ 1.4 │ 0.3 │ Setosa │
19. │ 5.7 │ 3.8 │ 1.7 │
```

The REST catalog `warehouse` must be an absolute `file:///` URI (e.g. `file:///tmp/ice-example/warehouse`). A relative form like `file://warehouse` is rejected by the server.
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,11 @@ public void initialize(Map<String, String> properties) {
warehouse.startsWith("file://"),
"\"%s\" must start with file://",
LOCALFILEIO_PROP_WAREHOUSE);
Preconditions.checkArgument(
warehouse.startsWith("file:///") || warehouse.equals("file://"),
"\"%s\" must use an absolute path (file:///abs/path), got: %s",
LOCALFILEIO_PROP_WAREHOUSE,
warehouse);
this.workDir = resolveWorkdir(warehouse, properties.get(LOCALFILEIO_PROP_BASEDIR));
Path warehousePath = resolveWarehousePath(warehouse, workDir);
if (!Files.isDirectory(warehousePath)) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,12 @@
import static org.assertj.core.api.Assertions.assertThatThrownBy;

import com.altinity.ice.internal.strings.Strings;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
Expand Down Expand Up @@ -58,14 +59,23 @@ public void tearDown() throws IOException {

@Test
public void testBasicFlow() throws IOException {
for (var warehouse : new String[] {"file://.", "file://", "file://x/y/z"}) {
tempDir.toFile().mkdirs();
new File(tempDir.toString(), Strings.removePrefix(warehouse, "file://")).mkdirs();
Path fooFile = tempDir.resolve(Strings.removePrefix(warehouse, "file://")).resolve("foo");
Files.writeString(fooFile, "foo_content");
Files.writeString(
tempDir.resolve(Strings.removePrefix(warehouse, "file://")).resolve("bar"),
"bar_content");
Path absTemp = tempDir.toAbsolutePath().normalize();
String[] warehouses =
new String[] {
absTemp.toUri().toString(), "file://", absTemp.resolve("x/y/z").toUri().toString()
};

for (String warehouse : warehouses) {
Path warehousePhysical =
"file://".equals(warehouse)
? absTemp
: Paths.get(Strings.removePrefix(warehouse, "file://"));
Files.createDirectories(warehousePhysical);
Files.writeString(warehousePhysical.resolve("foo"), "foo_content");
Files.writeString(warehousePhysical.resolve("bar"), "bar_content");

Path fooFile = warehousePhysical.resolve("foo");

try (LocalFileIO io = new LocalFileIO()) {
assertThatThrownBy(
() ->
Expand All @@ -81,12 +91,12 @@ public void testBasicFlow() throws IOException {
Function<String, String> warehouseLocation =
(String s) -> (warehouse.endsWith("/") ? warehouse : warehouse + "/") + s;

io.initialize(
Map.of(
LocalFileIO.LOCALFILEIO_PROP_BASEDIR,
tempDir.toString(),
LocalFileIO.LOCALFILEIO_PROP_WAREHOUSE,
warehouse));
Map<String, String> props = new HashMap<>();
props.put(LocalFileIO.LOCALFILEIO_PROP_WAREHOUSE, warehouse);
if ("file://".equals(warehouse)) {
props.put(LocalFileIO.LOCALFILEIO_PROP_BASEDIR, absTemp.toString());
}
io.initialize(props);

InputFile inputFile = io.newInputFile("foo");
OutputFile outputFile = io.newOutputFile("foo.out");
Expand Down Expand Up @@ -146,4 +156,15 @@ public void testBasicFlow() throws IOException {
}
}
}

@Test
public void testRelativeFileUriRejected() {
try (LocalFileIO io = new LocalFileIO()) {
assertThatThrownBy(
() ->
io.initialize(Map.of(LocalFileIO.LOCALFILEIO_PROP_WAREHOUSE, "file://warehouse")))
.isInstanceOf(IllegalArgumentException.class)
.hasMessageContaining("absolute path");
}
}
}
Loading