iceberg-streaming-examples/python/pyproject.toml at main · aws-samples/iceberg-streaming-examples · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
[project]
name = "iceberg-streaming-examples-py"
version = "1.0.0"
description = "PySpark counterpart of the Apache Iceberg + Kafka streaming examples (Iceberg v3, Spark 4.0.2)"
readme = "README.md"
requires-python = ">=3.10,<3.13"
authors = [{ name = "acmanjon", email = "acmanjon@amazon.com" }]
dependencies = [
    # Apache Spark 4.0.2 (Scala 2.13 build). The connectors (Kafka, Avro, Protobuf) and the
    # Iceberg runtime jars are pulled at session start via spark.jars.packages, see
    # iceberg_streaming.common.jobconfig.
    "pyspark==4.0.2",
    # Kafka producers / consumers (native, no schema registry). 2.2+ supports Python 3.12/3.13.
    "kafka-python>=2.2,<3",
    # zstd compression codec for the high-throughput Kafka producers.
    "zstandard>=0.22",
    # Protocol Buffers runtime for the producers (the *_pb2 modules are generated from
    # ../src/main/protobuf/VehicleTelemetry.proto, see scripts/gen_proto.sh).
    "protobuf>=5.27,<6",
    # Avro (de)serialization for the Avro producer/consumer.
    "fastavro>=1.9",
    # AWS SDK, used by the Glue / S3 Tables catalogs and the standard credential chain.
    "boto3>=1.34",
]

[dependency-groups]
# Dev-only tooling. Installed with `uv sync` (uv installs dependency-groups by default) and used to
# (re)generate the protobuf python bindings and run the unit tests.
dev = [
    "grpcio-tools>=1.62",
    "pytest>=8.0",
]

[tool.pytest.ini_options]
testpaths = ["tests"]

[project.scripts]
iot-native-ingest = "iceberg_streaming.iot.spark_native_iceberg_ingest:main"
iot-custom-ingest = "iceberg_streaming.iot.spark_custom_iceberg_ingest:main"
proto-udf = "iceberg_streaming.iot.spark_proto_udf:main"
iceberg-utils = "iceberg_streaming.iot.spark_iceberg_utils:main"
cdc-log-change = "iceberg_streaming.cdc.spark_log_change:main"
cdc-mirror = "iceberg_streaming.cdc.spark_cdc_mirror:main"
cdc-incremental = "iceberg_streaming.cdc.spark_incremental_pipeline:main"
cdc-streaming-mirror = "iceberg_streaming.cdc.spark_streaming_cdc_mirror:main"
iceberg-maintenance = "iceberg_streaming.maintenance.iceberg_maintenance:main"
scenario = "iceberg_streaming.scenarios.runner:main"
telemetry-producer = "iceberg_streaming.kafka.telemetry_producer:main"
telemetry-consumer = "iceberg_streaming.kafka.telemetry_consumer:main"
cdc-simulator = "iceberg_streaming.kafka.kafka_cdc_simulator:main"

[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"

[tool.hatch.build.targets.wheel]
packages = ["src/iceberg_streaming"]

[tool.uv]
package = true