Updates the dockerfiles for individual components and discarded main.py

etl-pyspark-v1.0/app/Dockerfile

@@ -0,0 +1,15 @@
+FROM python:3.11-slim-bullseye
+
+WORKDIR /app
+
+COPY nyc_taxi_etl.py .
+
+RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-recommends \
+    openjdk-17-jre-headless \
+    && rm -rf /var/lib/apt/lists/*
+RUN pip install --no-cache-dir pyspark==3.4.1 pyarrow pandas requests
+
+ENV JAVA_HOME=/usr/lib/jvm/java-17-openjdk-arm64
+ENV PATH=$JAVA_HOME/bin:$PATH
+
+CMD ["python", "nyc_taxi_etl.py"]

etl-pyspark-v1.0/test/local_app.sh

@@ -0,0 +1,80 @@
+#!/bin/bash
+# Runs the ETL app locally for each chunk
+# Place in etl-pyspark-v1.0/test/
+
+set -e
+
+# Set project root
+PROJECT_ROOT="$(realpath "$(dirname "$(pwd)")/../..")"
+cd "${PROJECT_ROOT}"
+
+# Validate script
+ETL_SCRIPT="${PROJECT_ROOT}/etl-pyspark-v1.0/app/nyc_taxi_etl.py"
+if [ ! -f "${ETL_SCRIPT}" ]; then
+    echo "Error: Script not found: ${ETL_SCRIPT}"
+    exit 1
+fi
+
+# Define directories
+SOURCE_DIR="${PROJECT_ROOT}/source_dir"
+CHUNKS_DIR="${PROJECT_ROOT}/chunks_dir"
+TASK_RESULTS_DIR="${PROJECT_ROOT}/task_results_dir"
+
+# Create directories
+for dir in "${SOURCE_DIR}" "${CHUNKS_DIR}" "${TASK_RESULTS_DIR}"; do
+    mkdir -p "${dir}"
+    chmod u+rwx "${dir}"
+done
+
+# Set environment variables
+export INPUT_DIR="${SOURCE_DIR}"
+export TASK_DIR="${CHUNKS_DIR}"
+export TASK_RESULTS_DIR="${TASK_RESULTS_DIR}"
+export PYTHONPATH="${PROJECT_ROOT}/etl-pyspark-v1.0/app:${PYTHONPATH}"
+
+# Log setup
+echo "Running ETL app locally"
+echo "Project root: ${PROJECT_ROOT}"
+echo "Python version: $(python3 --version)"
+echo "INPUT_DIR=${INPUT_DIR}"
+echo "TASK_DIR=${TASK_DIR}"
+echo "TASK_RESULTS_DIR=${TASK_RESULTS_DIR}"
+
+# Clear TASK_RESULTS_DIR
+rm -rf "${TASK_RESULTS_DIR}/*" 2>/dev/null || true
+
+# Process each chunk
+echo "Processing chunks from ${CHUNKS_DIR}..."
+if [ -z "$(ls -A "${CHUNKS_DIR}")" ]; then
+    echo "Error: No chunks found in ${CHUNKS_DIR}"
+    exit 1
+fi
+
+for chunk in "${CHUNKS_DIR}"/*.parquet; do
+    if [ -f "${chunk}" ]; then
+        chunk_name=$(basename "${chunk}")
+        task_id=$(echo "${chunk_name}" | sed 's/slice_\([0-9]*\)\.parquet/\1/')
+        input_path="${SOURCE_DIR}/data_${task_id}.bin"
+        echo "Processing chunk: ${chunk_name} (Task ID: ${task_id})"
+        cp "${chunk}" "${input_path}"
+        export INPUT_PATH="${input_path}"
+        export TASK_ID="${task_id}"
+        echo "Executing ${ETL_SCRIPT} for Task ID ${task_id}..."
+        python3 "${ETL_SCRIPT}"
+        rm -f "${input_path}"
+    fi
+done
+
+# Check output
+if [ -z "$(ls -A "${TASK_RESULTS_DIR}")" ]; then
+    echo "Error: No result files generated in ${TASK_RESULTS_DIR}"
+    exit 1
+fi
+echo "Result files generated:"
+ls "${TASK_RESULTS_DIR}"
+
+# Clean up environment variables
+unset INPUT_DIR TASK_DIR TASK_RESULTS_DIR INPUT_PATH TASK_ID
+export PYTHONPATH="${PYTHONPATH#${PROJECT_ROOT}/etl-pyspark-v1.0/app:}"
+
+echo "ETL app completed successfully."

parquet-assembler-v1.0/app/Dockerfile

@@ -0,0 +1,14 @@
+FROM python:3.11-slim
+
+WORKDIR /app
+
+# Update system packages and install security updates
+RUN apt-get update && apt-get upgrade -y && apt-get dist-upgrade -y && apt-get clean && rm -rf /var/lib/apt/lists/*
+
+COPY parquet_assembler.py .
+
+RUN apt-get update && apt-get upgrade -y && apt-get clean && rm -rf /var/lib/apt/lists/*
+RUN pip install --no-cache-dir --upgrade pip
+RUN pip install --no-cache-dir pyarrow pandas
+
+CMD ["python", "parquet_assembler.py"]