From 47ba91e79a6855d363ae2b70a8476ed1a6521b81 Mon Sep 17 00:00:00 2001 From: Jinkun Liu Date: Wed, 1 Apr 2026 00:56:13 +0800 Subject: [PATCH 01/35] [tools]Add an installation wizard --- tools/install.sh | 838 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 838 insertions(+) create mode 100755 tools/install.sh diff --git a/tools/install.sh b/tools/install.sh new file mode 100755 index 000000000..97a313fad --- /dev/null +++ b/tools/install.sh @@ -0,0 +1,838 @@ +#!/bin/bash +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +set -euo pipefail + +BOLD='\033[1m' +ACCENT='\033[38;2;255;77;77m' # coral-bright +INFO='\033[38;2;136;146;176m' # text-secondary +SUCCESS='\033[38;2;0;229;204m' # cyan-bright +WARN='\033[38;2;255;176;32m' # amber +ERROR='\033[38;2;230;57;70m' # coral-mid +MUTED='\033[38;2;90;100;128m' # text-muted +NC='\033[0m' # No Color + +TMPFILES=() +cleanup_tmpfiles() { + local f + for f in "${TMPFILES[@]:-}"; do + rm -rf "$f" 2>/dev/null || true + done +} +trap cleanup_tmpfiles EXIT + +mktempfile() { + local f + f="$(mktemp)" + TMPFILES+=("$f") + echo "$f" +} + +DOWNLOADER="" +detect_downloader() { + if command -v curl &> /dev/null; then + DOWNLOADER="curl" + return 0 + fi + if command -v wget &> /dev/null; then + DOWNLOADER="wget" + return 0 + fi + ui_error "Missing downloader (curl or wget required)" + exit 1 +} + +download_file() { + local url="$1" + local output="$2" + if [[ -z "$DOWNLOADER" ]]; then + detect_downloader + fi + if [[ "$DOWNLOADER" == "curl" ]]; then + curl -fsSL --proto '=https' --tlsv1.2 --retry 3 --retry-delay 1 --retry-connrefused -o "$output" "$url" + return + fi + wget -q --https-only --secure-protocol=TLSv1_2 --tries=3 --timeout=20 -O "$output" "$url" +} + +GUM_VERSION="${FLINK_AGENTS_GUM_VERSION:-0.17.0}" +GUM="" +GUM_STATUS="skipped" +GUM_REASON="" + +is_non_interactive_shell() { + if [[ "${NO_PROMPT:-0}" == "1" ]]; then + return 0 + fi + if [[ ! -t 0 || ! -t 1 ]]; then + return 0 + fi + return 1 +} + +gum_is_tty() { + if [[ -n "${NO_COLOR:-}" ]]; then + return 1 + fi + if [[ "${TERM:-dumb}" == "dumb" ]]; then + return 1 + fi + if [[ -t 2 || -t 1 ]]; then + return 0 + fi + if [[ -r /dev/tty && -w /dev/tty ]]; then + return 0 + fi + return 1 +} + +gum_detect_os() { + case "$(uname -s 2>/dev/null || true)" in + Darwin) echo "Darwin" ;; + Linux) echo "Linux" ;; + *) echo "unsupported" ;; + esac +} + +gum_detect_arch() { + case "$(uname -m 2>/dev/null || true)" in + x86_64|amd64) echo "x86_64" ;; + arm64|aarch64) echo "arm64" ;; + i386|i686) echo "i386" ;; + armv7l|armv7) echo "armv7" ;; + armv6l|armv6) echo "armv6" ;; + *) echo "unknown" ;; + esac +} + +verify_sha256sum_file() { + local checksums="$1" + if command -v sha256sum >/dev/null 2>&1; then + sha256sum --ignore-missing -c "$checksums" >/dev/null 2>&1 + return $? + fi + if command -v shasum >/dev/null 2>&1; then + shasum -a 256 --ignore-missing -c "$checksums" >/dev/null 2>&1 + return $? + fi + return 1 +} + +bootstrap_gum_temp() { + GUM="" + GUM_STATUS="skipped" + GUM_REASON="" + + if is_non_interactive_shell; then + GUM_REASON="non-interactive shell (auto-disabled)" + return 1 + fi + + if ! gum_is_tty; then + GUM_REASON="terminal does not support gum UI" + return 1 + fi + + if command -v gum >/dev/null 2>&1; then + GUM="gum" + GUM_STATUS="found" + GUM_REASON="already installed" + return 0 + fi + + if ! command -v tar >/dev/null 2>&1; then + GUM_REASON="tar not found" + return 1 + fi + + local os arch asset base gum_tmpdir gum_path + os="$(gum_detect_os)" + arch="$(gum_detect_arch)" + if [[ "$os" == "unsupported" || "$arch" == "unknown" ]]; then + GUM_REASON="unsupported os/arch ($os/$arch)" + return 1 + fi + + asset="gum_${GUM_VERSION}_${os}_${arch}.tar.gz" + base="https://github.com/charmbracelet/gum/releases/download/v${GUM_VERSION}" + + gum_tmpdir="$(mktemp -d)" + TMPFILES+=("$gum_tmpdir") + + if ! download_file "${base}/${asset}" "$gum_tmpdir/$asset"; then + GUM_REASON="download failed" + return 1 + fi + + if ! download_file "${base}/checksums.txt" "$gum_tmpdir/checksums.txt"; then + GUM_REASON="checksum unavailable or failed" + return 1 + fi + + if ! (cd "$gum_tmpdir" && verify_sha256sum_file "checksums.txt"); then + GUM_REASON="checksum unavailable or failed" + return 1 + fi + + if ! tar -xzf "$gum_tmpdir/$asset" -C "$gum_tmpdir" >/dev/null 2>&1; then + GUM_REASON="extract failed" + return 1 + fi + + gum_path="$(find "$gum_tmpdir" -type f -name gum 2>/dev/null | head -n1 || true)" + if [[ -z "$gum_path" ]]; then + GUM_REASON="gum binary missing after extract" + return 1 + fi + + chmod +x "$gum_path" >/dev/null 2>&1 || true + if [[ ! -x "$gum_path" ]]; then + GUM_REASON="gum binary is not executable" + return 1 + fi + + GUM="$gum_path" + GUM_STATUS="installed" + GUM_REASON="temp, verified" + return 0 +} + +print_gum_status() { + case "$GUM_STATUS" in + found) + ui_success "gum available (${GUM_REASON})" + ;; + installed) + ui_success "gum bootstrapped (${GUM_REASON}, v${GUM_VERSION})" + ;; + *) + if [[ -n "$GUM_REASON" && "$GUM_REASON" != "non-interactive shell (auto-disabled)" ]]; then + ui_info "gum skipped (${GUM_REASON})" + fi + ;; + esac +} + +print_installer_banner() { + if [[ -n "$GUM" ]]; then + local title card + title="$("$GUM" style --foreground "#ff4d4d" --bold "Apache Flink Agents Installer")" + card="$(printf '%s\n' "$title")" + "$GUM" style --border rounded --border-foreground "#ff4d4d" --padding "1 2" "$card" + echo "" + return + fi + + echo -e "${ACCENT}${BOLD}" + echo " Apache Flink Agents Installer" + echo "" +} + +detect_os_or_die() { + OS="unknown" + if [[ "$OSTYPE" == "darwin"* ]]; then + OS="macos" + elif [[ "$OSTYPE" == "linux-gnu"* ]] || [[ -n "${WSL_DISTRO_NAME:-}" ]]; then + OS="linux" + fi + + if [[ "$OS" == "unknown" ]]; then + ui_error "Unsupported operating system" + echo "This installer supports macOS and Linux (including WSL)." + exit 1 + fi + + ui_success "Detected: $OS" +} + +ui_info() { + local msg="$*" + if [[ -n "$GUM" ]]; then + "$GUM" log --level info "$msg" + else + echo -e "${MUTED}·${NC} ${msg}" + fi +} + +ui_warn() { + local msg="$*" + if [[ -n "$GUM" ]]; then + "$GUM" log --level warn "$msg" + else + echo -e "${WARN}!${NC} ${msg}" + fi +} + +ui_success() { + local msg="$*" + if [[ -n "$GUM" ]]; then + local mark + mark="$("$GUM" style --foreground "#00e5cc" --bold "✓")" + echo "${mark} ${msg}" + else + echo -e "${SUCCESS}✓${NC} ${msg}" + fi +} + +ui_error() { + local msg="$*" + if [[ -n "$GUM" ]]; then + "$GUM" log --level error "$msg" + else + echo -e "${ERROR}x${NC} ${msg}" + fi +} + +INSTALL_STAGE_TOTAL=3 +INSTALL_STAGE_CURRENT=0 + +ui_section() { + local title="$1" + if [[ -n "$GUM" ]]; then + "$GUM" style --bold --foreground "#ff4d4d" --padding "1 0" "$title" + else + echo "" + echo -e "${ACCENT}${BOLD}${title}${NC}" + fi +} + +ui_stage() { + local title="$1" + INSTALL_STAGE_CURRENT=$((INSTALL_STAGE_CURRENT + 1)) + ui_section "[${INSTALL_STAGE_CURRENT}/${INSTALL_STAGE_TOTAL}] ${title}" +} + +ui_kv() { + local key="$1" + local value="$2" + if [[ -n "$GUM" ]]; then + local key_part value_part + key_part="$("$GUM" style --foreground "#5a6480" --width 20 "$key")" + value_part="$("$GUM" style --bold "$value")" + "$GUM" join --horizontal "$key_part" "$value_part" + else + echo -e "${MUTED}${key}:${NC} ${value}" + fi +} + +ui_celebrate() { + local msg="$1" + if [[ -n "$GUM" ]]; then + "$GUM" style --bold --foreground "#00e5cc" "$msg" + else + echo -e "${SUCCESS}${BOLD}${msg}${NC}" + fi +} + +is_shell_function() { + local name="${1:-}" + [[ -n "$name" ]] && declare -F "$name" >/dev/null 2>&1 +} + +is_gum_raw_mode_failure() { + local err_log="$1" + [[ -s "$err_log" ]] || return 1 + grep -Eiq '(setrawmode|inappropriate ioctl for device)' "$err_log" +} + +configure_verbose() { + if [[ "$VERBOSE" != "1" ]]; then + return 0 + fi + set -x +} + +FLINK_VERSION="${FLINK_VERSION:-2.2.0}" +FLINK_AGENTS_VERSION="${FLINK_AGENTS_VERSION:-0.2.1}" +FLINK_SCALA_VERSION="${FLINK_SCALA_VERSION:-2.12}" +FLINK_MAJOR_MINOR="${FLINK_VERSION%.*}" +FLINK_BASE_URL="${FLINK_BASE_URL:-https://dlcdn.apache.org/flink}" + +INSTALL_FLINK="${INSTALL_FLINK:-ask}" +ENABLE_PYFLINK="${ENABLE_PYFLINK:-ask}" +INSTALL_DIR="${INSTALL_DIR:-$HOME/.local/flink}" +VENV_DIR="${VENV_DIR:-.flink-agents-env}" +NO_PROMPT="${NO_PROMPT:-0}" +VERBOSE="${FLINK_AGENTS_VERBOSE:-0}" +DRY_RUN="${FLINK_AGENTS_DRY_RUN:-0}" +VERIFY_INSTALL="${FLINK_AGENTS_VERIFY_INSTALL:-0}" +HELP=0 +PYFLINK_ACTUALLY_ENABLED=0 + +ARCHIVE_NAME="flink-${FLINK_VERSION}-bin-scala_${FLINK_SCALA_VERSION}.tgz" +ARCHIVE_URL="${FLINK_BASE_URL}/flink-${FLINK_VERSION}/${ARCHIVE_NAME}" + +require_cmd() { + command -v "$1" >/dev/null 2>&1 || die "Missing required command: $1" +} + +is_valid_tgz() { + local archive="$1" + [[ -f "$archive" ]] || return 1 + tar -tzf "$archive" >/dev/null 2>&1 +} + +is_promptable() { + if [[ "$NO_PROMPT" == "1" ]]; then + return 1 + fi + if [[ -r /dev/tty && -w /dev/tty ]]; then + return 0 + fi + return 1 +} + +choose_install_method_interactive() { + local prompt="$1" + + if ! is_promptable; then + return 1 + fi + + if [[ -n "$GUM" ]] && gum_is_tty; then + local selection + selection="$("$GUM" choose \ + --header "$prompt" \ + --cursor-prefix "❯ " \ + "Yes" "No" < /dev/tty || true)" + [[ "$selection" == "Yes" ]] + return + fi + + local answer="" + printf '%s [y/n]: ' "$prompt" > /dev/tty + read -r answer < /dev/tty || true + + [[ "$answer" =~ ^[Yy]$ ]] +} + +install_flink_if_needed() { + case "$INSTALL_FLINK" in + yes|true|1) + ;; + no|false|0) + ui_info "Skipping Flink download/install (INSTALL_FLINK=${INSTALL_FLINK})." + return + ;; + ask) + if ! choose_install_method_interactive "Do you want this script to download and install Flink ${FLINK_VERSION}?"; then + ui_info "Skipping Flink download/install by user choice." + return + fi + ;; + *) + die "Unsupported INSTALL_FLINK value: ${INSTALL_FLINK}. Use: ask|yes|no" + ;; + esac + + detect_downloader + require_cmd tar + + if ! mkdir -p "$INSTALL_DIR"; then + die "Failed to create INSTALL_DIR=$INSTALL_DIR. Please run with proper permissions or set INSTALL_DIR to a writable path." + fi + if [[ -f "${INSTALL_DIR}/${ARCHIVE_NAME}" ]] && ! is_valid_tgz "${INSTALL_DIR}/${ARCHIVE_NAME}"; then + ui_warn "Existing archive is corrupted; re-downloading: ${INSTALL_DIR}/${ARCHIVE_NAME}" + rm -f "${INSTALL_DIR}/${ARCHIVE_NAME}" + fi + + if [[ ! -f "${INSTALL_DIR}/${ARCHIVE_NAME}" ]]; then + ui_info "Downloading ${ARCHIVE_URL}" + download_file "$ARCHIVE_URL" "${INSTALL_DIR}/${ARCHIVE_NAME}" + else + ui_info "Reusing existing archive: ${INSTALL_DIR}/${ARCHIVE_NAME}" + fi + + if ! is_valid_tgz "${INSTALL_DIR}/${ARCHIVE_NAME}"; then + die "Downloaded archive is invalid or truncated: ${INSTALL_DIR}/${ARCHIVE_NAME}" + fi + + if [[ -d "${INSTALL_DIR}/flink-${FLINK_VERSION}" ]] && [[ ! -d "${INSTALL_DIR}/flink-${FLINK_VERSION}/lib" ]]; then + ui_warn "Existing Flink home is incomplete; re-extracting: ${INSTALL_DIR}/flink-${FLINK_VERSION}" + rm -rf "${INSTALL_DIR}/flink-${FLINK_VERSION}" + fi + + if [[ ! -d "${INSTALL_DIR}/flink-${FLINK_VERSION}" ]]; then + ui_info "Extracting Flink to ${INSTALL_DIR}" + tar -xzf "${INSTALL_DIR}/${ARCHIVE_NAME}" -C "$INSTALL_DIR" + else + ui_info "Reusing existing Flink home: ${INSTALL_DIR}/flink-${FLINK_VERSION}" + fi + + export FLINK_HOME="${INSTALL_DIR}/flink-${FLINK_VERSION}" +} + +resolve_flink_home() { + if [[ -z "${FLINK_HOME:-}" ]]; then + die "FLINK_HOME is not set. Set FLINK_HOME or run with INSTALL_FLINK=yes." + fi + [[ -d "$FLINK_HOME" ]] || die "FLINK_HOME does not exist: $FLINK_HOME" + [[ -d "$FLINK_HOME/lib" ]] || die "Invalid FLINK_HOME (missing lib directory): $FLINK_HOME" +} + +copy_pyflink_jar() { + local pyflink_jar="$FLINK_HOME/opt/flink-python-${FLINK_VERSION}.jar" + [[ -f "$pyflink_jar" ]] || die "Missing required PyFlink jar: $pyflink_jar" + + ui_info "Copying PyFlink jar into Flink lib" + cp "$pyflink_jar" "$FLINK_HOME/lib/" +} + +setup_python_env() { + check_python || die "Python environment check failed. Please install Python >=3.10." + + if [[ ! -d "$VENV_DIR" ]]; then + ui_info "Creating virtual environment: $VENV_DIR" + python3 -m venv "$VENV_DIR" + else + ui_info "Reusing existing virtual environment: $VENV_DIR" + fi + + source "$VENV_DIR/bin/activate" + + export PIP_PROGRESS_BAR=off + export PIP_NO_COLOR=1 + export PIP_NO_INPUT=1 + + ui_info "Installing Python packages" + python -m pip install --upgrade pip + python -m pip install "flink-agents==${FLINK_AGENTS_VERSION}" "apache-flink==${FLINK_VERSION}" +} + +copy_flink_agents_jars() { + local pkg_root + pkg_root="$(python - <<'PY' +import pathlib +import flink_agents +print(pathlib.Path(flink_agents.__file__).resolve().parent) +PY +)" + + local version_lib_dir="${pkg_root}/lib/flink-${FLINK_MAJOR_MINOR}" + + [[ -d "$version_lib_dir" ]] || die "Flink Agents lib directory not found: $version_lib_dir" + + local copied=0 + local jar + for jar in "$version_lib_dir"/flink-agents-dist-*.jar; do + [[ -f "$jar" ]] || continue + cp "$jar" "$FLINK_HOME/lib/" + copied=1 + done + + [[ "$copied" -eq 1 ]] || die "No flink-agents-dist jar found in: $version_lib_dir" +} + +should_enable_pyflink() { + case "$ENABLE_PYFLINK" in + yes|true|1) + return 0 + ;; + no|false|0) + return 1 + ;; + ask) + if choose_install_method_interactive "Create a Python venv with PyFlink and flink-agents? (Only needed for Python API users; Java users can select No)"; then + return 0 + fi + return 1 + ;; + *) + die "Unsupported ENABLE_PYFLINK value: ${ENABLE_PYFLINK}. Use: ask|yes|no" + ;; + esac +} + +print_usage() { + cat </install.sh | bash -s -- [options] + +Options: + --yes, -y Non-interactive mode (accept all defaults) + --install-flink Download and install Apache Flink + --enable-pyflink Enable PyFlink and install Python packages + --verbose Print debug output (set -x) + --dry-run Print install plan without making changes + --verify Run post-install verification checks + --help, -h Show this help + +Environment variables: + FLINK_VERSION Flink version to install (default: 2.2.0) + FLINK_AGENTS_VERSION Flink Agents version to install (default: 0.3.0) + FLINK_SCALA_VERSION Scala version suffix (default: 2.12) + FLINK_BASE_URL Mirror base URL (default: https://dlcdn.apache.org/flink) + INSTALL_FLINK ask|yes|no (default: ask) + ENABLE_PYFLINK ask|yes|no (default: ask) + INSTALL_DIR Flink install directory (default: \$HOME/.local/flink) + VENV_DIR Python venv directory (default: .flink-agents-env) + NO_PROMPT 1 to disable all prompts + FLINK_AGENTS_VERBOSE 1 to enable verbose output + FLINK_AGENTS_DRY_RUN 1 to enable dry-run mode + FLINK_AGENTS_VERIFY_INSTALL 1 to enable post-install verification + +Examples: + bash install.sh --install-flink --enable-pyflink --yes + FLINK_VERSION=2.2.0 bash install.sh --verbose + bash install.sh --dry-run +EOF +} + +check_java() { + if ! command -v java &>/dev/null; then + ui_warn "Java not found on PATH" + if [[ "$OS" == "macos" ]]; then + ui_info "Install Java: brew install openjdk@17" + elif [[ "$OS" == "linux" ]]; then + ui_info "Install Java: sudo apt install openjdk-17-jdk (Debian/Ubuntu)" + ui_info " sudo yum install java-17-openjdk-devel (RHEL/CentOS)" + fi + ui_info "Flink requires Java 11+ to run" + return 1 + fi + + local java_version_output + java_version_output="$(java -version 2>&1 | head -n1)" + local java_major="" + java_major="$(echo "$java_version_output" | sed -n 's/.*version "\([0-9]*\).*/\1/p')" + + if [[ -z "$java_major" ]]; then + ui_warn "Could not parse Java version from: $java_version_output" + return 1 + fi + + if [[ "$java_major" -lt 11 ]]; then + ui_error "Java $java_major detected, but Flink requires Java 11+" + ui_info "Please upgrade your Java installation" + return 1 + fi + + ui_success "Java $java_major found" + + if [[ -z "${JAVA_HOME:-}" ]]; then + ui_info "JAVA_HOME is not set (Flink will try to detect it automatically)" + fi + return 0 +} + +check_python() { + if ! command -v python3 &>/dev/null; then + ui_error "python3 not found on PATH" + return 1 + fi + + local py_version_output + py_version_output="$(python3 -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")' 2>/dev/null || true)" + + if [[ -z "$py_version_output" ]]; then + ui_error "Could not parse python3 version" + return 1 + fi + + local py_major py_minor + py_major="${py_version_output%%.*}" + py_minor="${py_version_output##*.}" + + if [[ "$py_major" -ne 3 ]] || [[ "$py_minor" -lt 10 ]]; then + ui_error "Python $py_major.$py_minor detected, but Flink Agents requires Python >=3.10 and <3.12" + return 1 + fi + + ui_success "Python $py_major.$py_minor found" + return 0 +} + +show_install_plan() { + ui_section "Install plan" + ui_kv "OS" "$OS" + ui_kv "Flink version" "$FLINK_VERSION" + ui_kv "Flink Agents version" "$FLINK_AGENTS_VERSION" + ui_kv "Install Flink" "$INSTALL_FLINK" + ui_kv "Install directory" "$INSTALL_DIR" + ui_kv "Enable PyFlink" "$ENABLE_PYFLINK" + if [[ "$ENABLE_PYFLINK" =~ ^(yes|true|1)$ ]] || [[ "$PYFLINK_ACTUALLY_ENABLED" -eq 1 ]]; then + ui_kv "Venv directory" "$VENV_DIR" + fi + if [[ -n "${FLINK_HOME:-}" ]]; then + ui_kv "FLINK_HOME" "$FLINK_HOME" + fi + if [[ -n "${JAVA_HOME:-}" ]]; then + ui_kv "JAVA_HOME" "$JAVA_HOME" + fi + if [[ "$DRY_RUN" == "1" ]]; then + ui_kv "Dry run" "yes" + fi + if [[ "$VERIFY_INSTALL" == "1" ]]; then + ui_kv "Verify" "yes" + fi +} + +verify_installation() { + if [[ "${VERIFY_INSTALL}" != "1" ]]; then + return 0 + fi + + ui_stage "Verifying installation" + + if [[ -x "$FLINK_HOME/bin/flink" ]]; then + local flink_ver + flink_ver="$("$FLINK_HOME/bin/flink" --version 2>/dev/null || true)" + if [[ -n "$flink_ver" ]]; then + ui_success "Flink binary: $flink_ver" + else + ui_warn "Flink binary exists but --version failed" + fi + else + ui_warn "Flink binary not found at $FLINK_HOME/bin/flink" + fi + + if [[ "$PYFLINK_ACTUALLY_ENABLED" -eq 1 ]]; then + if python -c "import flink_agents; print('flink-agents', flink_agents.__version__)" 2>/dev/null; then + ui_success "flink-agents Python package verified" + else + ui_error "flink-agents Python package import failed" + return 1 + fi + + local agents_jar_found=0 + for jar in "$FLINK_HOME/lib"/flink-agents-dist-*.jar; do + [[ -f "$jar" ]] && agents_jar_found=1 && break + done + if [[ "$agents_jar_found" -eq 1 ]]; then + ui_success "flink-agents-dist JAR found in FLINK_HOME/lib" + else + ui_error "flink-agents-dist JAR not found in $FLINK_HOME/lib" + return 1 + fi + + if [[ -f "$FLINK_HOME/lib/flink-python-${FLINK_VERSION}.jar" ]]; then + ui_success "flink-python JAR found in FLINK_HOME/lib" + else + ui_warn "flink-python-${FLINK_VERSION}.jar not found in $FLINK_HOME/lib" + fi + fi + + ui_success "Verification complete" +} + +show_footer_links() { + local docs_url="https://nightlies.apache.org/flink/flink-agents-docs-latest/" + local issues_url="https://github.com/apache/flink-agents/issues" + echo "" + ui_info "Documentation: ${docs_url}" + ui_info "Report issues: ${issues_url}" +} + +parse_args() { + while [[ $# -gt 0 ]]; do + case "$1" in + --yes|-y) + NO_PROMPT=1 + shift + ;; + --install-flink) + INSTALL_FLINK=yes + shift + ;; + --enable-pyflink|--enable-pyFlink) + ENABLE_PYFLINK=yes + shift + ;; + --verbose) + VERBOSE=1 + shift + ;; + --dry-run) + DRY_RUN=1 + shift + ;; + --verify) + VERIFY_INSTALL=1 + shift + ;; + --help|-h) + HELP=1 + shift + ;; + *) + ui_warn "Unknown option: $1" + shift + ;; + esac + done +} + +main() { + if [[ "$HELP" == "1" ]]; then + print_usage + return 0 + fi + + bootstrap_gum_temp || true + print_installer_banner + print_gum_status + detect_os_or_die + check_java || die "Java environment check failed. Please install Java 11 or newer." + + show_install_plan + + if [[ "$DRY_RUN" == "1" ]]; then + ui_success "Dry run complete (no changes made)" + return 0 + fi + + ui_stage "Installing Apache Flink" + install_flink_if_needed + resolve_flink_home + + ui_stage "Setting up Python environment" + if should_enable_pyflink; then + PYFLINK_ACTUALLY_ENABLED=1 + copy_pyflink_jar + setup_python_env + copy_flink_agents_jars + else + ui_info "Skipping PyFlink and Python package setup (ENABLE_PYFLINK=${ENABLE_PYFLINK})." + fi + + ui_stage "Finalizing" + + verify_installation + + echo "" + ui_celebrate "Apache Flink Agents installation finished!" + ui_success "FLINK_HOME=$FLINK_HOME" + if [[ "$PYFLINK_ACTUALLY_ENABLED" -eq 1 ]]; then + ui_info "To use Python environment in a new shell :" + ui_info " source ${VENV_DIR}/bin/activate" + fi + ui_info " export FLINK_HOME=${FLINK_HOME}" + + show_footer_links +} + +if [[ "${FLINK_AGENTS_INSTALL_SH_NO_RUN:-0}" != "1" ]]; then + parse_args "$@" + configure_verbose + main +fi + From 9724d34bc675882e7a7d24eed449f574af1d2f36 Mon Sep 17 00:00:00 2001 From: Jinkun Liu Date: Mon, 27 Apr 2026 11:23:59 +0800 Subject: [PATCH 02/35] [tools] Improve install.sh with download timeout, progress hint, and CLI flag rename --- tools/install.sh | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/tools/install.sh b/tools/install.sh index 97a313fad..d5cec877e 100755 --- a/tools/install.sh +++ b/tools/install.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash ################################################################################ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file @@ -65,10 +65,10 @@ download_file() { detect_downloader fi if [[ "$DOWNLOADER" == "curl" ]]; then - curl -fsSL --proto '=https' --tlsv1.2 --retry 3 --retry-delay 1 --retry-connrefused -o "$output" "$url" + curl -fsSL --proto '=https' --tlsv1.2 --connect-timeout 5 --max-time 15 --retry 1 --retry-delay 1 --retry-connrefused -o "$output" "$url" return fi - wget -q --https-only --secure-protocol=TLSv1_2 --tries=3 --timeout=20 -O "$output" "$url" + wget -q --https-only --secure-protocol=TLSv1_2 --tries=2 --timeout=15 -O "$output" "$url" } GUM_VERSION="${FLINK_AGENTS_GUM_VERSION:-0.17.0}" @@ -175,6 +175,8 @@ bootstrap_gum_temp() { gum_tmpdir="$(mktemp -d)" TMPFILES+=("$gum_tmpdir") + echo -e "${INFO}· Installing gum v${GUM_VERSION}, please wait...${NC}" + if ! download_file "${base}/${asset}" "$gum_tmpdir/$asset"; then GUM_REASON="download failed" return 1 @@ -568,7 +570,7 @@ Usage: curl -fsSL /install.sh | bash -s -- [options] Options: - --yes, -y Non-interactive mode (accept all defaults) + --non-interactive Non-interactive mode (accept all defaults) --install-flink Download and install Apache Flink --enable-pyflink Enable PyFlink and install Python packages --verbose Print debug output (set -x) @@ -578,7 +580,7 @@ Options: Environment variables: FLINK_VERSION Flink version to install (default: 2.2.0) - FLINK_AGENTS_VERSION Flink Agents version to install (default: 0.3.0) + FLINK_AGENTS_VERSION Flink Agents version to install (default: 0.2.1) FLINK_SCALA_VERSION Scala version suffix (default: 2.12) FLINK_BASE_URL Mirror base URL (default: https://dlcdn.apache.org/flink) INSTALL_FLINK ask|yes|no (default: ask) @@ -591,7 +593,7 @@ Environment variables: FLINK_AGENTS_VERIFY_INSTALL 1 to enable post-install verification Examples: - bash install.sh --install-flink --enable-pyflink --yes + bash install.sh --install-flink --enable-pyflink --non-interactive FLINK_VERSION=2.2.0 bash install.sh --verbose bash install.sh --dry-run EOF @@ -745,7 +747,7 @@ show_footer_links() { parse_args() { while [[ $# -gt 0 ]]; do case "$1" in - --yes|-y) + --non-interactive) NO_PROMPT=1 shift ;; From 9e9887c9f81d7e6b22f7ef823a6cd772ce887cc8 Mon Sep 17 00:00:00 2001 From: Jinkun Liu Date: Mon, 27 Apr 2026 11:31:35 +0800 Subject: [PATCH 03/35] [tools]Install shell add python version limitation --- tools/install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/install.sh b/tools/install.sh index d5cec877e..4f3e2cf94 100755 --- a/tools/install.sh +++ b/tools/install.sh @@ -654,7 +654,7 @@ check_python() { py_major="${py_version_output%%.*}" py_minor="${py_version_output##*.}" - if [[ "$py_major" -ne 3 ]] || [[ "$py_minor" -lt 10 ]]; then + if [[ "$py_major" -ne 3 ]] || [[ "$py_minor" -lt 10 ]] || [[ "$py_minor" -ge 12 ]]; then ui_error "Python $py_major.$py_minor detected, but Flink Agents requires Python >=3.10 and <3.12" return 1 fi From 8b05ba7333d8a9449e7a9d1d85dc07e30ef2feb1 Mon Sep 17 00:00:00 2001 From: Jinkun Liu Date: Mon, 27 Apr 2026 11:47:58 +0800 Subject: [PATCH 04/35] [tools]install shell add common jar download --- tools/install.sh | 39 +++++++++++++++++++++++++++++++-------- 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/tools/install.sh b/tools/install.sh index 4f3e2cf94..084d0a8c7 100755 --- a/tools/install.sh +++ b/tools/install.sh @@ -527,17 +527,29 @@ PY )" local version_lib_dir="${pkg_root}/lib/flink-${FLINK_MAJOR_MINOR}" + local common_lib_dir="${pkg_root}/lib/common" [[ -d "$version_lib_dir" ]] || die "Flink Agents lib directory not found: $version_lib_dir" + [[ -d "$common_lib_dir" ]] || die "Flink Agents common lib directory not found: $common_lib_dir" - local copied=0 local jar + + ui_info "Copying Flink Agents common jar into Flink lib" + local common_copied=0 + for jar in "$common_lib_dir"/flink-agents-dist-common-*.jar; do + [[ -f "$jar" ]] || continue + cp "$jar" "$FLINK_HOME/lib/" + common_copied=1 + done + [[ "$common_copied" -eq 1 ]] || die "No flink-agents-dist-common jar found in: $common_lib_dir" + + ui_info "Copying Flink Agents thin jar into Flink lib" + local copied=0 for jar in "$version_lib_dir"/flink-agents-dist-*.jar; do [[ -f "$jar" ]] || continue cp "$jar" "$FLINK_HOME/lib/" copied=1 done - [[ "$copied" -eq 1 ]] || die "No flink-agents-dist jar found in: $version_lib_dir" } @@ -715,14 +727,25 @@ verify_installation() { return 1 fi - local agents_jar_found=0 - for jar in "$FLINK_HOME/lib"/flink-agents-dist-*.jar; do - [[ -f "$jar" ]] && agents_jar_found=1 && break + local common_jar_found=0 + for jar in "$FLINK_HOME/lib"/flink-agents-dist-common-*.jar; do + [[ -f "$jar" ]] && common_jar_found=1 && break + done + if [[ "$common_jar_found" -eq 1 ]]; then + ui_success "flink-agents-dist-common JAR found in FLINK_HOME/lib" + else + ui_error "flink-agents-dist-common JAR not found in $FLINK_HOME/lib" + return 1 + fi + + local thin_jar_found=0 + for jar in "$FLINK_HOME/lib"/flink-agents-dist-flink-*-thin.jar; do + [[ -f "$jar" ]] && thin_jar_found=1 && break done - if [[ "$agents_jar_found" -eq 1 ]]; then - ui_success "flink-agents-dist JAR found in FLINK_HOME/lib" + if [[ "$thin_jar_found" -eq 1 ]]; then + ui_success "flink-agents-dist thin JAR found in FLINK_HOME/lib" else - ui_error "flink-agents-dist JAR not found in $FLINK_HOME/lib" + ui_error "flink-agents-dist thin JAR not found in $FLINK_HOME/lib" return 1 fi From f316d46201efb85753e9f506b7dee8764416d3ec Mon Sep 17 00:00:00 2001 From: Jinkun Liu Date: Mon, 27 Apr 2026 13:50:25 +0800 Subject: [PATCH 05/35] [tools] install.sh: drop redundant true/1/false/0 aliases --- tools/install.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/install.sh b/tools/install.sh index 084d0a8c7..1acbcfefa 100755 --- a/tools/install.sh +++ b/tools/install.sh @@ -426,9 +426,9 @@ choose_install_method_interactive() { install_flink_if_needed() { case "$INSTALL_FLINK" in - yes|true|1) + yes) ;; - no|false|0) + no) ui_info "Skipping Flink download/install (INSTALL_FLINK=${INSTALL_FLINK})." return ;; @@ -555,10 +555,10 @@ PY should_enable_pyflink() { case "$ENABLE_PYFLINK" in - yes|true|1) + yes) return 0 ;; - no|false|0) + no) return 1 ;; ask) @@ -683,7 +683,7 @@ show_install_plan() { ui_kv "Install Flink" "$INSTALL_FLINK" ui_kv "Install directory" "$INSTALL_DIR" ui_kv "Enable PyFlink" "$ENABLE_PYFLINK" - if [[ "$ENABLE_PYFLINK" =~ ^(yes|true|1)$ ]] || [[ "$PYFLINK_ACTUALLY_ENABLED" -eq 1 ]]; then + if [[ "$ENABLE_PYFLINK" == "yes" ]] || [[ "$PYFLINK_ACTUALLY_ENABLED" -eq 1 ]]; then ui_kv "Venv directory" "$VENV_DIR" fi if [[ -n "${FLINK_HOME:-}" ]]; then From f045e4eeb705c533098ddade0dc79d17a30cfc80 Mon Sep 17 00:00:00 2001 From: Jinkun Liu Date: Mon, 27 Apr 2026 13:51:47 +0800 Subject: [PATCH 06/35] [tools] install.sh: define missing die helper --- tools/install.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tools/install.sh b/tools/install.sh index 1acbcfefa..094570274 100755 --- a/tools/install.sh +++ b/tools/install.sh @@ -301,6 +301,11 @@ ui_error() { fi } +die() { + ui_error "$*" + exit 1 +} + INSTALL_STAGE_TOTAL=3 INSTALL_STAGE_CURRENT=0 From 150a4256a74a5fa78508f4b4de3e9668d8ccb023 Mon Sep 17 00:00:00 2001 From: Jinkun Liu Date: Mon, 27 Apr 2026 14:12:39 +0800 Subject: [PATCH 07/35] [tools] install.sh: prompt for FLINK_HOME when missing or invalid --- tools/install.sh | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/tools/install.sh b/tools/install.sh index 094570274..db6add874 100755 --- a/tools/install.sh +++ b/tools/install.sh @@ -485,12 +485,33 @@ install_flink_if_needed() { export FLINK_HOME="${INSTALL_DIR}/flink-${FLINK_VERSION}" } +prompt_flink_home_interactive() { + local input="" + if [[ -n "$GUM" ]] && gum_is_tty; then + input="$("$GUM" input \ + --header "Enter the path to your existing FLINK_HOME" \ + --placeholder "/path/to/flink-${FLINK_VERSION}" \ + --width 70 < /dev/tty || true)" + else + printf 'Enter FLINK_HOME path: ' > /dev/tty + read -r input < /dev/tty || true + fi + input="${input/#\~/$HOME}" + printf '%s' "$input" +} + resolve_flink_home() { - if [[ -z "${FLINK_HOME:-}" ]]; then - die "FLINK_HOME is not set. Set FLINK_HOME or run with INSTALL_FLINK=yes." + if [[ -z "${FLINK_HOME:-}" || ! -d "${FLINK_HOME}" || ! -d "${FLINK_HOME}/lib" ]]; then + if is_promptable; then + FLINK_HOME="$(prompt_flink_home_interactive)" + export FLINK_HOME + fi fi + + [[ -n "${FLINK_HOME:-}" ]] || die "FLINK_HOME is not set." [[ -d "$FLINK_HOME" ]] || die "FLINK_HOME does not exist: $FLINK_HOME" [[ -d "$FLINK_HOME/lib" ]] || die "Invalid FLINK_HOME (missing lib directory): $FLINK_HOME" + ui_success "FLINK_HOME resolved: $FLINK_HOME" } copy_pyflink_jar() { From 45e46637a2c5bcd04cbcb717e8308ff07c2515d4 Mon Sep 17 00:00:00 2001 From: Jinkun Liu Date: Mon, 27 Apr 2026 15:22:35 +0800 Subject: [PATCH 08/35] [tools] install.sh: consolidate Flink resolution into a single function --- tools/install.sh | 115 ++++++++++++++++++++++++++++++++++++----------- 1 file changed, 88 insertions(+), 27 deletions(-) diff --git a/tools/install.sh b/tools/install.sh index db6add874..8355b45b4 100755 --- a/tools/install.sh +++ b/tools/install.sh @@ -65,10 +65,10 @@ download_file() { detect_downloader fi if [[ "$DOWNLOADER" == "curl" ]]; then - curl -fsSL --proto '=https' --tlsv1.2 --connect-timeout 5 --max-time 15 --retry 1 --retry-delay 1 --retry-connrefused -o "$output" "$url" + curl -fsSL --proto '=https' --tlsv1.2 --connect-timeout 5 --max-time 30 --retry 2 --retry-delay 1 --retry-connrefused -o "$output" "$url" return fi - wget -q --https-only --secure-protocol=TLSv1_2 --tries=2 --timeout=15 -O "$output" "$url" + wget -q --https-only --secure-protocol=TLSv1_2 --tries=2 --timeout=30 -O "$output" "$url" } GUM_VERSION="${FLINK_AGENTS_GUM_VERSION:-0.17.0}" @@ -365,12 +365,18 @@ configure_verbose() { set -x } +FLINK_VERSION_EXPLICIT=0 +if [[ -n "${FLINK_VERSION:-}" ]]; then + FLINK_VERSION_EXPLICIT=1 +fi FLINK_VERSION="${FLINK_VERSION:-2.2.0}" FLINK_AGENTS_VERSION="${FLINK_AGENTS_VERSION:-0.2.1}" FLINK_SCALA_VERSION="${FLINK_SCALA_VERSION:-2.12}" -FLINK_MAJOR_MINOR="${FLINK_VERSION%.*}" FLINK_BASE_URL="${FLINK_BASE_URL:-https://dlcdn.apache.org/flink}" +FLINK_SUPPORTED_VERSIONS=("2.2.0" "2.1.1" "2.0.1" "1.20.3") +FLINK_RECOMMENDED_VERSION="2.2.0" + INSTALL_FLINK="${INSTALL_FLINK:-ask}" ENABLE_PYFLINK="${ENABLE_PYFLINK:-ask}" INSTALL_DIR="${INSTALL_DIR:-$HOME/.local/flink}" @@ -382,9 +388,6 @@ VERIFY_INSTALL="${FLINK_AGENTS_VERIFY_INSTALL:-0}" HELP=0 PYFLINK_ACTUALLY_ENABLED=0 -ARCHIVE_NAME="flink-${FLINK_VERSION}-bin-scala_${FLINK_SCALA_VERSION}.tgz" -ARCHIVE_URL="${FLINK_BASE_URL}/flink-${FLINK_VERSION}/${ARCHIVE_NAME}" - require_cmd() { command -v "$1" >/dev/null 2>&1 || die "Missing required command: $1" } @@ -429,18 +432,67 @@ choose_install_method_interactive() { [[ "$answer" =~ ^[Yy]$ ]] } +prompt_flink_version_interactive() { + if ! is_promptable; then + return 1 + fi + + local labels=() + local v + for v in "${FLINK_SUPPORTED_VERSIONS[@]}"; do + if [[ "$v" == "$FLINK_RECOMMENDED_VERSION" ]]; then + labels+=("$v (recommended)") + else + labels+=("$v") + fi + done + + local selection="" + if [[ -n "$GUM" ]] && gum_is_tty; then + selection="$("$GUM" choose \ + --header "Select Flink version" \ + --cursor-prefix "❯ " \ + "${labels[@]}" < /dev/tty || true)" + selection="${selection%% *}" + else + printf 'Select Flink version:\n' > /dev/tty + local i=1 + for v in "${FLINK_SUPPORTED_VERSIONS[@]}"; do + local suffix="" + [[ "$v" == "$FLINK_RECOMMENDED_VERSION" ]] && suffix=" (recommended)" + printf ' %d) %s%s\n' "$i" "$v" "$suffix" > /dev/tty + i=$((i+1)) + done + local answer="" + printf 'Enter choice [1-%d, default %s]: ' \ + "${#FLINK_SUPPORTED_VERSIONS[@]}" "$FLINK_RECOMMENDED_VERSION" > /dev/tty + read -r answer < /dev/tty || true + if [[ "$answer" =~ ^[0-9]+$ ]] \ + && (( answer >= 1 && answer <= ${#FLINK_SUPPORTED_VERSIONS[@]} )); then + selection="${FLINK_SUPPORTED_VERSIONS[$((answer-1))]}" + fi + fi + + if [[ -z "$selection" ]]; then + selection="$FLINK_RECOMMENDED_VERSION" + fi + FLINK_VERSION="$selection" + return 0 +} + install_flink_if_needed() { + local skip_install=0 case "$INSTALL_FLINK" in yes) ;; no) ui_info "Skipping Flink download/install (INSTALL_FLINK=${INSTALL_FLINK})." - return + skip_install=1 ;; ask) - if ! choose_install_method_interactive "Do you want this script to download and install Flink ${FLINK_VERSION}?"; then + if ! choose_install_method_interactive "Do you want this script to download and install Apache Flink?"; then ui_info "Skipping Flink download/install by user choice." - return + skip_install=1 fi ;; *) @@ -448,6 +500,28 @@ install_flink_if_needed() { ;; esac + if [[ "$skip_install" -eq 1 ]]; then + if [[ -z "${FLINK_HOME:-}" || ! -d "${FLINK_HOME}" || ! -d "${FLINK_HOME}/lib" ]]; then + if is_promptable; then + FLINK_HOME="$(prompt_flink_home_interactive)" + export FLINK_HOME + fi + fi + [[ -n "${FLINK_HOME:-}" ]] || die "FLINK_HOME is not set." + [[ -d "$FLINK_HOME" ]] || die "FLINK_HOME does not exist: $FLINK_HOME" + [[ -d "$FLINK_HOME/lib" ]] || die "Invalid FLINK_HOME (missing lib directory): $FLINK_HOME" + ui_success "FLINK_HOME resolved: $FLINK_HOME" + FLINK_MAJOR_MINOR="${FLINK_VERSION%.*}" + return + fi + + if [[ "$FLINK_VERSION_EXPLICIT" -eq 0 ]]; then + prompt_flink_version_interactive || true + fi + + local ARCHIVE_NAME="flink-${FLINK_VERSION}-bin-scala_${FLINK_SCALA_VERSION}.tgz" + local ARCHIVE_URL="${FLINK_BASE_URL}/flink-${FLINK_VERSION}/${ARCHIVE_NAME}" + detect_downloader require_cmd tar @@ -483,37 +557,25 @@ install_flink_if_needed() { fi export FLINK_HOME="${INSTALL_DIR}/flink-${FLINK_VERSION}" + FLINK_MAJOR_MINOR="${FLINK_VERSION%.*}" + ui_success "FLINK_HOME resolved: $FLINK_HOME" } prompt_flink_home_interactive() { local input="" if [[ -n "$GUM" ]] && gum_is_tty; then input="$("$GUM" input \ - --header "Enter the path to your existing FLINK_HOME" \ + --header "Enter the path to your existing FLINK_HOME which version >= 1.20" \ --placeholder "/path/to/flink-${FLINK_VERSION}" \ --width 70 < /dev/tty || true)" else - printf 'Enter FLINK_HOME path: ' > /dev/tty + printf 'Enter the path to your existing FLINK_HOME which version >= 1.20: ' > /dev/tty read -r input < /dev/tty || true fi input="${input/#\~/$HOME}" printf '%s' "$input" } -resolve_flink_home() { - if [[ -z "${FLINK_HOME:-}" || ! -d "${FLINK_HOME}" || ! -d "${FLINK_HOME}/lib" ]]; then - if is_promptable; then - FLINK_HOME="$(prompt_flink_home_interactive)" - export FLINK_HOME - fi - fi - - [[ -n "${FLINK_HOME:-}" ]] || die "FLINK_HOME is not set." - [[ -d "$FLINK_HOME" ]] || die "FLINK_HOME does not exist: $FLINK_HOME" - [[ -d "$FLINK_HOME/lib" ]] || die "Invalid FLINK_HOME (missing lib directory): $FLINK_HOME" - ui_success "FLINK_HOME resolved: $FLINK_HOME" -} - copy_pyflink_jar() { local pyflink_jar="$FLINK_HOME/opt/flink-python-${FLINK_VERSION}.jar" [[ -f "$pyflink_jar" ]] || die "Missing required PyFlink jar: $pyflink_jar" @@ -617,7 +679,7 @@ Options: --help, -h Show this help Environment variables: - FLINK_VERSION Flink version to install (default: 2.2.0) + FLINK_VERSION Flink version (default: 2.2.0; supported: 2.2.0, 2.1.1, 2.0.1, 1.20.3) FLINK_AGENTS_VERSION Flink Agents version to install (default: 0.2.1) FLINK_SCALA_VERSION Scala version suffix (default: 2.12) FLINK_BASE_URL Mirror base URL (default: https://dlcdn.apache.org/flink) @@ -853,7 +915,6 @@ main() { ui_stage "Installing Apache Flink" install_flink_if_needed - resolve_flink_home ui_stage "Setting up Python environment" if should_enable_pyflink; then From 0823d1ef4ef8a27351826f4faa4f1fc23acf9fa2 Mon Sep 17 00:00:00 2001 From: Jinkun Liu Date: Mon, 27 Apr 2026 17:11:29 +0800 Subject: [PATCH 09/35] [tools]install.sh: support define install location by user --- tools/install.sh | 80 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/tools/install.sh b/tools/install.sh index 8355b45b4..7d5e76012 100755 --- a/tools/install.sh +++ b/tools/install.sh @@ -379,7 +379,15 @@ FLINK_RECOMMENDED_VERSION="2.2.0" INSTALL_FLINK="${INSTALL_FLINK:-ask}" ENABLE_PYFLINK="${ENABLE_PYFLINK:-ask}" +INSTALL_DIR_EXPLICIT=0 +if [[ -n "${INSTALL_DIR:-}" ]]; then + INSTALL_DIR_EXPLICIT=1 +fi INSTALL_DIR="${INSTALL_DIR:-$HOME/.local/flink}" +VENV_DIR_EXPLICIT=0 +if [[ -n "${VENV_DIR:-}" ]]; then + VENV_DIR_EXPLICIT=1 +fi VENV_DIR="${VENV_DIR:-.flink-agents-env}" NO_PROMPT="${NO_PROMPT:-0}" VERBOSE="${FLINK_AGENTS_VERBOSE:-0}" @@ -525,6 +533,13 @@ install_flink_if_needed() { detect_downloader require_cmd tar + if [[ "$INSTALL_DIR_EXPLICIT" -eq 0 ]] && is_promptable; then + INSTALL_DIR="$(prompt_path_choice_interactive \ + "Choose Flink install directory" \ + "$INSTALL_DIR" \ + "/path/to/flink-install-dir")" + fi + if ! mkdir -p "$INSTALL_DIR"; then die "Failed to create INSTALL_DIR=$INSTALL_DIR. Please run with proper permissions or set INSTALL_DIR to a writable path." fi @@ -576,6 +591,64 @@ prompt_flink_home_interactive() { printf '%s' "$input" } +prompt_path_choice_interactive() { + local header="$1" + local default_path="$2" + local placeholder="$3" + + if ! is_promptable; then + printf '%s' "$default_path" + return 0 + fi + + local default_label="Default: ${default_path}" + local custom_label="Custom..." + local selection="" + + if [[ -n "$GUM" ]] && gum_is_tty; then + selection="$("$GUM" choose \ + --header "$header" \ + --cursor-prefix "❯ " \ + "$default_label" "$custom_label" < /dev/tty || true)" + else + printf '%s\n' "$header" > /dev/tty + printf ' 1) %s\n' "$default_label" > /dev/tty + printf ' 2) %s\n' "$custom_label" > /dev/tty + local answer="" + printf 'Enter choice [1-2, default 1]: ' > /dev/tty + read -r answer < /dev/tty || true + case "$answer" in + 2) selection="$custom_label" ;; + *) selection="$default_label" ;; + esac + fi + + if [[ "$selection" != "$custom_label" ]]; then + printf '%s' "$default_path" + return 0 + fi + + local input="" + if [[ -n "$GUM" ]] && gum_is_tty; then + input="$("$GUM" input \ + --header "$header" \ + --placeholder "$placeholder" \ + --width 70 < /dev/tty || true)" + else + printf 'Enter custom path: ' > /dev/tty + read -r input < /dev/tty || true + fi + input="${input/#\~/$HOME}" + + if [[ -z "$input" ]]; then + ui_warn "Empty path; falling back to default: $default_path" + printf '%s' "$default_path" + return 0 + fi + + printf '%s' "$input" +} + copy_pyflink_jar() { local pyflink_jar="$FLINK_HOME/opt/flink-python-${FLINK_VERSION}.jar" [[ -f "$pyflink_jar" ]] || die "Missing required PyFlink jar: $pyflink_jar" @@ -587,6 +660,13 @@ copy_pyflink_jar() { setup_python_env() { check_python || die "Python environment check failed. Please install Python >=3.10." + if [[ "$VENV_DIR_EXPLICIT" -eq 0 ]] && is_promptable; then + VENV_DIR="$(prompt_path_choice_interactive \ + "Choose Python venv directory" \ + "$VENV_DIR" \ + "/path/to/venv")" + fi + if [[ ! -d "$VENV_DIR" ]]; then ui_info "Creating virtual environment: $VENV_DIR" python3 -m venv "$VENV_DIR" From 68c9f6ccc920447254dfd310d5499a90abcdfae9 Mon Sep 17 00:00:00 2001 From: Jinkun Liu Date: Mon, 27 Apr 2026 18:13:11 +0800 Subject: [PATCH 10/35] [tools]install.sh support define python bin by user --- tools/install.sh | 109 ++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 93 insertions(+), 16 deletions(-) diff --git a/tools/install.sh b/tools/install.sh index 7d5e76012..db5e7ea42 100755 --- a/tools/install.sh +++ b/tools/install.sh @@ -389,6 +389,7 @@ if [[ -n "${VENV_DIR:-}" ]]; then VENV_DIR_EXPLICIT=1 fi VENV_DIR="${VENV_DIR:-.flink-agents-env}" +PYTHON_BIN="${PYTHON_BIN:-}" NO_PROMPT="${NO_PROMPT:-0}" VERBOSE="${FLINK_AGENTS_VERBOSE:-0}" DRY_RUN="${FLINK_AGENTS_DRY_RUN:-0}" @@ -657,8 +658,22 @@ copy_pyflink_jar() { cp "$pyflink_jar" "$FLINK_HOME/lib/" } +create_venv() { + local venv_err + venv_err="$(mktempfile)" + if "$PYTHON_BIN" -m venv "$VENV_DIR" 2>"$venv_err"; then + return 0 + fi + + ui_error "Failed to create virtual environment at $VENV_DIR" + if [[ -s "$venv_err" ]]; then + sed 's/^/ /' "$venv_err" >&2 || true + fi + die "Virtual environment creation failed" +} + setup_python_env() { - check_python || die "Python environment check failed. Please install Python >=3.10." + resolve_python if [[ "$VENV_DIR_EXPLICIT" -eq 0 ]] && is_promptable; then VENV_DIR="$(prompt_path_choice_interactive \ @@ -669,7 +684,7 @@ setup_python_env() { if [[ ! -d "$VENV_DIR" ]]; then ui_info "Creating virtual environment: $VENV_DIR" - python3 -m venv "$VENV_DIR" + create_venv else ui_info "Reusing existing virtual environment: $VENV_DIR" fi @@ -681,7 +696,6 @@ setup_python_env() { export PIP_NO_INPUT=1 ui_info "Installing Python packages" - python -m pip install --upgrade pip python -m pip install "flink-agents==${FLINK_AGENTS_VERSION}" "apache-flink==${FLINK_VERSION}" } @@ -756,6 +770,7 @@ Options: --verbose Print debug output (set -x) --dry-run Print install plan without making changes --verify Run post-install verification checks + --python Path to a Python3 interpreter (overrides PATH lookup) --help, -h Show this help Environment variables: @@ -767,6 +782,7 @@ Environment variables: ENABLE_PYFLINK ask|yes|no (default: ask) INSTALL_DIR Flink install directory (default: \$HOME/.local/flink) VENV_DIR Python venv directory (default: .flink-agents-env) + PYTHON_BIN Path to Python3 interpreter (default: auto-detect python3 on PATH) NO_PROMPT 1 to disable all prompts FLINK_AGENTS_VERBOSE 1 to enable verbose output FLINK_AGENTS_DRY_RUN 1 to enable dry-run mode @@ -816,30 +832,77 @@ check_java() { return 0 } -check_python() { - if ! command -v python3 &>/dev/null; then - ui_error "python3 not found on PATH" - return 1 - fi +validate_python_bin() { + local bin="$1" + [[ -n "$bin" ]] || return 1 + command -v "$bin" >/dev/null 2>&1 || return 1 local py_version_output - py_version_output="$(python3 -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")' 2>/dev/null || true)" - - if [[ -z "$py_version_output" ]]; then - ui_error "Could not parse python3 version" - return 1 - fi + py_version_output="$("$bin" -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")' 2>/dev/null || true)" + [[ -n "$py_version_output" ]] || return 1 local py_major py_minor py_major="${py_version_output%%.*}" py_minor="${py_version_output##*.}" if [[ "$py_major" -ne 3 ]] || [[ "$py_minor" -lt 10 ]] || [[ "$py_minor" -ge 12 ]]; then - ui_error "Python $py_major.$py_minor detected, but Flink Agents requires Python >=3.10 and <3.12" return 1 fi + return 0 +} + +prompt_python_bin_interactive() { + local input="" + if [[ -n "$GUM" ]] && gum_is_tty; then + input="$("$GUM" input \ + --header "Enter path to a Python interpreter" \ + --placeholder "/path/to/python3" \ + --width 70 < /dev/tty || true)" + else + printf 'Enter path to a Python interpreter: ' > /dev/tty + read -r input < /dev/tty || true + fi + input="${input/#\~/$HOME}" + printf '%s' "$input" +} + +resolve_python() { + if [[ -n "$PYTHON_BIN" ]]; then + if validate_python_bin "$PYTHON_BIN"; then + ui_success "Using Python: $PYTHON_BIN" + return 0 + fi + die "PYTHON_BIN is invalid or unsupported (need >=3.10 and <3.12): $PYTHON_BIN" + fi + + if validate_python_bin python3; then + PYTHON_BIN="python3" + local v + v="$(python3 -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")' 2>/dev/null)" + ui_success "Python $v found on PATH" + return 0 + fi + + if command -v python3 >/dev/null 2>&1; then + ui_warn "python3 on PATH is incompatible (Flink Agents requires Python >=3.10 and <3.12)" + else + ui_warn "python3 not found on PATH" + fi + + if ! is_promptable; then + die "No compatible Python found. Set PYTHON_BIN or pass --python ." + fi - ui_success "Python $py_major.$py_minor found" + local input + input="$(prompt_python_bin_interactive)" + if [[ -z "$input" ]]; then + die "No Python interpreter provided." + fi + if ! validate_python_bin "$input"; then + die "Provided Python is invalid or unsupported (need >=3.10 and <3.12): $input" + fi + PYTHON_BIN="$input" + ui_success "Using Python: $PYTHON_BIN" return 0 } @@ -853,6 +916,9 @@ show_install_plan() { ui_kv "Enable PyFlink" "$ENABLE_PYFLINK" if [[ "$ENABLE_PYFLINK" == "yes" ]] || [[ "$PYFLINK_ACTUALLY_ENABLED" -eq 1 ]]; then ui_kv "Venv directory" "$VENV_DIR" + if [[ -n "$PYTHON_BIN" ]]; then + ui_kv "Python interpreter" "$PYTHON_BIN" + fi fi if [[ -n "${FLINK_HOME:-}" ]]; then ui_kv "FLINK_HOME" "$FLINK_HOME" @@ -962,6 +1028,17 @@ parse_args() { VERIFY_INSTALL=1 shift ;; + --python) + if [[ $# -lt 2 ]]; then + die "--python requires a path argument" + fi + PYTHON_BIN="$2" + shift 2 + ;; + --python=*) + PYTHON_BIN="${1#*=}" + shift + ;; --help|-h) HELP=1 shift From 012ea9977b8b96f87bfa7d3dcd7e1997b926ed0b Mon Sep 17 00:00:00 2001 From: Jinkun Liu Date: Mon, 27 Apr 2026 19:07:45 +0800 Subject: [PATCH 11/35] [tools] install.sh supprot plan --- tools/install.sh | 156 ++++++++++++++++++++++++++++------------------- 1 file changed, 93 insertions(+), 63 deletions(-) diff --git a/tools/install.sh b/tools/install.sh index db5e7ea42..e70beec80 100755 --- a/tools/install.sh +++ b/tools/install.sh @@ -306,7 +306,7 @@ die() { exit 1 } -INSTALL_STAGE_TOTAL=3 +INSTALL_STAGE_TOTAL=5 INSTALL_STAGE_CURRENT=0 ui_section() { @@ -377,8 +377,8 @@ FLINK_BASE_URL="${FLINK_BASE_URL:-https://dlcdn.apache.org/flink}" FLINK_SUPPORTED_VERSIONS=("2.2.0" "2.1.1" "2.0.1" "1.20.3") FLINK_RECOMMENDED_VERSION="2.2.0" -INSTALL_FLINK="${INSTALL_FLINK:-ask}" -ENABLE_PYFLINK="${ENABLE_PYFLINK:-ask}" +INSTALL_FLINK="${INSTALL_FLINK:-Ask}" +ENABLE_PYFLINK="${ENABLE_PYFLINK:-Ask}" INSTALL_DIR_EXPLICIT=0 if [[ -n "${INSTALL_DIR:-}" ]]; then INSTALL_DIR_EXPLICIT=1 @@ -489,27 +489,23 @@ prompt_flink_version_interactive() { return 0 } -install_flink_if_needed() { - local skip_install=0 +plan_flink() { case "$INSTALL_FLINK" in - yes) - ;; - no) - ui_info "Skipping Flink download/install (INSTALL_FLINK=${INSTALL_FLINK})." - skip_install=1 + Yes|No) ;; - ask) - if ! choose_install_method_interactive "Do you want this script to download and install Apache Flink?"; then - ui_info "Skipping Flink download/install by user choice." - skip_install=1 + Ask) + if choose_install_method_interactive "Do you want this script to download and install Apache Flink?"; then + INSTALL_FLINK=Yes + else + INSTALL_FLINK=No fi ;; *) - die "Unsupported INSTALL_FLINK value: ${INSTALL_FLINK}. Use: ask|yes|no" + die "Unsupported INSTALL_FLINK value: ${INSTALL_FLINK}. Use: Ask|Yes|No" ;; esac - if [[ "$skip_install" -eq 1 ]]; then + if [[ "$INSTALL_FLINK" == "No" ]]; then if [[ -z "${FLINK_HOME:-}" || ! -d "${FLINK_HOME}" || ! -d "${FLINK_HOME}/lib" ]]; then if is_promptable; then FLINK_HOME="$(prompt_flink_home_interactive)" @@ -519,7 +515,6 @@ install_flink_if_needed() { [[ -n "${FLINK_HOME:-}" ]] || die "FLINK_HOME is not set." [[ -d "$FLINK_HOME" ]] || die "FLINK_HOME does not exist: $FLINK_HOME" [[ -d "$FLINK_HOME/lib" ]] || die "Invalid FLINK_HOME (missing lib directory): $FLINK_HOME" - ui_success "FLINK_HOME resolved: $FLINK_HOME" FLINK_MAJOR_MINOR="${FLINK_VERSION%.*}" return fi @@ -528,12 +523,6 @@ install_flink_if_needed() { prompt_flink_version_interactive || true fi - local ARCHIVE_NAME="flink-${FLINK_VERSION}-bin-scala_${FLINK_SCALA_VERSION}.tgz" - local ARCHIVE_URL="${FLINK_BASE_URL}/flink-${FLINK_VERSION}/${ARCHIVE_NAME}" - - detect_downloader - require_cmd tar - if [[ "$INSTALL_DIR_EXPLICIT" -eq 0 ]] && is_promptable; then INSTALL_DIR="$(prompt_path_choice_interactive \ "Choose Flink install directory" \ @@ -541,6 +530,70 @@ install_flink_if_needed() { "/path/to/flink-install-dir")" fi + FLINK_HOME="${INSTALL_DIR}/flink-${FLINK_VERSION}" + FLINK_MAJOR_MINOR="${FLINK_VERSION%.*}" +} + +plan_pyflink() { + case "$ENABLE_PYFLINK" in + Yes|No) + ;; + Ask) + if choose_install_method_interactive "Create a Python venv with PyFlink and flink-agents? (Only needed for Python API users; Java users can select No)"; then + ENABLE_PYFLINK=Yes + else + ENABLE_PYFLINK=No + fi + ;; + *) + die "Unsupported ENABLE_PYFLINK value: ${ENABLE_PYFLINK}. Use: Ask|Yes|No" + ;; + esac + + if [[ "$ENABLE_PYFLINK" == "No" ]]; then + return + fi + + PYFLINK_ACTUALLY_ENABLED=1 + resolve_python + + if [[ "$VENV_DIR_EXPLICIT" -eq 0 ]] && is_promptable; then + VENV_DIR="$(prompt_path_choice_interactive \ + "Choose Python venv directory" \ + "$VENV_DIR" \ + "/path/to/venv")" + fi + + case "$VENV_DIR" in + /*) ;; + *) VENV_DIR="$PWD/$VENV_DIR" ;; + esac +} + +confirm_install_plan() { + if [[ "$NO_PROMPT" == "1" ]] || ! is_promptable; then + return 0 + fi + if choose_install_method_interactive "Proceed with installation?"; then + return 0 + fi + ui_info "Installation cancelled by user." + exit 0 +} + +install_flink_if_needed() { + if [[ "$INSTALL_FLINK" == "No" ]]; then + ui_info "Skipping Flink download/install (using existing FLINK_HOME)." + ui_success "FLINK_HOME resolved: $FLINK_HOME" + return + fi + + local ARCHIVE_NAME="flink-${FLINK_VERSION}-bin-scala_${FLINK_SCALA_VERSION}.tgz" + local ARCHIVE_URL="${FLINK_BASE_URL}/flink-${FLINK_VERSION}/${ARCHIVE_NAME}" + + detect_downloader + require_cmd tar + if ! mkdir -p "$INSTALL_DIR"; then die "Failed to create INSTALL_DIR=$INSTALL_DIR. Please run with proper permissions or set INSTALL_DIR to a writable path." fi @@ -572,8 +625,7 @@ install_flink_if_needed() { ui_info "Reusing existing Flink home: ${INSTALL_DIR}/flink-${FLINK_VERSION}" fi - export FLINK_HOME="${INSTALL_DIR}/flink-${FLINK_VERSION}" - FLINK_MAJOR_MINOR="${FLINK_VERSION%.*}" + export FLINK_HOME ui_success "FLINK_HOME resolved: $FLINK_HOME" } @@ -673,15 +725,6 @@ create_venv() { } setup_python_env() { - resolve_python - - if [[ "$VENV_DIR_EXPLICIT" -eq 0 ]] && is_promptable; then - VENV_DIR="$(prompt_path_choice_interactive \ - "Choose Python venv directory" \ - "$VENV_DIR" \ - "/path/to/venv")" - fi - if [[ ! -d "$VENV_DIR" ]]; then ui_info "Creating virtual environment: $VENV_DIR" create_venv @@ -735,26 +778,6 @@ PY [[ "$copied" -eq 1 ]] || die "No flink-agents-dist jar found in: $version_lib_dir" } -should_enable_pyflink() { - case "$ENABLE_PYFLINK" in - yes) - return 0 - ;; - no) - return 1 - ;; - ask) - if choose_install_method_interactive "Create a Python venv with PyFlink and flink-agents? (Only needed for Python API users; Java users can select No)"; then - return 0 - fi - return 1 - ;; - *) - die "Unsupported ENABLE_PYFLINK value: ${ENABLE_PYFLINK}. Use: ask|yes|no" - ;; - esac -} - print_usage() { cat < Date: Tue, 28 Apr 2026 00:26:47 +0800 Subject: [PATCH 12/35] [tools] install.sh: refactor style to align with install_example.sh --- tools/install.sh | 166 +++++++++++++++++++++-------------------------- 1 file changed, 73 insertions(+), 93 deletions(-) diff --git a/tools/install.sh b/tools/install.sh index e70beec80..1c133fa83 100755 --- a/tools/install.sh +++ b/tools/install.sh @@ -175,7 +175,7 @@ bootstrap_gum_temp() { gum_tmpdir="$(mktemp -d)" TMPFILES+=("$gum_tmpdir") - echo -e "${INFO}· Installing gum v${GUM_VERSION}, please wait...${NC}" + ui_info "Installing gum v${GUM_VERSION}, please wait..." if ! download_file "${base}/${asset}" "$gum_tmpdir/$asset"; then GUM_REASON="download failed" @@ -233,10 +233,9 @@ print_gum_status() { print_installer_banner() { if [[ -n "$GUM" ]]; then - local title card + local title title="$("$GUM" style --foreground "#ff4d4d" --bold "Apache Flink Agents Installer")" - card="$(printf '%s\n' "$title")" - "$GUM" style --border rounded --border-foreground "#ff4d4d" --padding "1 2" "$card" + "$GUM" style --border rounded --border-foreground "#ff4d4d" --padding "1 2" "$title" echo "" return fi @@ -347,28 +346,19 @@ ui_celebrate() { fi } -is_shell_function() { - local name="${1:-}" - [[ -n "$name" ]] && declare -F "$name" >/dev/null 2>&1 -} - -is_gum_raw_mode_failure() { - local err_log="$1" - [[ -s "$err_log" ]] || return 1 - grep -Eiq '(setrawmode|inappropriate ioctl for device)' "$err_log" -} - -configure_verbose() { - if [[ "$VERBOSE" != "1" ]]; then - return 0 +mark_explicit() { + local var="$1" + if [[ -n "${!var:-}" ]]; then + printf -v "${var}_EXPLICIT" '%s' '1' + else + printf -v "${var}_EXPLICIT" '%s' '0' fi - set -x } -FLINK_VERSION_EXPLICIT=0 -if [[ -n "${FLINK_VERSION:-}" ]]; then - FLINK_VERSION_EXPLICIT=1 -fi +mark_explicit FLINK_VERSION +mark_explicit INSTALL_DIR +mark_explicit VENV_DIR + FLINK_VERSION="${FLINK_VERSION:-2.2.0}" FLINK_AGENTS_VERSION="${FLINK_AGENTS_VERSION:-0.2.1}" FLINK_SCALA_VERSION="${FLINK_SCALA_VERSION:-2.12}" @@ -379,15 +369,7 @@ FLINK_RECOMMENDED_VERSION="2.2.0" INSTALL_FLINK="${INSTALL_FLINK:-Ask}" ENABLE_PYFLINK="${ENABLE_PYFLINK:-Ask}" -INSTALL_DIR_EXPLICIT=0 -if [[ -n "${INSTALL_DIR:-}" ]]; then - INSTALL_DIR_EXPLICIT=1 -fi INSTALL_DIR="${INSTALL_DIR:-$HOME/.local/flink}" -VENV_DIR_EXPLICIT=0 -if [[ -n "${VENV_DIR:-}" ]]; then - VENV_DIR_EXPLICIT=1 -fi VENV_DIR="${VENV_DIR:-.flink-agents-env}" PYTHON_BIN="${PYTHON_BIN:-}" NO_PROMPT="${NO_PROMPT:-0}" @@ -397,6 +379,46 @@ VERIFY_INSTALL="${FLINK_AGENTS_VERIFY_INSTALL:-0}" HELP=0 PYFLINK_ACTUALLY_ENABLED=0 +print_usage() { + cat </install.sh | bash -s -- [options] + +Options: + --non-interactive Non-interactive mode (accept all defaults) + --install-flink Download and install Apache Flink + --enable-pyflink Enable PyFlink and install Python packages + --verbose Print debug output (set -x) + --dry-run Print install plan without making changes + --verify Run post-install verification checks + --python Path to a Python3 interpreter (overrides PATH lookup) + --help, -h Show this help + +Environment variables: + FLINK_VERSION Flink version + FLINK_AGENTS_VERSION Flink Agents version to install + FLINK_SCALA_VERSION Scala version suffix (default: 2.12) + FLINK_BASE_URL Mirror base URL (default: https://dlcdn.apache.org/flink) + INSTALL_FLINK Ask|Yes|No (default: Ask) + ENABLE_PYFLINK Ask|Yes|No (default: Ask) + INSTALL_DIR Flink install directory (default: \$HOME/.local/flink) + VENV_DIR Python venv directory (default: .flink-agents-env) + PYTHON_BIN Path to Python3 interpreter (default: auto-detect python3 on PATH) + NO_PROMPT 1 to disable all prompts + FLINK_AGENTS_VERBOSE 1 to enable verbose output + FLINK_AGENTS_DRY_RUN 1 to enable dry-run mode + FLINK_AGENTS_VERIFY_INSTALL 1 to enable post-install verification + +Examples: + bash install.sh --install-flink --enable-pyflink --non-interactive + FLINK_VERSION=2.2.0 bash install.sh --verbose + bash install.sh --dry-run +EOF +} + require_cmd() { command -v "$1" >/dev/null 2>&1 || die "Missing required command: $1" } @@ -508,7 +530,7 @@ plan_flink() { if [[ "$INSTALL_FLINK" == "No" ]]; then if [[ -z "${FLINK_HOME:-}" || ! -d "${FLINK_HOME}" || ! -d "${FLINK_HOME}/lib" ]]; then if is_promptable; then - FLINK_HOME="$(prompt_flink_home_interactive)" + FLINK_HOME="$(prompt_path_input "Enter the path to your existing FLINK_HOME (version >= 1.20)" "/path/to/flink-${FLINK_VERSION}")" export FLINK_HOME fi fi @@ -629,15 +651,17 @@ install_flink_if_needed() { ui_success "FLINK_HOME resolved: $FLINK_HOME" } -prompt_flink_home_interactive() { +prompt_path_input() { + local header="$1" + local placeholder="$2" local input="" if [[ -n "$GUM" ]] && gum_is_tty; then input="$("$GUM" input \ - --header "Enter the path to your existing FLINK_HOME which version >= 1.20" \ - --placeholder "/path/to/flink-${FLINK_VERSION}" \ + --header "$header" \ + --placeholder "$placeholder" \ --width 70 < /dev/tty || true)" else - printf 'Enter the path to your existing FLINK_HOME which version >= 1.20: ' > /dev/tty + printf '%s: ' "$header" > /dev/tty read -r input < /dev/tty || true fi input="${input/#\~/$HOME}" @@ -778,46 +802,6 @@ PY [[ "$copied" -eq 1 ]] || die "No flink-agents-dist jar found in: $version_lib_dir" } -print_usage() { - cat </install.sh | bash -s -- [options] - -Options: - --non-interactive Non-interactive mode (accept all defaults) - --install-flink Download and install Apache Flink - --enable-pyflink Enable PyFlink and install Python packages - --verbose Print debug output (set -x) - --dry-run Print install plan without making changes - --verify Run post-install verification checks - --python Path to a Python3 interpreter (overrides PATH lookup) - --help, -h Show this help - -Environment variables: - FLINK_VERSION Flink version - FLINK_AGENTS_VERSION Flink Agents version to install - FLINK_SCALA_VERSION Scala version suffix (default: 2.12) - FLINK_BASE_URL Mirror base URL (default: https://dlcdn.apache.org/flink) - INSTALL_FLINK Ask|Yes|No (default: Ask) - ENABLE_PYFLINK Ask|Yes|No (default: Ask) - INSTALL_DIR Flink install directory (default: \$HOME/.local/flink) - VENV_DIR Python venv directory (default: .flink-agents-env) - PYTHON_BIN Path to Python3 interpreter (default: auto-detect python3 on PATH) - NO_PROMPT 1 to disable all prompts - FLINK_AGENTS_VERBOSE 1 to enable verbose output - FLINK_AGENTS_DRY_RUN 1 to enable dry-run mode - FLINK_AGENTS_VERIFY_INSTALL 1 to enable post-install verification - -Examples: - bash install.sh --install-flink --enable-pyflink --non-interactive - FLINK_VERSION=2.2.0 bash install.sh --verbose - bash install.sh --dry-run -EOF -} - check_java() { if ! command -v java &>/dev/null; then ui_warn "Java not found on PATH" @@ -834,7 +818,7 @@ check_java() { local java_version_output java_version_output="$(java -version 2>&1 | head -n1)" local java_major="" - java_major="$(echo "$java_version_output" | sed -n 's/.*version "\([0-9]*\).*/\1/p')" + java_major="$(echo "$java_version_output" | sed -E -n 's/.*version "?([0-9]+).*/\1/p')" if [[ -z "$java_major" ]]; then ui_warn "Could not parse Java version from: $java_version_output" @@ -874,21 +858,6 @@ validate_python_bin() { return 0 } -prompt_python_bin_interactive() { - local input="" - if [[ -n "$GUM" ]] && gum_is_tty; then - input="$("$GUM" input \ - --header "Enter path to a Python interpreter" \ - --placeholder "/path/to/python3" \ - --width 70 < /dev/tty || true)" - else - printf 'Enter path to a Python interpreter: ' > /dev/tty - read -r input < /dev/tty || true - fi - input="${input/#\~/$HOME}" - printf '%s' "$input" -} - resolve_python() { if [[ -n "$PYTHON_BIN" ]]; then if validate_python_bin "$PYTHON_BIN"; then @@ -917,7 +886,7 @@ resolve_python() { fi local input - input="$(prompt_python_bin_interactive)" + input="$(prompt_path_input "Enter path to a Python interpreter" "/path/to/python3")" if [[ -z "$input" ]]; then die "No Python interpreter provided." fi @@ -1074,6 +1043,17 @@ parse_args() { done } +configure_verbose() { + if [[ "$VERBOSE" != "1" ]]; then + return 0 + fi + if [[ "$NPM_LOGLEVEL" == "error" ]]; then + NPM_LOGLEVEL="notice" + fi + NPM_SILENT_FLAG="" + set -x +} + main() { if [[ "$HELP" == "1" ]]; then print_usage From 852c6032e3d9453239c184e127b78b271e29e6d2 Mon Sep 17 00:00:00 2001 From: Jinkun Liu Date: Wed, 29 Apr 2026 22:18:47 +0800 Subject: [PATCH 13/35] [tools]Fix download time limit --- tools/install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/install.sh b/tools/install.sh index 1c133fa83..d8db4afb6 100755 --- a/tools/install.sh +++ b/tools/install.sh @@ -65,10 +65,10 @@ download_file() { detect_downloader fi if [[ "$DOWNLOADER" == "curl" ]]; then - curl -fsSL --proto '=https' --tlsv1.2 --connect-timeout 5 --max-time 30 --retry 2 --retry-delay 1 --retry-connrefused -o "$output" "$url" + curl -fsSL --proto '=https' --tlsv1.2 --retry 3 --max-time 600 --retry-delay 1 -- --retry-connrefused -o "$output" "$url" return fi - wget -q --https-only --secure-protocol=TLSv1_2 --tries=2 --timeout=30 -O "$output" "$url" + wget -q --https-only --secure-protocol=TLSv1_2 --tries=3 --timeout=600 -O "$output" "$url" } GUM_VERSION="${FLINK_AGENTS_GUM_VERSION:-0.17.0}" From 5a1d3178a6e45a98bf766bdac52d4a865c16805c Mon Sep 17 00:00:00 2001 From: Jinkun Liu Date: Wed, 13 May 2026 21:44:42 +0800 Subject: [PATCH 14/35] [tools] install.sh: update section titles for clarity --- tools/install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/install.sh b/tools/install.sh index d8db4afb6..b99540809 100755 --- a/tools/install.sh +++ b/tools/install.sh @@ -65,7 +65,7 @@ download_file() { detect_downloader fi if [[ "$DOWNLOADER" == "curl" ]]; then - curl -fsSL --proto '=https' --tlsv1.2 --retry 3 --max-time 600 --retry-delay 1 -- --retry-connrefused -o "$output" "$url" + curl -fsSL --proto '=https' --tlsv1.2 --retry 3 --max-time 600 --retry-delay 1 --retry-connrefused -o "$output" "$url" return fi wget -q --https-only --secure-protocol=TLSv1_2 --tries=3 --timeout=600 -O "$output" "$url" @@ -899,7 +899,7 @@ resolve_python() { } show_install_plan() { - ui_section "Install plan" + ui_section "Installation plan" ui_kv "OS" "$OS" ui_kv "Flink version" "$FLINK_VERSION" ui_kv "Flink Agents version" "$FLINK_AGENTS_VERSION" From 68d6759abfbfe5117e5ab20ef4c20d7212c0291d Mon Sep 17 00:00:00 2001 From: Jinkun Liu Date: Wed, 13 May 2026 22:15:46 +0800 Subject: [PATCH 15/35] [tools] install.sh: enhance curl and wget commands with progress indicators --- tools/install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/install.sh b/tools/install.sh index b99540809..cc73b2db2 100755 --- a/tools/install.sh +++ b/tools/install.sh @@ -65,10 +65,10 @@ download_file() { detect_downloader fi if [[ "$DOWNLOADER" == "curl" ]]; then - curl -fsSL --proto '=https' --tlsv1.2 --retry 3 --max-time 600 --retry-delay 1 --retry-connrefused -o "$output" "$url" + curl -fL --progress-bar --proto '=https' --tlsv1.2 --retry 3 --max-time 600 --retry-delay 1 --retry-connrefused -o "$output" "$url" return fi - wget -q --https-only --secure-protocol=TLSv1_2 --tries=3 --timeout=600 -O "$output" "$url" + wget -q --show-progress --https-only --secure-protocol=TLSv1_2 --tries=3 --timeout=600 -O "$output" "$url" } GUM_VERSION="${FLINK_AGENTS_GUM_VERSION:-0.17.0}" From 8302853e6495c92c84a8b7087b1a5fa32470bdc5 Mon Sep 17 00:00:00 2001 From: Jinkun Liu Date: Mon, 18 May 2026 00:34:38 +0800 Subject: [PATCH 16/35] [tools] install.sh: exit cleanly on Ctrl+C during interactive prompts --- tools/install.sh | 36 +++++++++++++++++++++++++----------- 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/tools/install.sh b/tools/install.sh index cc73b2db2..394a16bdc 100755 --- a/tools/install.sh +++ b/tools/install.sh @@ -305,6 +305,11 @@ die() { exit 1 } +die_cancelled() { + ui_info "Cancelled by user" + exit 130 +} + INSTALL_STAGE_TOTAL=5 INSTALL_STAGE_CURRENT=0 @@ -447,18 +452,19 @@ choose_install_method_interactive() { fi if [[ -n "$GUM" ]] && gum_is_tty; then - local selection + local selection _rc=0 selection="$("$GUM" choose \ --header "$prompt" \ --cursor-prefix "❯ " \ - "Yes" "No" < /dev/tty || true)" + "Yes" "No" < /dev/tty)" || _rc=$? + (( _rc == 0 )) || die_cancelled [[ "$selection" == "Yes" ]] return fi local answer="" printf '%s [y/n]: ' "$prompt" > /dev/tty - read -r answer < /dev/tty || true + read -r answer < /dev/tty || die_cancelled [[ "$answer" =~ ^[Yy]$ ]] } @@ -480,10 +486,12 @@ prompt_flink_version_interactive() { local selection="" if [[ -n "$GUM" ]] && gum_is_tty; then + local _rc=0 selection="$("$GUM" choose \ --header "Select Flink version" \ --cursor-prefix "❯ " \ - "${labels[@]}" < /dev/tty || true)" + "${labels[@]}" < /dev/tty)" || _rc=$? + (( _rc == 0 )) || die_cancelled selection="${selection%% *}" else printf 'Select Flink version:\n' > /dev/tty @@ -497,7 +505,7 @@ prompt_flink_version_interactive() { local answer="" printf 'Enter choice [1-%d, default %s]: ' \ "${#FLINK_SUPPORTED_VERSIONS[@]}" "$FLINK_RECOMMENDED_VERSION" > /dev/tty - read -r answer < /dev/tty || true + read -r answer < /dev/tty || die_cancelled if [[ "$answer" =~ ^[0-9]+$ ]] \ && (( answer >= 1 && answer <= ${#FLINK_SUPPORTED_VERSIONS[@]} )); then selection="${FLINK_SUPPORTED_VERSIONS[$((answer-1))]}" @@ -656,13 +664,15 @@ prompt_path_input() { local placeholder="$2" local input="" if [[ -n "$GUM" ]] && gum_is_tty; then + local _rc=0 input="$("$GUM" input \ --header "$header" \ --placeholder "$placeholder" \ - --width 70 < /dev/tty || true)" + --width 70 < /dev/tty)" || _rc=$? + (( _rc == 0 )) || die_cancelled else printf '%s: ' "$header" > /dev/tty - read -r input < /dev/tty || true + read -r input < /dev/tty || die_cancelled fi input="${input/#\~/$HOME}" printf '%s' "$input" @@ -683,17 +693,19 @@ prompt_path_choice_interactive() { local selection="" if [[ -n "$GUM" ]] && gum_is_tty; then + local _rc=0 selection="$("$GUM" choose \ --header "$header" \ --cursor-prefix "❯ " \ - "$default_label" "$custom_label" < /dev/tty || true)" + "$default_label" "$custom_label" < /dev/tty)" || _rc=$? + (( _rc == 0 )) || die_cancelled else printf '%s\n' "$header" > /dev/tty printf ' 1) %s\n' "$default_label" > /dev/tty printf ' 2) %s\n' "$custom_label" > /dev/tty local answer="" printf 'Enter choice [1-2, default 1]: ' > /dev/tty - read -r answer < /dev/tty || true + read -r answer < /dev/tty || die_cancelled case "$answer" in 2) selection="$custom_label" ;; *) selection="$default_label" ;; @@ -707,13 +719,15 @@ prompt_path_choice_interactive() { local input="" if [[ -n "$GUM" ]] && gum_is_tty; then + local _rc=0 input="$("$GUM" input \ --header "$header" \ --placeholder "$placeholder" \ - --width 70 < /dev/tty || true)" + --width 70 < /dev/tty)" || _rc=$? + (( _rc == 0 )) || die_cancelled else printf 'Enter custom path: ' > /dev/tty - read -r input < /dev/tty || true + read -r input < /dev/tty || die_cancelled fi input="${input/#\~/$HOME}" From 25fb0a3f496ea6735de759783cc6c85dec07402a Mon Sep 17 00:00:00 2001 From: Jinkun Liu Date: Mon, 18 May 2026 10:09:31 +0800 Subject: [PATCH 17/35] [tools] install.sh: re-prompt for FLINK_HOME on invalid input --- tools/install.sh | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/tools/install.sh b/tools/install.sh index 394a16bdc..d05ed5683 100755 --- a/tools/install.sh +++ b/tools/install.sh @@ -536,11 +536,18 @@ plan_flink() { esac if [[ "$INSTALL_FLINK" == "No" ]]; then - if [[ -z "${FLINK_HOME:-}" || ! -d "${FLINK_HOME}" || ! -d "${FLINK_HOME}/lib" ]]; then - if is_promptable; then + if is_promptable; then + while [[ -z "${FLINK_HOME:-}" || ! -d "${FLINK_HOME}" || ! -d "${FLINK_HOME}/lib" ]]; do + if [[ -n "${FLINK_HOME:-}" ]]; then + if [[ ! -d "${FLINK_HOME}" ]]; then + ui_warn "Path does not exist: ${FLINK_HOME}" + else + ui_warn "Not a valid Flink home (missing 'lib' directory): ${FLINK_HOME}" + fi + fi FLINK_HOME="$(prompt_path_input "Enter the path to your existing FLINK_HOME (version >= 1.20)" "/path/to/flink-${FLINK_VERSION}")" - export FLINK_HOME - fi + done + export FLINK_HOME fi [[ -n "${FLINK_HOME:-}" ]] || die "FLINK_HOME is not set." [[ -d "$FLINK_HOME" ]] || die "FLINK_HOME does not exist: $FLINK_HOME" From e8f9c0ea2c00a7adecddd047499e8daf8c3174d4 Mon Sep 17 00:00:00 2001 From: WenjinXie Date: Mon, 18 May 2026 15:22:21 +0800 Subject: [PATCH 18/35] [ci] Add comprehensive tests for install script. Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/workflows/ci.yml | 15 ++ tools/.rat-excludes | 8 +- tools/test/.gitignore | 1 + tools/test/helpers/load.bash | 44 ++++++ tools/test/helpers/shim.bash | 88 ++++++++++++ tools/test/integration/.gitkeep | 0 .../test/integration/bootstrap_gum_temp.bats | 121 ++++++++++++++++ tools/test/integration/download_file.bats | 88 ++++++++++++ tools/test/integration/dry_run.bats | 41 ++++++ tools/test/integration/help.bats | 16 +++ .../integration/install_flink_if_needed.bats | 132 ++++++++++++++++++ tools/test/run.sh | 53 +++++++ tools/test/unit/is_valid_tgz.bats | 44 ++++++ tools/test/unit/mark_explicit.bats | 31 ++++ tools/test/unit/parse_args.bats | 82 +++++++++++ tools/test/unit/platform_detect.bats | 98 +++++++++++++ tools/test/unit/shim_self_test.bats | 50 +++++++ tools/test/unit/smoke.bats | 18 +++ tools/test/unit/ui_helpers.bats | 54 +++++++ tools/test/unit/validate_python_bin.bats | 73 ++++++++++ 20 files changed, 1056 insertions(+), 1 deletion(-) create mode 100644 tools/test/.gitignore create mode 100644 tools/test/helpers/load.bash create mode 100644 tools/test/helpers/shim.bash create mode 100644 tools/test/integration/.gitkeep create mode 100644 tools/test/integration/bootstrap_gum_temp.bats create mode 100644 tools/test/integration/download_file.bats create mode 100644 tools/test/integration/dry_run.bats create mode 100644 tools/test/integration/help.bats create mode 100644 tools/test/integration/install_flink_if_needed.bats create mode 100755 tools/test/run.sh create mode 100644 tools/test/unit/is_valid_tgz.bats create mode 100644 tools/test/unit/mark_explicit.bats create mode 100644 tools/test/unit/parse_args.bats create mode 100644 tools/test/unit/platform_detect.bats create mode 100644 tools/test/unit/shim_self_test.bats create mode 100644 tools/test/unit/smoke.bats create mode 100644 tools/test/unit/ui_helpers.bats create mode 100644 tools/test/unit/validate_python_bin.bats diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 191bcc3f0..5ba8f6d8d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,6 +48,21 @@ jobs: - name: Check code style run: ./tools/lint.sh -c + install_sh_tests: + name: install.sh tests (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] + steps: + - uses: actions/checkout@v4 + - name: Install bash 4+ (macOS) + if: runner.os == 'macOS' + run: brew install bash + - name: Run install.sh tests + run: bash tools/test/run.sh + build_backend_tests: name: ut-build-backend runs-on: ubuntu-latest diff --git a/tools/.rat-excludes b/tools/.rat-excludes index bea5f93f8..11e47ca79 100644 --- a/tools/.rat-excludes +++ b/tools/.rat-excludes @@ -4,6 +4,10 @@ docs .git .gitignore .gitmodules +\.bats-cache +.*\.bats$ +.*\.bash$ +\.gitkeep .*\.lock$ .venv/* .idea/* @@ -16,4 +20,6 @@ PULL_REQUEST_TEMPLATE.md .pytest_cache/* .ruff_cache/* .*\.egg-info/* -licenses/* \ No newline at end of file +licenses/* +skills/* +.*\.yaml$ \ No newline at end of file diff --git a/tools/test/.gitignore b/tools/test/.gitignore new file mode 100644 index 000000000..74b450371 --- /dev/null +++ b/tools/test/.gitignore @@ -0,0 +1 @@ +.bats-cache/ diff --git a/tools/test/helpers/load.bash b/tools/test/helpers/load.bash new file mode 100644 index 000000000..b46e2b707 --- /dev/null +++ b/tools/test/helpers/load.bash @@ -0,0 +1,44 @@ +# Helpers used by every bats file. Loaded via `load 'helpers/load'`. + +# Sources install.sh with the no-run hook so main() is skipped. +load_install_sh() { + export FLINK_AGENTS_INSTALL_SH_NO_RUN=1 + # shellcheck disable=SC1090 + source "${BATS_TEST_DIRNAME}/../../install.sh" + # Note: install.sh sets `set -euo pipefail`, which matches the + # strict mode bats itself enables. Leave it on so bats's ERR trap + # can detect failed assertions. Use `run ` (which captures + # $status in a subshell) for tests that exercise failure paths. +} + +# Resets every module-level variable install.sh defines, so tests don't +# leak state into one another. +reset_install_sh_state() { + TMPFILES=() + INSTALL_STAGE_CURRENT=0 + GUM="" + GUM_STATUS="skipped" + GUM_REASON="" + DOWNLOADER="" + HELP=0 + DRY_RUN=0 + NO_PROMPT=0 + VERBOSE=0 + VERIFY_INSTALL=0 + INSTALL_FLINK="Ask" + ENABLE_PYFLINK="Ask" + PYTHON_BIN="" + PYFLINK_ACTUALLY_ENABLED=0 + FLINK_VERSION="2.2.0" + FLINK_AGENTS_VERSION="0.2.1" + FLINK_SCALA_VERSION="2.12" + FLINK_BASE_URL="https://dlcdn.apache.org/flink" + FLINK_SUPPORTED_VERSIONS=("2.2.0" "2.1.1" "2.0.1" "1.20.3") + FLINK_RECOMMENDED_VERSION="2.2.0" + INSTALL_DIR="$HOME/.local/flink" + VENV_DIR=".flink-agents-env" + GUM_VERSION="0.17.0" + FLINK_VERSION_EXPLICIT=0 + INSTALL_DIR_EXPLICIT=0 + VENV_DIR_EXPLICIT=0 +} diff --git a/tools/test/helpers/shim.bash b/tools/test/helpers/shim.bash new file mode 100644 index 000000000..1d60a5b10 --- /dev/null +++ b/tools/test/helpers/shim.bash @@ -0,0 +1,88 @@ +# PATH-shim helpers for integration tests. Loaded via `load 'helpers/shim'`. +# +# After `shim_setup` runs, $BATS_TEST_TMPDIR/bin is prepended to PATH and +# every shimmed binary appends one tab-separated line per invocation to +# $BATS_TEST_TMPDIR/calls/.log. +# +# `shim_bin_missing` additionally registers a name in SHIM_MISSING_NAMES; +# the `command` function override below intercepts `command -v ` for +# those names and returns 1, so install.sh's `command -v curl &>/dev/null` +# checks see them as unavailable. + +shim_setup() { + SHIM_DIR="$BATS_TEST_TMPDIR/bin" + SHIM_CALLS="$BATS_TEST_TMPDIR/calls" + SHIM_MISSING_NAMES=() + mkdir -p "$SHIM_DIR" "$SHIM_CALLS" + export PATH="$SHIM_DIR:$PATH" +} + +# Override `command` as a shell function. When SHIM_MISSING_NAMES is empty +# (i.e. shim_setup hasn't been called or nothing has been marked missing), +# this is fully transparent — every call falls through to `builtin command`. +command() { + if [[ "$1" == "-v" ]]; then + local q="$2" + local m + for m in "${SHIM_MISSING_NAMES[@]:-}"; do + if [[ "$q" == "$m" ]]; then + return 1 + fi + done + fi + builtin command "$@" +} + +# Replace `name` with a stub that records argv and exits with `exit_code` (default 0). +shim_bin() { + local name="$1" exit_code="${2:-0}" + cat >"$SHIM_DIR/$name" <> "$SHIM_CALLS/$name.log" +exit $exit_code +EOF + chmod +x "$SHIM_DIR/$name" +} + +# Replace `name` with a stub that records argv then runs an arbitrary shell body. +# Note: `body` must not contain a line that is exactly `EOF`, or it will +# terminate the heredoc prematurely. +shim_bin_script() { + local name="$1" body="$2" + cat >"$SHIM_DIR/$name" <> "$SHIM_CALLS/$name.log" +$body +EOF + chmod +x "$SHIM_DIR/$name" +} + +# Make `name` resolve to "missing" for both `command -v` checks and actual +# invocation: register it in SHIM_MISSING_NAMES (the `command` override +# returns 1 for these) and drop an exit-127 stub on PATH (so direct +# invocation doesn't accidentally hit the real system binary). +shim_bin_missing() { + local name="$1" + SHIM_MISSING_NAMES+=("$name") + cat >"$SHIM_DIR/$name" <<'EOF' +#!/usr/bin/env bash +exit 127 +EOF + chmod +x "$SHIM_DIR/$name" +} + +# Print all recorded calls for `name`, one per line, tab-separated argv. +shim_calls() { + local name="$1" + cat "$SHIM_CALLS/$name.log" 2>/dev/null || true +} + +# Print the number of times `name` was invoked. +shim_call_count() { + local name="$1" + if [[ ! -f "$SHIM_CALLS/$name.log" ]]; then + echo 0 + return + fi + wc -l < "$SHIM_CALLS/$name.log" | tr -d ' ' +} diff --git a/tools/test/integration/.gitkeep b/tools/test/integration/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/tools/test/integration/bootstrap_gum_temp.bats b/tools/test/integration/bootstrap_gum_temp.bats new file mode 100644 index 000000000..43fdbb2ae --- /dev/null +++ b/tools/test/integration/bootstrap_gum_temp.bats @@ -0,0 +1,121 @@ +#!/usr/bin/env bats + +setup() { + load '../helpers/load' + load '../helpers/shim' + load_install_sh + reset_install_sh_state + shim_setup + # Force interactive-shell + tty path so bootstrap doesn't auto-skip. + NO_PROMPT=0 + # gum_is_tty falls back to checking /dev/tty readability; bats provides one. + # If the test env happens not to, override via TERM/NO_COLOR being unset. + unset NO_COLOR + TERM=xterm + # Make sure `gum` isn't already on PATH so we exercise the install branch. + shim_bin_missing gum + # Provide tar so the early "tar not found" branch is skipped. + shim_bin tar + # Real uname is fine here — we just need a supported os/arch. + + # Override is_non_interactive_shell to report "interactive" by default so + # tests that exercise the download path are not short-circuited by stdin/ + # stdout not being ttys in the bats process. + is_non_interactive_shell() { return 1; } + + # Override gum_is_tty to report "has tty" by default so the second guard + # in bootstrap_gum_temp does not short-circuit the download path either. + gum_is_tty() { return 0; } +} + +@test "bootstrap_gum_temp: non-interactive shell auto-skips with no downloads" { + # Restore the real check and set NO_PROMPT to force non-interactive. + is_non_interactive_shell() { + [[ "${NO_PROMPT:-0}" == "1" ]] + } + NO_PROMPT=1 + run bootstrap_gum_temp + [ "$status" -ne 0 ] + [ "$GUM" = "" ] + [ "$GUM_STATUS" = "skipped" ] + [ "$(shim_call_count curl)" = "0" ] + [ "$(shim_call_count wget)" = "0" ] +} + +@test "bootstrap_gum_temp: download failure sets reason='download failed'" { + DOWNLOADER=curl + shim_bin curl 22 + bootstrap_gum_temp || true + [ "$GUM" = "" ] + [ "$GUM_STATUS" = "skipped" ] + [ "$GUM_REASON" = "download failed" ] +} + +@test "bootstrap_gum_temp: checksum failure sets reason='checksum unavailable or failed'" { + DOWNLOADER=curl + # curl always succeeds (both asset and checksums.txt downloads succeed) + shim_bin curl + # sha256sum / shasum both fail + shim_bin sha256sum 1 + shim_bin shasum 1 + bootstrap_gum_temp || true + [ "$GUM_REASON" = "checksum unavailable or failed" ] +} + +@test "bootstrap_gum_temp: extract failure sets reason='extract failed'" { + DOWNLOADER=curl + shim_bin curl + shim_bin sha256sum + shim_bin shasum + shim_bin tar 1 + bootstrap_gum_temp || true + [ "$GUM_REASON" = "extract failed" ] +} + +@test "bootstrap_gum_temp: missing gum binary after extract sets reason" { + DOWNLOADER=curl + shim_bin curl + shim_bin sha256sum + shim_bin shasum + # tar 'succeeds' but produces nothing + shim_bin tar + bootstrap_gum_temp || true + [ "$GUM_REASON" = "gum binary missing after extract" ] +} + +@test "bootstrap_gum_temp: gum already on PATH is reported as found" { + # Clear the missing-names list so command -v gum sees the shim on PATH. + SHIM_MISSING_NAMES=() + shim_bin gum + bootstrap_gum_temp + [ "$GUM" = "gum" ] + [ "$GUM_STATUS" = "found" ] +} + +@test "bootstrap_gum_temp: successful download+extract sets GUM_STATUS=installed" { + DOWNLOADER=curl + # curl writes a dummy archive to the -o path + shim_bin_script curl ' +out=""; prev="" +for a in "$@"; do + [[ "$prev" == "-o" ]] && out="$a" + prev="$a" +done +[[ -n "$out" ]] && printf "fake" > "$out" +' + shim_bin sha256sum + shim_bin shasum + # tar extracts a fake gum binary into the temp dir + shim_bin_script tar ' +if [[ "$1" == "-xzf" ]]; then + dest="$4" + mkdir -p "$dest" + printf "#!/bin/bash\n" > "$dest/gum" + chmod +x "$dest/gum" +fi +' + bootstrap_gum_temp + [ "$GUM_STATUS" = "installed" ] + [ -n "$GUM" ] + [ -x "$GUM" ] +} diff --git a/tools/test/integration/download_file.bats b/tools/test/integration/download_file.bats new file mode 100644 index 000000000..c4e6a5572 --- /dev/null +++ b/tools/test/integration/download_file.bats @@ -0,0 +1,88 @@ +#!/usr/bin/env bats + +setup() { + load '../helpers/load' + load '../helpers/shim' + load_install_sh + reset_install_sh_state + shim_setup +} + +@test "download_file: curl is invoked with the exact expected argv" { + DOWNLOADER=curl + shim_bin curl + + download_file "https://example.test/x.tgz" "$BATS_TEST_TMPDIR/out" + + [ "$(shim_call_count curl)" = "1" ] + local got + got="$(shim_calls curl)" + local expected + expected=$'-fL\t--progress-bar\t--proto\t=https\t--tlsv1.2\t--retry\t3\t--max-time\t600\t--retry-delay\t1\t--retry-connrefused\t-o\t'"$BATS_TEST_TMPDIR/out"$'\thttps://example.test/x.tgz' + [ "$got" = "$expected" ] +} + +@test "download_file: curl argv contains no stray '--' token (PR #599 regression guard)" { + DOWNLOADER=curl + shim_bin curl + + download_file "https://example.test/x.tgz" "$BATS_TEST_TMPDIR/out" + + local got + got="$(shim_calls curl)" + # A bare '--' token would appear as tab-bracketed or at line edges. + # Use POSIX [ ] with string-prefix stripping: if the pattern is absent, + # stripping it leaves got unchanged (strings are equal → [ ] succeeds). + # bash 3.2 does not trigger set -e for [[ ]] failures, so [ ] is required. + [ "${got#*$'\t--\t'}" = "$got" ] + [ "${got%$'\t--'}" = "$got" ] + [ "${got#'--'}" = "$got" ] +} + +@test "download_file: curl failure propagates" { + DOWNLOADER=curl + shim_bin curl 22 + + run download_file "https://example.test/x.tgz" "$BATS_TEST_TMPDIR/out" + [ "$status" -ne 0 ] +} + +@test "download_file: wget is invoked with the exact expected argv" { + DOWNLOADER=wget + shim_bin wget + + download_file "https://example.test/x.tgz" "$BATS_TEST_TMPDIR/out" + + [ "$(shim_call_count wget)" = "1" ] + local got + got="$(shim_calls wget)" + local expected + expected=$'-q\t--show-progress\t--https-only\t--secure-protocol=TLSv1_2\t--tries=3\t--timeout=600\t-O\t'"$BATS_TEST_TMPDIR/out"$'\thttps://example.test/x.tgz' + [ "$got" = "$expected" ] +} + +@test "detect_downloader: picks curl when available" { + DOWNLOADER="" + shim_bin curl + shim_bin wget + detect_downloader + [ "$DOWNLOADER" = "curl" ] +} + +@test "detect_downloader: falls back to wget when curl is missing" { + DOWNLOADER="" + shim_bin_missing curl + shim_bin wget + detect_downloader + [ "$DOWNLOADER" = "wget" ] +} + +@test "detect_downloader: dies when neither curl nor wget is available" { + DOWNLOADER="" + shim_bin_missing curl + shim_bin_missing wget + run detect_downloader + [ "$status" -ne 0 ] + # Use POSIX [ ] for set -e compatibility on bash 3.2 (see also Test 2). + [ "${output#*Missing downloader}" != "$output" ] +} diff --git a/tools/test/integration/dry_run.bats b/tools/test/integration/dry_run.bats new file mode 100644 index 000000000..db5caf3a7 --- /dev/null +++ b/tools/test/integration/dry_run.bats @@ -0,0 +1,41 @@ +#!/usr/bin/env bats + +setup() { + load '../helpers/load' + load '../helpers/shim' + shim_setup + # Fake FLINK_HOME so plan_flink (with INSTALL_FLINK=No) passes validation. + export FLINK_HOME="$BATS_TEST_TMPDIR/flink-home" + mkdir -p "$FLINK_HOME/lib" + # Fake `java` so check_java passes. + shim_bin_script java " +case \"\$1\" in + -version) echo 'openjdk version \"17.0.2\"' >&2 ;; +esac +" +} + +@test "--dry-run --non-interactive prints plan and makes no external calls" { + run bash "${BATS_TEST_DIRNAME}/../../install.sh" --dry-run --non-interactive + [ "$status" -eq 0 ] + case "$output" in *"Installation plan"*) ;; *) false ;; esac + case "$output" in *"Dry run complete"*) ;; *) false ;; esac + # No downloader should have been invoked. + [ "$(shim_call_count curl)" = "0" ] + [ "$(shim_call_count wget)" = "0" ] +} + +@test "--dry-run --install-flink --non-interactive shows Install Flink: Yes" { + run bash "${BATS_TEST_DIRNAME}/../../install.sh" --dry-run --install-flink --non-interactive + [ "$status" -eq 0 ] + case "$output" in *"Install Flink"*) ;; *) false ;; esac + case "$output" in *"Yes"*) ;; *) false ;; esac + case "$output" in *"Dry run complete"*) ;; *) false ;; esac +} + +@test "INSTALL_DIR=. does not produce a double-slash FLINK_HOME (review feedback guard)" { + cd "$BATS_TEST_TMPDIR" + run env INSTALL_DIR="." bash "${BATS_TEST_DIRNAME}/../../install.sh" --dry-run --install-flink --non-interactive + [ "$status" -eq 0 ] + case "$output" in *".//flink-"*) false ;; *) ;; esac +} diff --git a/tools/test/integration/help.bats b/tools/test/integration/help.bats new file mode 100644 index 000000000..d77335e37 --- /dev/null +++ b/tools/test/integration/help.bats @@ -0,0 +1,16 @@ +#!/usr/bin/env bats + +@test "--help prints usage and exits 0" { + run bash "${BATS_TEST_DIRNAME}/../../install.sh" --help + [ "$status" -eq 0 ] + case "$output" in *"Apache Flink Agents Installer"*) ;; *) false ;; esac + case "$output" in *"Options:"*) ;; *) false ;; esac + case "$output" in *"--install-flink"*) ;; *) false ;; esac + case "$output" in *"--dry-run"*) ;; *) false ;; esac +} + +@test "-h prints usage and exits 0" { + run bash "${BATS_TEST_DIRNAME}/../../install.sh" -h + [ "$status" -eq 0 ] + case "$output" in *"Apache Flink Agents Installer"*) ;; *) false ;; esac +} diff --git a/tools/test/integration/install_flink_if_needed.bats b/tools/test/integration/install_flink_if_needed.bats new file mode 100644 index 000000000..7bbdceeee --- /dev/null +++ b/tools/test/integration/install_flink_if_needed.bats @@ -0,0 +1,132 @@ +#!/usr/bin/env bats + +setup() { + load '../helpers/load' + load '../helpers/shim' + load_install_sh + reset_install_sh_state + shim_setup + INSTALL_DIR="$BATS_TEST_TMPDIR/install" + FLINK_VERSION="2.2.0" + FLINK_SCALA_VERSION="2.12" + FLINK_BASE_URL="https://example.test/flink" +} + +# A tar shim that succeeds for both `-tzf` (validity check) and `-xzf` (extract). +# On extract it materializes a minimal Flink home with a lib/ dir. +configure_tar_shim() { + shim_bin_script tar " +case \"\$1\" in + -tzf) exit 0 ;; + -xzf) + # arg order: -xzf -C (\$4 = dest) + mkdir -p \"\$4/flink-${FLINK_VERSION}/lib\" + ;; + *) exit 0 ;; +esac +" +} + +@test "install_flink_if_needed: INSTALL_FLINK=No uses pre-existing FLINK_HOME" { + INSTALL_FLINK=No + FLINK_HOME="$BATS_TEST_TMPDIR/preexisting" + mkdir -p "$FLINK_HOME/lib" + run install_flink_if_needed + [ "$status" -eq 0 ] + [ "$(shim_call_count curl)" = "0" ] + [ "$(shim_call_count wget)" = "0" ] +} + +@test "install_flink_if_needed: fresh install downloads then extracts" { + INSTALL_FLINK=Yes + DOWNLOADER=curl + FLINK_HOME="$INSTALL_DIR/flink-$FLINK_VERSION" + # curl shim writes a placeholder archive at the requested -o path. + shim_bin_script curl " +# argv pattern: ... -o +out=\"\" +prev=\"\" +for a in \"\$@\"; do + if [[ \"\$prev\" == \"-o\" ]]; then out=\"\$a\"; fi + prev=\"\$a\" +done +[[ -n \"\$out\" ]] && printf 'fake tgz' > \"\$out\" +" + configure_tar_shim + + run install_flink_if_needed + [ "$status" -eq 0 ] + [ "$(shim_call_count curl)" = "1" ] + case "$(shim_calls curl)" in + *"https://example.test/flink/flink-2.2.0/flink-2.2.0-bin-scala_2.12.tgz"*) ;; + *) false ;; + esac + [ -d "$INSTALL_DIR/flink-$FLINK_VERSION/lib" ] +} + +@test "install_flink_if_needed: existing valid archive is reused (no download)" { + INSTALL_FLINK=Yes + DOWNLOADER=curl + mkdir -p "$INSTALL_DIR" + # Pre-seed a "valid" archive — tar shim will report it as valid via -tzf. + printf 'existing' > "$INSTALL_DIR/flink-${FLINK_VERSION}-bin-scala_${FLINK_SCALA_VERSION}.tgz" + shim_bin curl + configure_tar_shim + FLINK_HOME="$INSTALL_DIR/flink-$FLINK_VERSION" + + run install_flink_if_needed + [ "$status" -eq 0 ] + [ "$(shim_call_count curl)" = "0" ] +} + +@test "install_flink_if_needed: corrupt existing archive triggers re-download" { + INSTALL_FLINK=Yes + DOWNLOADER=curl + mkdir -p "$INSTALL_DIR" + local archive="$INSTALL_DIR/flink-${FLINK_VERSION}-bin-scala_${FLINK_SCALA_VERSION}.tgz" + printf 'existing' > "$archive" + # tar reports the existing archive as INVALID, then valid on the re-download. + local state="$BATS_TEST_TMPDIR/tar_state" + : > "$state" + shim_bin_script tar " +case \"\$1\" in + -tzf) + if [[ ! -s '$state' ]]; then + echo bad > '$state' + exit 1 + fi + exit 0 + ;; + -xzf) + mkdir -p \"\$4/flink-${FLINK_VERSION}/lib\" + ;; +esac +" + shim_bin_script curl " +out=\"\"; prev=\"\" +for a in \"\$@\"; do + [[ \"\$prev\" == \"-o\" ]] && out=\"\$a\" + prev=\"\$a\" +done +[[ -n \"\$out\" ]] && printf 'freshly downloaded' > \"\$out\" +" + FLINK_HOME="$INSTALL_DIR/flink-$FLINK_VERSION" + + run install_flink_if_needed + [ "$status" -eq 0 ] + [ "$(shim_call_count curl)" = "1" ] +} + +@test "install_flink_if_needed: incomplete extracted dir triggers re-extract" { + INSTALL_FLINK=Yes + DOWNLOADER=curl + mkdir -p "$INSTALL_DIR/flink-${FLINK_VERSION}" # missing lib/ + printf 'archive' > "$INSTALL_DIR/flink-${FLINK_VERSION}-bin-scala_${FLINK_SCALA_VERSION}.tgz" + shim_bin curl + configure_tar_shim + FLINK_HOME="$INSTALL_DIR/flink-$FLINK_VERSION" + + run install_flink_if_needed + [ "$status" -eq 0 ] + [ -d "$INSTALL_DIR/flink-$FLINK_VERSION/lib" ] +} diff --git a/tools/test/run.sh b/tools/test/run.sh new file mode 100755 index 000000000..c5d303d73 --- /dev/null +++ b/tools/test/run.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +# Bash 4+ is required: bash 3.2 (macOS default) does not trigger `set -e` +# on `[[ ]]` failures or fire the ERR trap on them, which means many +# substring assertions in this suite would silently pass on bash 3.2. +# Force a clean failure here rather than mislead developers. +if [ -z "${BASH_VERSION:-}" ] || [ "${BASH_VERSION%%.*}" -lt 4 ]; then + echo "ERROR: bash >= 4 required (detected: ${BASH_VERSION:-unknown})." >&2 + echo "macOS ships bash 3.2 at /bin/bash; install bash 4+ via Homebrew:" >&2 + echo " brew install bash" >&2 + echo "Then run with the new bash, e.g.:" >&2 + echo " /opt/homebrew/bin/bash $0" >&2 + exit 1 +fi + +set -euo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +CACHE="$HERE/.bats-cache" + +clone_pinned() { + local name="$1" url="$2" tag="$3" + if [[ ! -d "$CACHE/$name" ]]; then + echo "Fetching $name@$tag" >&2 + git clone --quiet --depth 1 --branch "$tag" "$url" "$CACHE/$name" + fi +} + +mkdir -p "$CACHE" +clone_pinned bats-core https://github.com/bats-core/bats-core.git v1.11.0 +clone_pinned bats-support https://github.com/bats-core/bats-support.git v0.3.0 +clone_pinned bats-assert https://github.com/bats-core/bats-assert.git v2.1.0 + +export BATS_LIB_PATH="$CACHE" + +exec "$CACHE/bats-core/bin/bats" --recursive "$HERE/unit" "$HERE/integration" diff --git a/tools/test/unit/is_valid_tgz.bats b/tools/test/unit/is_valid_tgz.bats new file mode 100644 index 000000000..784022b87 --- /dev/null +++ b/tools/test/unit/is_valid_tgz.bats @@ -0,0 +1,44 @@ +#!/usr/bin/env bats + +setup() { + load '../helpers/load' + load_install_sh + reset_install_sh_state + WORK="$BATS_TEST_TMPDIR/tgz" + mkdir -p "$WORK" +} + +@test "is_valid_tgz: missing file is invalid" { + run is_valid_tgz "$WORK/nope.tgz" + [ "$status" -ne 0 ] +} + +@test "is_valid_tgz: empty file is invalid" { + printf '\x00' > "$WORK/empty.tgz" + run is_valid_tgz "$WORK/empty.tgz" + [ "$status" -ne 0 ] +} + +@test "is_valid_tgz: random bytes are invalid" { + printf 'not a tarball at all\n' > "$WORK/junk.tgz" + run is_valid_tgz "$WORK/junk.tgz" + [ "$status" -ne 0 ] +} + +@test "is_valid_tgz: a real tgz is valid" { + mkdir -p "$WORK/src" + echo hello > "$WORK/src/file.txt" + tar -czf "$WORK/good.tgz" -C "$WORK/src" . + run is_valid_tgz "$WORK/good.tgz" + [ "$status" -eq 0 ] +} + +@test "is_valid_tgz: truncated tgz is invalid" { + mkdir -p "$WORK/src" + head -c 4096 /dev/urandom > "$WORK/src/blob" + tar -czf "$WORK/full.tgz" -C "$WORK/src" . + # truncate to first 32 bytes (gzip header only) + head -c 32 "$WORK/full.tgz" > "$WORK/trunc.tgz" + run is_valid_tgz "$WORK/trunc.tgz" + [ "$status" -ne 0 ] +} diff --git a/tools/test/unit/mark_explicit.bats b/tools/test/unit/mark_explicit.bats new file mode 100644 index 000000000..68a1111f3 --- /dev/null +++ b/tools/test/unit/mark_explicit.bats @@ -0,0 +1,31 @@ +#!/usr/bin/env bats + +setup() { + load '../helpers/load' + load_install_sh + reset_install_sh_state +} + +@test "mark_explicit: unset variable yields _EXPLICIT=0" { + unset MY_VAR MY_VAR_EXPLICIT + mark_explicit MY_VAR + [ "$MY_VAR_EXPLICIT" = "0" ] +} + +@test "mark_explicit: empty variable yields _EXPLICIT=0" { + MY_VAR="" + mark_explicit MY_VAR + [ "$MY_VAR_EXPLICIT" = "0" ] +} + +@test "mark_explicit: non-empty variable yields _EXPLICIT=1" { + MY_VAR="something" + mark_explicit MY_VAR + [ "$MY_VAR_EXPLICIT" = "1" ] +} + +@test "mark_explicit: zero-string non-empty is _EXPLICIT=1" { + MY_VAR="0" + mark_explicit MY_VAR + [ "$MY_VAR_EXPLICIT" = "1" ] +} diff --git a/tools/test/unit/parse_args.bats b/tools/test/unit/parse_args.bats new file mode 100644 index 000000000..76084472f --- /dev/null +++ b/tools/test/unit/parse_args.bats @@ -0,0 +1,82 @@ +#!/usr/bin/env bats + +setup() { + load '../helpers/load' + load_install_sh + reset_install_sh_state +} + +@test "parse_args: --non-interactive sets NO_PROMPT=1" { + parse_args --non-interactive + [ "$NO_PROMPT" = "1" ] +} + +@test "parse_args: --install-flink sets INSTALL_FLINK=Yes" { + parse_args --install-flink + [ "$INSTALL_FLINK" = "Yes" ] +} + +@test "parse_args: --enable-pyflink sets ENABLE_PYFLINK=Yes" { + parse_args --enable-pyflink + [ "$ENABLE_PYFLINK" = "Yes" ] +} + +@test "parse_args: --enable-pyFlink alias is honored" { + parse_args --enable-pyFlink + [ "$ENABLE_PYFLINK" = "Yes" ] +} + +@test "parse_args: --verbose sets VERBOSE=1" { + parse_args --verbose + [ "$VERBOSE" = "1" ] +} + +@test "parse_args: --dry-run sets DRY_RUN=1" { + parse_args --dry-run + [ "$DRY_RUN" = "1" ] +} + +@test "parse_args: --verify sets VERIFY_INSTALL=1" { + parse_args --verify + [ "$VERIFY_INSTALL" = "1" ] +} + +@test "parse_args: --python sets PYTHON_BIN" { + parse_args --python /opt/py/bin/python3 + [ "$PYTHON_BIN" = "/opt/py/bin/python3" ] +} + +@test "parse_args: --python= sets PYTHON_BIN" { + parse_args --python=/usr/local/bin/python3.11 + [ "$PYTHON_BIN" = "/usr/local/bin/python3.11" ] +} + +@test "parse_args: --python without arg dies" { + run parse_args --python + [ "$status" -ne 0 ] + [[ "$output" == *"--python requires a path argument"* ]] +} + +@test "parse_args: --help sets HELP=1" { + parse_args --help + [ "$HELP" = "1" ] +} + +@test "parse_args: -h sets HELP=1" { + parse_args -h + [ "$HELP" = "1" ] +} + +@test "parse_args: unknown flag warns but does not die" { + run parse_args --no-such-flag + [ "$status" -eq 0 ] + [[ "$output" == *"Unknown option: --no-such-flag"* ]] +} + +@test "parse_args: combined flags all apply" { + parse_args --non-interactive --install-flink --enable-pyflink --verify + [ "$NO_PROMPT" = "1" ] + [ "$INSTALL_FLINK" = "Yes" ] + [ "$ENABLE_PYFLINK" = "Yes" ] + [ "$VERIFY_INSTALL" = "1" ] +} diff --git a/tools/test/unit/platform_detect.bats b/tools/test/unit/platform_detect.bats new file mode 100644 index 000000000..c13400805 --- /dev/null +++ b/tools/test/unit/platform_detect.bats @@ -0,0 +1,98 @@ +#!/usr/bin/env bats + +setup() { + load '../helpers/load' + load '../helpers/shim' + load_install_sh + reset_install_sh_state + shim_setup +} + +# Helper: install a fake `uname` that emits a chosen string for -s and -m. +fake_uname() { + local sysname="$1" machine="$2" + shim_bin_script uname " +case \"\$1\" in + -s) echo '$sysname' ;; + -m) echo '$machine' ;; + *) echo 'fake' ;; +esac +" +} + +@test "gum_detect_os: Darwin" { + fake_uname Darwin x86_64 + [ "$(gum_detect_os)" = "Darwin" ] +} + +@test "gum_detect_os: Linux" { + fake_uname Linux x86_64 + [ "$(gum_detect_os)" = "Linux" ] +} + +@test "gum_detect_os: unknown kernel" { + fake_uname FreeBSD x86_64 + [ "$(gum_detect_os)" = "unsupported" ] +} + +@test "gum_detect_arch: x86_64 stays x86_64" { + fake_uname Linux x86_64 + [ "$(gum_detect_arch)" = "x86_64" ] +} + +@test "gum_detect_arch: amd64 maps to x86_64" { + fake_uname Linux amd64 + [ "$(gum_detect_arch)" = "x86_64" ] +} + +@test "gum_detect_arch: arm64 stays arm64" { + fake_uname Darwin arm64 + [ "$(gum_detect_arch)" = "arm64" ] +} + +@test "gum_detect_arch: aarch64 maps to arm64" { + fake_uname Linux aarch64 + [ "$(gum_detect_arch)" = "arm64" ] +} + +@test "gum_detect_arch: i686 maps to i386" { + fake_uname Linux i686 + [ "$(gum_detect_arch)" = "i386" ] +} + +@test "gum_detect_arch: armv7l maps to armv7" { + fake_uname Linux armv7l + [ "$(gum_detect_arch)" = "armv7" ] +} + +@test "gum_detect_arch: unknown machine is unknown" { + fake_uname Linux riscv64 + [ "$(gum_detect_arch)" = "unknown" ] +} + +@test "detect_os_or_die: macOS via OSTYPE" { + OSTYPE="darwin23" + detect_os_or_die + [ "$OS" = "macos" ] +} + +@test "detect_os_or_die: linux via OSTYPE" { + OSTYPE="linux-gnu" + detect_os_or_die + [ "$OS" = "linux" ] +} + +@test "detect_os_or_die: WSL via WSL_DISTRO_NAME" { + OSTYPE="something-unknown" + WSL_DISTRO_NAME="Ubuntu" + detect_os_or_die + [ "$OS" = "linux" ] +} + +@test "detect_os_or_die: unsupported OSTYPE dies" { + OSTYPE="cygwin" + unset WSL_DISTRO_NAME + run detect_os_or_die + [ "$status" -ne 0 ] + [[ "$output" == *"Unsupported operating system"* ]] +} diff --git a/tools/test/unit/shim_self_test.bats b/tools/test/unit/shim_self_test.bats new file mode 100644 index 000000000..04def76a7 --- /dev/null +++ b/tools/test/unit/shim_self_test.bats @@ -0,0 +1,50 @@ +#!/usr/bin/env bats + +bats_require_minimum_version 1.5.0 + +setup() { + load '../helpers/load' + load '../helpers/shim' + load_install_sh + reset_install_sh_state + shim_setup +} + +@test "shim_bin records argv for a single call" { + shim_bin fake_tool + fake_tool --foo bar baz + [ "$(shim_call_count fake_tool)" = "1" ] + run shim_calls fake_tool + [ "$output" = "--foo bar baz" ] +} + +@test "shim_bin can simulate non-zero exit" { + shim_bin fake_tool 7 + run fake_tool + [ "$status" -eq 7 ] +} + +@test "shim_bin_script can write output files" { + shim_bin_script fake_tar 'touch "$2/marker"' + mkdir -p "$BATS_TEST_TMPDIR/out" + fake_tar -x "$BATS_TEST_TMPDIR/out" + [ -f "$BATS_TEST_TMPDIR/out/marker" ] +} + +@test "PATH shim is preferred over real binary" { + shim_bin curl + run command -v curl + [[ "$output" == "$BATS_TEST_TMPDIR/bin/curl" ]] +} + +@test "shim_bin_missing makes command -v report missing" { + shim_bin_missing tool_x + run command -v tool_x + [ "$status" -ne 0 ] +} + +@test "shim_bin_missing makes direct invocation exit 127" { + shim_bin_missing tool_y + run -127 tool_y --some arg + [ "$status" -eq 127 ] +} diff --git a/tools/test/unit/smoke.bats b/tools/test/unit/smoke.bats new file mode 100644 index 000000000..08ced7ee2 --- /dev/null +++ b/tools/test/unit/smoke.bats @@ -0,0 +1,18 @@ +#!/usr/bin/env bats + +setup() { + load '../helpers/load' + load_install_sh + reset_install_sh_state +} + +@test "install.sh sources without executing main" { + [ "$(type -t parse_args)" = "function" ] + [ "$(type -t download_file)" = "function" ] +} + +@test "reset_install_sh_state restores defaults" { + NO_PROMPT=1 + reset_install_sh_state + [ "$NO_PROMPT" = "0" ] +} diff --git a/tools/test/unit/ui_helpers.bats b/tools/test/unit/ui_helpers.bats new file mode 100644 index 000000000..61ad31d00 --- /dev/null +++ b/tools/test/unit/ui_helpers.bats @@ -0,0 +1,54 @@ +#!/usr/bin/env bats + +setup() { + load '../helpers/load' + load_install_sh + reset_install_sh_state + # Force the non-gum fallback branch. + GUM="" +} + +@test "ui_info: prints the message in fallback branch" { + run ui_info "hello world" + [ "$status" -eq 0 ] + [[ "$output" == *"hello world"* ]] +} + +@test "ui_warn: prints the message in fallback branch" { + run ui_warn "be careful" + [ "$status" -eq 0 ] + [[ "$output" == *"be careful"* ]] +} + +@test "ui_success: prints the message in fallback branch" { + run ui_success "all good" + [ "$status" -eq 0 ] + [[ "$output" == *"all good"* ]] +} + +@test "ui_error: prints the message in fallback branch" { + run ui_error "uh oh" + [ "$status" -eq 0 ] + [[ "$output" == *"uh oh"* ]] +} + +@test "ui_kv: prints key and value" { + run ui_kv "Flink version" "2.2.0" + [ "$status" -eq 0 ] + [[ "$output" == *"Flink version"* ]] + [[ "$output" == *"2.2.0"* ]] +} + +@test "ui_stage: increments stage counter and prints title" { + INSTALL_STAGE_CURRENT=0 + run ui_stage "First thing" + [ "$status" -eq 0 ] + [[ "$output" == *"First thing"* ]] + [[ "$output" == *"[1/${INSTALL_STAGE_TOTAL}]"* ]] +} + +@test "die: prints message and exits non-zero" { + run die "fatal boom" + [ "$status" -ne 0 ] + [[ "$output" == *"fatal boom"* ]] +} diff --git a/tools/test/unit/validate_python_bin.bats b/tools/test/unit/validate_python_bin.bats new file mode 100644 index 000000000..87998bd18 --- /dev/null +++ b/tools/test/unit/validate_python_bin.bats @@ -0,0 +1,73 @@ +#!/usr/bin/env bats + +setup() { + load '../helpers/load' + load '../helpers/shim' + load_install_sh + reset_install_sh_state + shim_setup +} + +# Helper: write a fake python that reports a chosen version. +fake_python() { + local ver="$1" + shim_bin_script fake_py " +case \"\$*\" in + *'sys.version_info.major'*) + echo '$ver' + ;; +esac +" +} + +@test "validate_python_bin: empty path is rejected" { + run validate_python_bin "" + [ "$status" -ne 0 ] +} + +@test "validate_python_bin: missing binary is rejected" { + run validate_python_bin /no/such/bin + [ "$status" -ne 0 ] +} + +@test "validate_python_bin: Python 3.10 is accepted" { + fake_python "3.10" + run validate_python_bin fake_py + [ "$status" -eq 0 ] +} + +@test "validate_python_bin: Python 3.11 is accepted" { + fake_python "3.11" + run validate_python_bin fake_py + [ "$status" -eq 0 ] +} + +@test "validate_python_bin: Python 3.9 is rejected" { + fake_python "3.9" + run validate_python_bin fake_py + [ "$status" -ne 0 ] +} + +@test "validate_python_bin: Python 3.12 is rejected" { + fake_python "3.12" + run validate_python_bin fake_py + [ "$status" -ne 0 ] +} + +@test "validate_python_bin: Python 3.13 is rejected" { + fake_python "3.13" + run validate_python_bin fake_py + [ "$status" -ne 0 ] +} + +@test "validate_python_bin: Python 2.7 is rejected" { + fake_python "2.7" + run validate_python_bin fake_py + [ "$status" -ne 0 ] +} + +@test "validate_python_bin: binary that prints nothing is rejected" { + shim_bin_script fake_py 'exit 0' + run validate_python_bin fake_py + [ "$status" -ne 0 ] +} From 525adad503925536514cfd8a471b949f2439d035 Mon Sep 17 00:00:00 2001 From: WenjinXie Date: Mon, 18 May 2026 18:20:26 +0800 Subject: [PATCH 19/35] [tools] install.sh: rolling pip install output Co-Authored-By: Claude Opus 4.7 (1M context) --- tools/install.sh | 62 ++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 60 insertions(+), 2 deletions(-) diff --git a/tools/install.sh b/tools/install.sh index d05ed5683..baca0a985 100755 --- a/tools/install.sh +++ b/tools/install.sh @@ -769,6 +769,62 @@ create_venv() { die "Virtual environment creation failed" } +# Run `python -m pip install ` with reduced terminal noise. +# Strategy depends on environment: +# VERBOSE=1 : full output, pass-through +# gum available + tty : `gum spin --show-error` (spinner; pip output +# is hidden unless pip exits non-zero) +# plain tty (no gum) : single rolling status line via \r\033[K; +# full output goes to a temp log and is shown +# only on failure +# non-tty (CI / piped) : pip --quiet (errors still surface) +pip_install_quiet() { + local pkgs=("$@") + + if [[ "$VERBOSE" == "1" ]]; then + python -m pip install "${pkgs[@]}" + return + fi + + if [[ -n "$GUM" ]] && gum_is_tty; then + "$GUM" spin --show-error \ + --title "pip install ${pkgs[*]}" \ + -- python -m pip install "${pkgs[@]}" + return + fi + + if [[ ! -t 1 ]]; then + python -m pip install -q "${pkgs[@]}" + return + fi + + local log + log="$(mktempfile)" + local cols max + cols="$(tput cols 2>/dev/null || echo 80)" + local prefix=" · " + max=$((cols - ${#prefix} - 4)) + (( max < 20 )) && max=20 + + local rc=0 + { + python -m pip install "${pkgs[@]}" 2>&1 \ + | while IFS= read -r line; do + printf '%s\n' "$line" >> "$log" + printf '\r\033[K%s%s' "$prefix" "${line:0:$max}" + done + } || rc=$? + + # Clear the rolling status line. + printf '\r\033[K' + + if (( rc != 0 )); then + ui_error "pip install failed; full output:" + sed 's/^/ /' "$log" >&2 + return $rc + fi +} + setup_python_env() { if [[ ! -d "$VENV_DIR" ]]; then ui_info "Creating virtual environment: $VENV_DIR" @@ -783,8 +839,10 @@ setup_python_env() { export PIP_NO_COLOR=1 export PIP_NO_INPUT=1 - ui_info "Installing Python packages" - python -m pip install "flink-agents==${FLINK_AGENTS_VERSION}" "apache-flink==${FLINK_VERSION}" + ui_info "Installing Python packages (may take a few minutes)..." + pip_install_quiet \ + "flink-agents==${FLINK_AGENTS_VERSION}" \ + "apache-flink==${FLINK_VERSION}" } copy_flink_agents_jars() { From 54ccd1b93ae0bfb7243641fc461eea0c5fdb41c4 Mon Sep 17 00:00:00 2001 From: WenjinXie Date: Mon, 18 May 2026 19:09:44 +0800 Subject: [PATCH 20/35] [tools] install.sh: allow editing the install plan at confirm prompt Co-Authored-By: Claude Opus 4.7 (1M context) --- tools/install.sh | 187 +++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 182 insertions(+), 5 deletions(-) diff --git a/tools/install.sh b/tools/install.sh index baca0a985..3b27fa601 100755 --- a/tools/install.sh +++ b/tools/install.sh @@ -607,15 +607,192 @@ plan_pyflink() { esac } +# Echo one of: "confirm" | "edit" | "cancel" +confirm_plan_action_interactive() { + if [[ -n "$GUM" ]] && gum_is_tty; then + local selection _rc=0 + selection="$("$GUM" choose \ + --header "Proceed with installation?" \ + --cursor-prefix "❯ " \ + "Proceed" "Edit a setting" "Cancel" < /dev/tty)" || _rc=$? + (( _rc == 0 )) || die_cancelled + case "$selection" in + Proceed) echo confirm ;; + "Edit a setting") echo edit ;; + *) echo cancel ;; + esac + return + fi + + printf 'Proceed with installation?\n' > /dev/tty + printf ' 1) Proceed\n' > /dev/tty + printf ' 2) Edit a setting\n' > /dev/tty + printf ' 3) Cancel\n' > /dev/tty + local answer="" + printf 'Enter choice [1-3, default 1]: ' > /dev/tty + read -r answer < /dev/tty || die_cancelled + case "$answer" in + 2) echo edit ;; + 3) echo cancel ;; + *) echo confirm ;; + esac +} + +# Prompt the user to pick a Flink home interactively. Loops until a valid +# path (exists and has a lib/ subdir) is provided. Mirrors the loop in +# plan_flink so the two stay in sync. +edit_prompt_flink_home() { + FLINK_HOME="" + while [[ -z "${FLINK_HOME:-}" || ! -d "${FLINK_HOME}" || ! -d "${FLINK_HOME}/lib" ]]; do + if [[ -n "${FLINK_HOME:-}" ]]; then + if [[ ! -d "${FLINK_HOME}" ]]; then + ui_warn "Path does not exist: ${FLINK_HOME}" + else + ui_warn "Not a valid Flink home (missing 'lib' directory): ${FLINK_HOME}" + fi + fi + FLINK_HOME="$(prompt_path_input "Enter the path to your existing FLINK_HOME (version >= 1.20)" "/path/to/flink-${FLINK_VERSION}")" + done + export FLINK_HOME +} + +# Show a menu of currently-applicable plan fields and re-prompt for the +# selected one. After the edit, recomputes FLINK_HOME / FLINK_MAJOR_MINOR +# so the next show_install_plan reflects the change. +edit_plan_interactive() { + local labels=() + local actions=() + + labels+=("Install Flink: $INSTALL_FLINK") + actions+=("install_flink") + + if [[ "$INSTALL_FLINK" == "Yes" ]]; then + labels+=("Flink version: $FLINK_VERSION") + actions+=("flink_version") + labels+=("Install directory: $INSTALL_DIR") + actions+=("install_dir") + else + labels+=("FLINK_HOME: ${FLINK_HOME:-}") + actions+=("flink_home") + fi + + labels+=("Enable PyFlink: $ENABLE_PYFLINK") + actions+=("enable_pyflink") + + if [[ "$ENABLE_PYFLINK" == "Yes" ]]; then + labels+=("Venv directory: $VENV_DIR") + actions+=("venv_dir") + fi + + labels+=("Back") + actions+=("back") + + local picked_index=-1 + if [[ -n "$GUM" ]] && gum_is_tty; then + local _rc=0 + local selected="" + selected="$("$GUM" choose \ + --header "Edit a setting" \ + --cursor-prefix "❯ " \ + "${labels[@]}" < /dev/tty)" || _rc=$? + (( _rc == 0 )) || die_cancelled + local i=0 + for l in "${labels[@]}"; do + if [[ "$l" == "$selected" ]]; then + picked_index=$i + break + fi + i=$((i+1)) + done + else + printf 'Edit a setting:\n' > /dev/tty + local i=1 + for l in "${labels[@]}"; do + printf ' %d) %s\n' "$i" "$l" > /dev/tty + i=$((i+1)) + done + local answer="" + printf 'Enter choice [1-%d]: ' "${#labels[@]}" > /dev/tty + read -r answer < /dev/tty || die_cancelled + if [[ "$answer" =~ ^[0-9]+$ ]] && (( answer >= 1 && answer <= ${#labels[@]} )); then + picked_index=$((answer - 1)) + fi + fi + + (( picked_index < 0 )) && return 0 + + local action="${actions[$picked_index]}" + case "$action" in + install_flink) + if choose_install_method_interactive "Install Flink?"; then + INSTALL_FLINK=Yes + FLINK_HOME="${INSTALL_DIR}/flink-${FLINK_VERSION}" + else + INSTALL_FLINK=No + edit_prompt_flink_home + fi + FLINK_MAJOR_MINOR="${FLINK_VERSION%.*}" + ;; + flink_version) + prompt_flink_version_interactive || true + FLINK_HOME="${INSTALL_DIR}/flink-${FLINK_VERSION}" + FLINK_MAJOR_MINOR="${FLINK_VERSION%.*}" + ;; + install_dir) + INSTALL_DIR="$(prompt_path_choice_interactive \ + "Choose Flink install directory" \ + "$INSTALL_DIR" \ + "/path/to/flink-install-dir")" + FLINK_HOME="${INSTALL_DIR}/flink-${FLINK_VERSION}" + ;; + flink_home) + edit_prompt_flink_home + ;; + enable_pyflink) + if choose_install_method_interactive "Enable PyFlink?"; then + if [[ "$ENABLE_PYFLINK" != "Yes" ]]; then + ENABLE_PYFLINK=Yes + PYFLINK_ACTUALLY_ENABLED=1 + resolve_python + fi + else + ENABLE_PYFLINK=No + fi + ;; + venv_dir) + VENV_DIR="$(prompt_path_choice_interactive \ + "Choose Python venv directory" \ + "$VENV_DIR" \ + "/path/to/venv")" + case "$VENV_DIR" in + /*) ;; + *) VENV_DIR="$PWD/$VENV_DIR" ;; + esac + ;; + back|*) + ;; + esac +} + confirm_install_plan() { if [[ "$NO_PROMPT" == "1" ]] || ! is_promptable; then return 0 fi - if choose_install_method_interactive "Proceed with installation?"; then - return 0 - fi - ui_info "Installation cancelled by user." - exit 0 + while true; do + case "$(confirm_plan_action_interactive)" in + confirm) return 0 ;; + edit) + edit_plan_interactive + # Re-display the updated plan so the user sees their change + # in context before re-prompting. + show_install_plan + ;; + cancel) + ui_info "Installation cancelled by user." + exit 0 + ;; + esac + done } install_flink_if_needed() { From 13f0f5d8fb991ae5e4995b01d87508af1007416f Mon Sep 17 00:00:00 2001 From: WenjinXie Date: Wed, 20 May 2026 21:17:41 +0800 Subject: [PATCH 21/35] [tools] install.sh: normalize_path helper to fix double-slash and tilde paths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reviewer feedback (#7b): user-supplied INSTALL_DIR / FLINK_HOME / VENV_DIR with trailing slashes, leading "~", "." or consecutive "/" produced ugly or broken paths like "/x//flink-2.2.0". Add a small normalize_path() that: - expands "~" against $HOME, - anchors relative paths at $PWD, - folds "/./" and trailing "/.", - collapses runs of "/", - strips trailing "/" (except root). Route every path input through it: prompt_path_input, the default and custom branches of prompt_path_choice_interactive, the explicit / env-var paths in plan_flink (Yes + No), plan_pyflink (replacing the relative→ absolute case statement), and the install_dir / venv_dir actions in edit_plan_interactive. Tests: - 9 unit tests covering ~, relative, "." , trailing "/", consecutive "/", "/./", combined. - 2 integration tests on the dry-run plan: trailing slash, "//" in INSTALL_DIR no longer leak into FLINK_HOME. Co-Authored-By: Claude Opus 4.7 --- tools/install.sh | 58 +++++++++++++++++++------- tools/test/integration/dry_run.bats | 13 ++++++ tools/test/unit/normalize_path.bats | 64 +++++++++++++++++++++++++++++ 3 files changed, 121 insertions(+), 14 deletions(-) create mode 100644 tools/test/unit/normalize_path.bats diff --git a/tools/install.sh b/tools/install.sh index 3b27fa601..89f117a9c 100755 --- a/tools/install.sh +++ b/tools/install.sh @@ -428,6 +428,39 @@ require_cmd() { command -v "$1" >/dev/null 2>&1 || die "Missing required command: $1" } +# Normalize a filesystem path so downstream string interpolation can rely on +# a consistent shape. Empty input is passed through (callers handle "user +# entered nothing"). Otherwise: +# - "~" prefix expands to $HOME +# - relative paths are anchored at $PWD +# - "/./" segments and trailing "/." are folded away +# - trailing slashes are stripped, except for the root "/" +# - runs of '/' collapse to a single '/' +normalize_path() { + local p="$1" + if [[ -z "$p" ]]; then + printf '' + return 0 + fi + p="${p/#\~/$HOME}" + if [[ "$p" != /* ]]; then + p="$PWD/$p" + fi + while [[ "$p" == *//* ]]; do + p="${p//\/\//\/}" + done + while [[ "$p" == */./* ]]; do + p="${p//\/.\//\/}" + done + while [[ "$p" == */. ]]; do + p="${p%/.}" + done + while [[ "${#p}" -gt 1 && "$p" == */ ]]; do + p="${p%/}" + done + printf '%s' "$p" +} + is_valid_tgz() { local archive="$1" [[ -f "$archive" ]] || return 1 @@ -536,6 +569,9 @@ plan_flink() { esac if [[ "$INSTALL_FLINK" == "No" ]]; then + if [[ -n "${FLINK_HOME:-}" ]]; then + FLINK_HOME="$(normalize_path "$FLINK_HOME")" + fi if is_promptable; then while [[ -z "${FLINK_HOME:-}" || ! -d "${FLINK_HOME}" || ! -d "${FLINK_HOME}/lib" ]]; do if [[ -n "${FLINK_HOME:-}" ]]; then @@ -567,6 +603,7 @@ plan_flink() { "/path/to/flink-install-dir")" fi + INSTALL_DIR="$(normalize_path "$INSTALL_DIR")" FLINK_HOME="${INSTALL_DIR}/flink-${FLINK_VERSION}" FLINK_MAJOR_MINOR="${FLINK_VERSION%.*}" } @@ -601,10 +638,7 @@ plan_pyflink() { "/path/to/venv")" fi - case "$VENV_DIR" in - /*) ;; - *) VENV_DIR="$PWD/$VENV_DIR" ;; - esac + VENV_DIR="$(normalize_path "$VENV_DIR")" } # Echo one of: "confirm" | "edit" | "cancel" @@ -743,6 +777,7 @@ edit_plan_interactive() { "Choose Flink install directory" \ "$INSTALL_DIR" \ "/path/to/flink-install-dir")" + INSTALL_DIR="$(normalize_path "$INSTALL_DIR")" FLINK_HOME="${INSTALL_DIR}/flink-${FLINK_VERSION}" ;; flink_home) @@ -764,10 +799,7 @@ edit_plan_interactive() { "Choose Python venv directory" \ "$VENV_DIR" \ "/path/to/venv")" - case "$VENV_DIR" in - /*) ;; - *) VENV_DIR="$PWD/$VENV_DIR" ;; - esac + VENV_DIR="$(normalize_path "$VENV_DIR")" ;; back|*) ;; @@ -858,8 +890,7 @@ prompt_path_input() { printf '%s: ' "$header" > /dev/tty read -r input < /dev/tty || die_cancelled fi - input="${input/#\~/$HOME}" - printf '%s' "$input" + printf '%s' "$(normalize_path "$input")" } prompt_path_choice_interactive() { @@ -897,7 +928,7 @@ prompt_path_choice_interactive() { fi if [[ "$selection" != "$custom_label" ]]; then - printf '%s' "$default_path" + printf '%s' "$(normalize_path "$default_path")" return 0 fi @@ -913,15 +944,14 @@ prompt_path_choice_interactive() { printf 'Enter custom path: ' > /dev/tty read -r input < /dev/tty || die_cancelled fi - input="${input/#\~/$HOME}" if [[ -z "$input" ]]; then ui_warn "Empty path; falling back to default: $default_path" - printf '%s' "$default_path" + printf '%s' "$(normalize_path "$default_path")" return 0 fi - printf '%s' "$input" + printf '%s' "$(normalize_path "$input")" } copy_pyflink_jar() { diff --git a/tools/test/integration/dry_run.bats b/tools/test/integration/dry_run.bats index db5caf3a7..f16982140 100644 --- a/tools/test/integration/dry_run.bats +++ b/tools/test/integration/dry_run.bats @@ -39,3 +39,16 @@ esac [ "$status" -eq 0 ] case "$output" in *".//flink-"*) false ;; *) ;; esac } + +@test "INSTALL_DIR with trailing slash does not produce double-slash (review #7b)" { + run env INSTALL_DIR="/tmp/flink-test/" bash "${BATS_TEST_DIRNAME}/../../install.sh" --dry-run --install-flink --non-interactive + [ "$status" -eq 0 ] + case "$output" in *"//"*) false ;; *) ;; esac + case "$output" in *"/tmp/flink-test"*) ;; *) false ;; esac +} + +@test "INSTALL_DIR with consecutive slashes is collapsed (review #7b)" { + run env INSTALL_DIR="/tmp//flink-test" bash "${BATS_TEST_DIRNAME}/../../install.sh" --dry-run --install-flink --non-interactive + [ "$status" -eq 0 ] + case "$output" in *"//flink-"*) false ;; *) ;; esac +} diff --git a/tools/test/unit/normalize_path.bats b/tools/test/unit/normalize_path.bats new file mode 100644 index 000000000..96ecd578f --- /dev/null +++ b/tools/test/unit/normalize_path.bats @@ -0,0 +1,64 @@ +#!/usr/bin/env bats + +setup() { + load '../helpers/load' + load_install_sh + reset_install_sh_state +} + +@test "normalize_path: empty input yields empty string" { + run normalize_path "" + [ "$status" -eq 0 ] + [ "$output" = "" ] +} + +@test "normalize_path: tilde expands to \$HOME" { + HOME="/home/u" run normalize_path "~/foo" + [ "$status" -eq 0 ] + [ "$output" = "/home/u/foo" ] +} + +@test "normalize_path: relative path resolves against PWD" { + cd "$BATS_TEST_TMPDIR" + run normalize_path "bar" + [ "$status" -eq 0 ] + [ "$output" = "$BATS_TEST_TMPDIR/bar" ] +} + +@test "normalize_path: dot relative is collapsed to current dir" { + cd "$BATS_TEST_TMPDIR" + run normalize_path "." + [ "$status" -eq 0 ] + [ "$output" = "$BATS_TEST_TMPDIR" ] +} + +@test "normalize_path: trailing slash is stripped" { + run normalize_path "/x/y/" + [ "$status" -eq 0 ] + [ "$output" = "/x/y" ] +} + +@test "normalize_path: multiple trailing slashes are stripped" { + run normalize_path "/x/y///" + [ "$status" -eq 0 ] + [ "$output" = "/x/y" ] +} + +@test "normalize_path: consecutive slashes are collapsed" { + run normalize_path "/x//y///z" + [ "$status" -eq 0 ] + [ "$output" = "/x/y/z" ] +} + +@test "normalize_path: root '/' is preserved" { + run normalize_path "/" + [ "$status" -eq 0 ] + [ "$output" = "/" ] +} + +@test "normalize_path: combined tilde + dot + trailing slash" { + HOME=/h + run normalize_path "~/a/./b/" + [ "$status" -eq 0 ] + [ "$output" = "/h/a/b" ] +} From 0e721917136d8f5c5cb15c0715155c365afe1d49 Mon Sep 17 00:00:00 2001 From: WenjinXie Date: Wed, 20 May 2026 21:19:49 +0800 Subject: [PATCH 22/35] [tools] install.sh: auto-detect Flink version from existing FLINK_HOME MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reviewer feedback (#2): when users chose "I already have Flink" and entered a FLINK_HOME, the installer kept the default FLINK_VERSION=2.2.0 internally and later failed looking for opt/flink-python-2.2.0.jar. Add detect_flink_version_from_home(): - fast path: parse the version out of lib/flink-dist-.jar — instant. - slow path: run "${FLINK_HOME}/bin/flink --version" — authoritative but spins up a JVM (3-10s). Announce progress before invoking so the user doesn't think the installer hung. plan_flink (No branch) now calls it after path validation, prints "FLINK_HOME accepted" + "Detected Flink version: X.Y.Z" so the user sees the result, and falls back to a warning (keep current FLINK_VERSION) when the layout is non-standard. Tests cover the fast/slow paths, stderr noise, no-source-available, and that the progress line only appears on the slow path. Co-Authored-By: Claude Opus 4.7 --- tools/install.sh | 41 ++++++++++ .../unit/detect_flink_version_from_home.bats | 80 +++++++++++++++++++ 2 files changed, 121 insertions(+) create mode 100644 tools/test/unit/detect_flink_version_from_home.bats diff --git a/tools/install.sh b/tools/install.sh index 89f117a9c..b2216e865 100755 --- a/tools/install.sh +++ b/tools/install.sh @@ -552,6 +552,40 @@ prompt_flink_version_interactive() { return 0 } +# Populate FLINK_VERSION from an existing FLINK_HOME. Tries to parse +# `lib/flink-dist-.jar` first because it's instant; falls back to +# `bin/flink --version`, which is authoritative but spins up a JVM and can +# take 3-10s on cold start. Returns 0 on success. +detect_flink_version_from_home() { + [[ -n "${FLINK_HOME:-}" && -d "$FLINK_HOME" ]] || return 1 + + local major_minor_patch="" + + # Fast path: filename inspection. Avoids JVM startup entirely. + local jar + for jar in "$FLINK_HOME/lib"/flink-dist-[0-9]*.jar; do + [[ -f "$jar" ]] || continue + major_minor_patch="$(basename "$jar" | sed -E -n 's/^flink-dist-([0-9]+\.[0-9]+\.[0-9]+)\.jar$/\1/p')" + [[ -n "$major_minor_patch" ]] && break + done + + # Slow path: ask the Flink CLI. JVM startup means a few seconds of + # silence, so signal what's happening. + if [[ -z "$major_minor_patch" && -x "$FLINK_HOME/bin/flink" ]]; then + ui_info "Detecting Flink version (running '${FLINK_HOME}/bin/flink --version', this may take a few seconds)..." + local out + out="$("$FLINK_HOME/bin/flink" --version 2>/dev/null || true)" + major_minor_patch="$(printf '%s\n' "$out" | sed -E -n 's/.*Version:[[:space:]]*([0-9]+\.[0-9]+\.[0-9]+).*/\1/p' | head -n1)" + fi + + if [[ -z "$major_minor_patch" ]]; then + return 1 + fi + + FLINK_VERSION="$major_minor_patch" + return 0 +} + plan_flink() { case "$INSTALL_FLINK" in Yes|No) @@ -588,6 +622,13 @@ plan_flink() { [[ -n "${FLINK_HOME:-}" ]] || die "FLINK_HOME is not set." [[ -d "$FLINK_HOME" ]] || die "FLINK_HOME does not exist: $FLINK_HOME" [[ -d "$FLINK_HOME/lib" ]] || die "Invalid FLINK_HOME (missing lib directory): $FLINK_HOME" + ui_success "FLINK_HOME accepted: $FLINK_HOME" + if detect_flink_version_from_home; then + ui_success "Detected Flink version: $FLINK_VERSION" + else + ui_warn "Could not auto-detect Flink version from $FLINK_HOME; assuming ${FLINK_VERSION}." + ui_warn "If JAR copy fails, set FLINK_VERSION explicitly (e.g. FLINK_VERSION=2.1.1 bash install.sh)." + fi FLINK_MAJOR_MINOR="${FLINK_VERSION%.*}" return fi diff --git a/tools/test/unit/detect_flink_version_from_home.bats b/tools/test/unit/detect_flink_version_from_home.bats new file mode 100644 index 000000000..d78894ed8 --- /dev/null +++ b/tools/test/unit/detect_flink_version_from_home.bats @@ -0,0 +1,80 @@ +#!/usr/bin/env bats + +setup() { + load '../helpers/load' + load_install_sh + reset_install_sh_state + FLINK_HOME="$BATS_TEST_TMPDIR/flink-home" + mkdir -p "$FLINK_HOME/bin" "$FLINK_HOME/lib" +} + +@test "detect_flink_version_from_home: prefers bin/flink --version when runnable" { + cat > "$FLINK_HOME/bin/flink" <<'EOF' +#!/usr/bin/env bash +echo "Version: 2.1.1, Commit ID: abc" +EOF + chmod +x "$FLINK_HOME/bin/flink" + FLINK_VERSION="" + run detect_flink_version_from_home + [ "$status" -eq 0 ] + [ "$FLINK_VERSION" = "2.1.1" ] || { + # `run` executes in subshell — re-run inline to read the assignment. + FLINK_VERSION="" + detect_flink_version_from_home + [ "$FLINK_VERSION" = "2.1.1" ] + } +} + +@test "detect_flink_version_from_home: falls back to lib/flink-dist-*.jar" { + : > "$FLINK_HOME/lib/flink-dist-2.0.1.jar" + FLINK_VERSION="" + detect_flink_version_from_home + [ "$FLINK_VERSION" = "2.0.1" ] +} + +@test "detect_flink_version_from_home: returns non-zero when no source available" { + FLINK_VERSION="" + run detect_flink_version_from_home + [ "$status" -ne 0 ] +} + +@test "detect_flink_version_from_home: ignores stderr noise from flink --version" { + cat > "$FLINK_HOME/bin/flink" <<'EOF' +#!/usr/bin/env bash +echo "WARNING: some noise" >&2 +echo "Version: 1.20.3, Commit ID: deadbeef" +EOF + chmod +x "$FLINK_HOME/bin/flink" + FLINK_VERSION="" + detect_flink_version_from_home + [ "$FLINK_VERSION" = "1.20.3" ] +} + +@test "detect_flink_version_from_home: jar fallback picks first match deterministically" { + : > "$FLINK_HOME/lib/flink-dist-2.2.0.jar" + : > "$FLINK_HOME/lib/flink-dist-other-1.0.jar" + FLINK_VERSION="" + detect_flink_version_from_home + [ "$FLINK_VERSION" = "2.2.0" ] +} + +@test "detect_flink_version_from_home: announces progress before invoking bin/flink" { + # No dist jar in lib/ -> forces the slow path that runs bin/flink --version. + cat > "$FLINK_HOME/bin/flink" <<'EOF' +#!/usr/bin/env bash +echo "Version: 2.1.1, Commit ID: abc" +EOF + chmod +x "$FLINK_HOME/bin/flink" + FLINK_VERSION="" + run detect_flink_version_from_home + [ "$status" -eq 0 ] + case "$output" in *"Detecting Flink version"*) ;; *) false ;; esac +} + +@test "detect_flink_version_from_home: fast jar path does NOT announce progress" { + : > "$FLINK_HOME/lib/flink-dist-2.0.1.jar" + FLINK_VERSION="" + run detect_flink_version_from_home + [ "$status" -eq 0 ] + case "$output" in *"Detecting Flink version"*) false ;; *) ;; esac +} From 1306adefcf807055e7114a61a19766a5df4c73c8 Mon Sep 17 00:00:00 2001 From: WenjinXie Date: Wed, 20 May 2026 21:23:59 +0800 Subject: [PATCH 23/35] [tools] install.sh: install flink-agents JAR via direct ASF download Reviewer feedback (#1) and follow-up: stop extracting the JAR out of the Python wheel; download the official artifact straight from the ASF mirror exactly like the Flink tarball, and verify it with the SHA512 sidecar published next to the JAR on downloads.apache.org. Effect: Java-only users no longer need any Python at all. PyFlink users still get their venv from pip, but the JAR in FLINK_HOME/lib comes from the same authoritative source for both audiences. Implementation: - flink_agents_jar_relpath() builds flink-agents-/flink-agents-dist-flink--.jar - install_flink_agents_jar() downloads to FLINK_HOME/lib (skips if already present), then verifies. - verify_flink_agents_jar_sha512() best-effort: missing sidecar / missing sha512 tool / malformed hash all degrade to a warning so a transient mirror lag never blocks an install; an actual hash mismatch is a hard die(). - Replaces copy_flink_agents_jars() (wheel extraction) and removes the short-lived install_agent_jars_without_pyflink() temp-venv shim. - verify_installation now checks the single flink-agents-dist-flink JAR (no separate common / -thin JARs after the wheel was retired). - print_usage documents FLINK_AGENTS_BASE_URL / FLINK_AGENTS_CHECKSUM_BASE_URL. Five integration tests cover relpath construction, mirror+sidecar URLs, JAR reuse, empty-download failure, and downloader failure. Co-Authored-By: Claude Opus 4.7 --- tools/install.sh | 154 +++++++++++------- .../integration/install_flink_agents_jar.bats | 98 +++++++++++ 2 files changed, 197 insertions(+), 55 deletions(-) create mode 100644 tools/test/integration/install_flink_agents_jar.bats diff --git a/tools/install.sh b/tools/install.sh index b2216e865..3b2f70452 100755 --- a/tools/install.sh +++ b/tools/install.sh @@ -368,6 +368,12 @@ FLINK_VERSION="${FLINK_VERSION:-2.2.0}" FLINK_AGENTS_VERSION="${FLINK_AGENTS_VERSION:-0.2.1}" FLINK_SCALA_VERSION="${FLINK_SCALA_VERSION:-2.12}" FLINK_BASE_URL="${FLINK_BASE_URL:-https://dlcdn.apache.org/flink}" +# Flink Agents JARs live next to Flink on the ASF mirror network. Override +# this to point at archive.apache.org if you need a non-current release. +FLINK_AGENTS_BASE_URL="${FLINK_AGENTS_BASE_URL:-https://dlcdn.apache.org/flink}" +# Direct (non-mirrored) ASF download host. We hit it for the SHA512 sidecar +# because the mirror network does not redistribute checksum files. +FLINK_AGENTS_CHECKSUM_BASE_URL="${FLINK_AGENTS_CHECKSUM_BASE_URL:-https://downloads.apache.org/flink}" FLINK_SUPPORTED_VERSIONS=("2.2.0" "2.1.1" "2.0.1" "1.20.3") FLINK_RECOMMENDED_VERSION="2.2.0" @@ -406,7 +412,9 @@ Environment variables: FLINK_VERSION Flink version FLINK_AGENTS_VERSION Flink Agents version to install FLINK_SCALA_VERSION Scala version suffix (default: 2.12) - FLINK_BASE_URL Mirror base URL (default: https://dlcdn.apache.org/flink) + FLINK_BASE_URL Mirror base URL for Flink (default: https://dlcdn.apache.org/flink) + FLINK_AGENTS_BASE_URL Mirror base URL for Flink Agents JARs (default: https://dlcdn.apache.org/flink) + FLINK_AGENTS_CHECKSUM_BASE_URL Direct ASF URL for SHA512 sidecars (default: https://downloads.apache.org/flink) INSTALL_FLINK Ask|Yes|No (default: Ask) ENABLE_PYFLINK Ask|Yes|No (default: Ask) INSTALL_DIR Flink install directory (default: \$HOME/.local/flink) @@ -1093,40 +1101,90 @@ setup_python_env() { "apache-flink==${FLINK_VERSION}" } -copy_flink_agents_jars() { - local pkg_root - pkg_root="$(python - <<'PY' -import pathlib -import flink_agents -print(pathlib.Path(flink_agents.__file__).resolve().parent) -PY -)" +# Compute the relative path of the flink-agents JAR under the ASF mirror, +# e.g. flink-agents-0.2.1/flink-agents-dist-flink-2.2-0.2.1.jar +flink_agents_jar_relpath() { + printf '%s' "flink-agents-${FLINK_AGENTS_VERSION}/flink-agents-dist-flink-${FLINK_MAJOR_MINOR}-${FLINK_AGENTS_VERSION}.jar" +} - local version_lib_dir="${pkg_root}/lib/flink-${FLINK_MAJOR_MINOR}" - local common_lib_dir="${pkg_root}/lib/common" +# Compare a downloaded artifact against its .sha512 sidecar. The sidecar +# format from ASF is " " (single line). Best-effort: a +# missing sidecar or missing sha512 tool warns but does not fail (mirrors +# may not always carry the checksum on time). +verify_flink_agents_jar_sha512() { + local jar="$1" + local sha_url="$2" + local sha_file + sha_file="$(mktempfile)" + + if ! download_file "$sha_url" "$sha_file" >/dev/null 2>&1; then + ui_warn "SHA512 sidecar unavailable (${sha_url}); skipping verification" + return 0 + fi - [[ -d "$version_lib_dir" ]] || die "Flink Agents lib directory not found: $version_lib_dir" - [[ -d "$common_lib_dir" ]] || die "Flink Agents common lib directory not found: $common_lib_dir" + local expected actual + expected="$(awk 'NR==1 {print $1}' "$sha_file" 2>/dev/null || true)" + if [[ -z "$expected" ]]; then + ui_warn "SHA512 sidecar is empty; skipping verification" + return 0 + fi + # SHA-512 hex is exactly 128 lowercase/upper hex chars. If the sidecar + # didn't look like a real ASF .sha512 file, warn instead of failing. + if [[ ! "$expected" =~ ^[A-Fa-f0-9]{128}$ ]]; then + ui_warn "SHA512 sidecar at ${sha_url} is not a valid hash; skipping verification" + return 0 + fi - local jar + if command -v sha512sum >/dev/null 2>&1; then + actual="$(sha512sum "$jar" | awk '{print $1}')" + elif command -v shasum >/dev/null 2>&1; then + actual="$(shasum -a 512 "$jar" | awk '{print $1}')" + else + ui_warn "Neither sha512sum nor shasum available; skipping SHA512 verification" + return 0 + fi - ui_info "Copying Flink Agents common jar into Flink lib" - local common_copied=0 - for jar in "$common_lib_dir"/flink-agents-dist-common-*.jar; do - [[ -f "$jar" ]] || continue - cp "$jar" "$FLINK_HOME/lib/" - common_copied=1 - done - [[ "$common_copied" -eq 1 ]] || die "No flink-agents-dist-common jar found in: $common_lib_dir" + if [[ "${expected,,}" != "${actual,,}" ]]; then + die "SHA512 mismatch for $(basename "$jar") (expected ${expected:0:16}…, got ${actual:0:16}…)" + fi + ui_success "SHA512 verified" +} - ui_info "Copying Flink Agents thin jar into Flink lib" - local copied=0 - for jar in "$version_lib_dir"/flink-agents-dist-*.jar; do - [[ -f "$jar" ]] || continue - cp "$jar" "$FLINK_HOME/lib/" - copied=1 - done - [[ "$copied" -eq 1 ]] || die "No flink-agents-dist jar found in: $version_lib_dir" +# Download flink-agents-dist-flink--.jar +# directly from the ASF mirror into FLINK_HOME/lib. This replaces the older +# wheel-extraction path, so Java-only users no longer need Python at all. +install_flink_agents_jar() { + [[ -n "${FLINK_HOME:-}" ]] || die "FLINK_HOME is not set" + [[ -d "$FLINK_HOME/lib" ]] || die "Invalid FLINK_HOME (missing lib): $FLINK_HOME" + + local relpath jar_name target url sha_url + relpath="$(flink_agents_jar_relpath)" + jar_name="$(basename "$relpath")" + target="$FLINK_HOME/lib/$jar_name" + url="${FLINK_AGENTS_BASE_URL}/${relpath}" + sha_url="${FLINK_AGENTS_CHECKSUM_BASE_URL}/${relpath}.sha512" + + detect_downloader + + if [[ -f "$target" ]]; then + ui_info "Reusing existing JAR: ${target}" + verify_flink_agents_jar_sha512 "$target" "$sha_url" || true + return 0 + fi + + ui_info "Downloading ${url}" + if ! download_file "$url" "$target"; then + rm -f "$target" 2>/dev/null || true + die "Failed to download flink-agents JAR from ${url}" + fi + + if [[ ! -s "$target" ]]; then + rm -f "$target" 2>/dev/null || true + die "Downloaded an empty file: ${target}" + fi + + verify_flink_agents_jar_sha512 "$target" "$sha_url" + ui_success "Installed: ${jar_name} → ${FLINK_HOME}/lib" } check_java() { @@ -1272,6 +1330,14 @@ verify_installation() { ui_warn "Flink binary not found at $FLINK_HOME/bin/flink" fi + local expected_jar="$FLINK_HOME/lib/flink-agents-dist-flink-${FLINK_MAJOR_MINOR}-${FLINK_AGENTS_VERSION}.jar" + if [[ -f "$expected_jar" ]]; then + ui_success "flink-agents JAR found: $(basename "$expected_jar")" + else + ui_error "Expected flink-agents JAR missing: $expected_jar" + return 1 + fi + if [[ "$PYFLINK_ACTUALLY_ENABLED" -eq 1 ]]; then if python -c "import flink_agents; print('flink-agents', flink_agents.__version__)" 2>/dev/null; then ui_success "flink-agents Python package verified" @@ -1280,28 +1346,6 @@ verify_installation() { return 1 fi - local common_jar_found=0 - for jar in "$FLINK_HOME/lib"/flink-agents-dist-common-*.jar; do - [[ -f "$jar" ]] && common_jar_found=1 && break - done - if [[ "$common_jar_found" -eq 1 ]]; then - ui_success "flink-agents-dist-common JAR found in FLINK_HOME/lib" - else - ui_error "flink-agents-dist-common JAR not found in $FLINK_HOME/lib" - return 1 - fi - - local thin_jar_found=0 - for jar in "$FLINK_HOME/lib"/flink-agents-dist-flink-*-thin.jar; do - [[ -f "$jar" ]] && thin_jar_found=1 && break - done - if [[ "$thin_jar_found" -eq 1 ]]; then - ui_success "flink-agents-dist thin JAR found in FLINK_HOME/lib" - else - ui_error "flink-agents-dist thin JAR not found in $FLINK_HOME/lib" - return 1 - fi - if [[ -f "$FLINK_HOME/lib/flink-python-${FLINK_VERSION}.jar" ]]; then ui_success "flink-python JAR found in FLINK_HOME/lib" else @@ -1411,13 +1455,13 @@ main() { ui_stage "Installing Apache Flink" install_flink_if_needed - ui_stage "Setting up Python environment" + ui_stage "Installing Flink Agents" + install_flink_agents_jar if [[ "$ENABLE_PYFLINK" == "Yes" ]]; then copy_pyflink_jar setup_python_env - copy_flink_agents_jars else - ui_info "Skipping PyFlink and Python package setup (ENABLE_PYFLINK=${ENABLE_PYFLINK})." + ui_info "Skipping Python venv (ENABLE_PYFLINK=No)." fi ui_stage "Finalizing" diff --git a/tools/test/integration/install_flink_agents_jar.bats b/tools/test/integration/install_flink_agents_jar.bats new file mode 100644 index 000000000..146967cc9 --- /dev/null +++ b/tools/test/integration/install_flink_agents_jar.bats @@ -0,0 +1,98 @@ +#!/usr/bin/env bats + +setup() { + load '../helpers/load' + load '../helpers/shim' + load_install_sh + reset_install_sh_state + shim_setup + FLINK_HOME="$BATS_TEST_TMPDIR/flink-home" + mkdir -p "$FLINK_HOME/lib" + FLINK_VERSION="2.2.0" + FLINK_MAJOR_MINOR="2.2" + FLINK_AGENTS_VERSION="0.2.1" + FLINK_AGENTS_BASE_URL="https://mirror.test/flink" + FLINK_AGENTS_CHECKSUM_BASE_URL="https://downloads.test/flink" +} + +# A curl shim that writes whatever was requested at -o into a fake file +# whose body is "fake-jar-" — enough to satisfy the "non-empty" +# check, and lets us verify which URLs were requested. +configure_curl_shim() { + shim_bin_script curl ' +out=""; url=""; prev="" +for a in "$@"; do + [[ "$prev" == "-o" ]] && out="$a" + case "$a" in -*|"") ;; *) url="$a" ;; esac + prev="$a" +done +[[ -n "$out" ]] && printf "fake-content-%s" "$(basename "$out")" > "$out" +' +} + +@test "flink_agents_jar_relpath: builds expected ASF mirror path" { + FLINK_MAJOR_MINOR="2.1" + FLINK_AGENTS_VERSION="0.2.0" + run flink_agents_jar_relpath + [ "$status" -eq 0 ] + [ "$output" = "flink-agents-0.2.0/flink-agents-dist-flink-2.1-0.2.0.jar" ] +} + +@test "install_flink_agents_jar: downloads from mirror, verifies, lands in FLINK_HOME/lib" { + DOWNLOADER=curl + configure_curl_shim + + run install_flink_agents_jar + [ "$status" -eq 0 ] + [ -f "$FLINK_HOME/lib/flink-agents-dist-flink-2.2-0.2.1.jar" ] + + # Two curl calls expected: jar from mirror, sha512 sidecar from downloads.apache.org. + case "$(shim_calls curl)" in + *"https://mirror.test/flink/flink-agents-0.2.1/flink-agents-dist-flink-2.2-0.2.1.jar"*) ;; + *) false ;; + esac + case "$(shim_calls curl)" in + *"https://downloads.test/flink/flink-agents-0.2.1/flink-agents-dist-flink-2.2-0.2.1.jar.sha512"*) ;; + *) false ;; + esac +} + +@test "install_flink_agents_jar: reuses existing JAR, skips re-download" { + DOWNLOADER=curl + configure_curl_shim + # Pre-seed the target JAR. + : > "$FLINK_HOME/lib/flink-agents-dist-flink-2.2-0.2.1.jar" + + run install_flink_agents_jar + [ "$status" -eq 0 ] + # Only the sha512 sidecar should have been fetched, never the JAR itself. + case "$(shim_calls curl)" in + *"flink-agents-dist-flink-2.2-0.2.1.jar"*"flink-agents-dist-flink-2.2-0.2.1.jar"*) false ;; + *) ;; + esac +} + +@test "install_flink_agents_jar: empty downloaded file is a hard error" { + DOWNLOADER=curl + # Shim that "succeeds" but writes nothing. + shim_bin_script curl ' +out=""; prev="" +for a in "$@"; do + [[ "$prev" == "-o" ]] && out="$a" + prev="$a" +done +[[ -n "$out" ]] && : > "$out" +' + run install_flink_agents_jar + [ "$status" -ne 0 ] + case "$output" in *"empty file"*) ;; *) false ;; esac + [ ! -f "$FLINK_HOME/lib/flink-agents-dist-flink-2.2-0.2.1.jar" ] +} + +@test "install_flink_agents_jar: download failure surfaces a clean error" { + DOWNLOADER=curl + shim_bin curl 22 # curl exit 22 = HTTP error + run install_flink_agents_jar + [ "$status" -ne 0 ] + case "$output" in *"Failed to download flink-agents JAR"*) ;; *) false ;; esac +} From 36ecab7131ddaf6e34fb7651ad07c9eb5ed003cb Mon Sep 17 00:00:00 2001 From: WenjinXie Date: Wed, 20 May 2026 21:26:39 +0800 Subject: [PATCH 24/35] [tools] install.sh: split install plan into Environment + Plan sections MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reviewer feedback (#3, #5, alignment): the install plan mashed detected-environment facts (OS, Java, JAVA_HOME) together with editable configuration in one flat list, and the JAVA_HOME row implied it was something the user could pick. It also leaked editable-looking "Install directory" / "Flink version" rows when the user had said "don't install Flink". Rewrite show_install_plan(): Environment (read-only) OS: macos / linux Java: or "not found" JAVA_HOME: /path or Python: (X.Y.Z) or Installation plan Flink Agents version: … Install Flink: Yes / No ├─ Flink version: … # Yes only ├─ Install directory: … # Yes only └─ FLINK_HOME: /p (vX.Y) # No only — detected version inline Enable PyFlink: Yes / No └─ Venv directory: … # Yes only Also widen the key column: ui_kv now takes a UI_KV_KEY_WIDTH=24 and appends the colon itself in both gum and fallback branches, so "Flink Agents version" no longer collides with its value (the previous 20-char width packed key + value into "Flink Agents version0.2.1"). Five unit tests cover the Environment section being present, version+dir on Yes, FLINK_HOME-with-version on No, the JAVA_HOME unset hint, and that the new aligned format is rendered. Co-Authored-By: Claude Opus 4.7 --- tools/install.sh | 53 +++++++++++++----- tools/test/unit/show_install_plan.bats | 74 ++++++++++++++++++++++++++ 2 files changed, 113 insertions(+), 14 deletions(-) create mode 100644 tools/test/unit/show_install_plan.bats diff --git a/tools/install.sh b/tools/install.sh index 3b2f70452..763205f7c 100755 --- a/tools/install.sh +++ b/tools/install.sh @@ -329,16 +329,19 @@ ui_stage() { ui_section "[${INSTALL_STAGE_CURRENT}/${INSTALL_STAGE_TOTAL}] ${title}" } +UI_KV_KEY_WIDTH=24 + ui_kv() { local key="$1" local value="$2" + local labeled="${key}:" if [[ -n "$GUM" ]]; then local key_part value_part - key_part="$("$GUM" style --foreground "#5a6480" --width 20 "$key")" + key_part="$("$GUM" style --foreground "#5a6480" --width "$UI_KV_KEY_WIDTH" "$labeled")" value_part="$("$GUM" style --bold "$value")" "$GUM" join --horizontal "$key_part" "$value_part" else - echo -e "${MUTED}${key}:${NC} ${value}" + printf "${MUTED}%-${UI_KV_KEY_WIDTH}s${NC} %s\n" "$labeled" "$value" fi } @@ -1284,24 +1287,46 @@ resolve_python() { } show_install_plan() { - ui_section "Installation plan" + ui_section "Environment (read-only)" ui_kv "OS" "$OS" - ui_kv "Flink version" "$FLINK_VERSION" + local java_summary="not found" + if command -v java >/dev/null 2>&1; then + local jv + jv="$(java -version 2>&1 | head -n1 | sed -E 's/^[^ ]+ version "?([^"]+)"?.*/\1/')" + java_summary="${jv:-detected}" + fi + ui_kv "Java" "$java_summary" + if [[ -n "${JAVA_HOME:-}" ]]; then + ui_kv "JAVA_HOME" "$JAVA_HOME" + else + ui_kv "JAVA_HOME" "" + fi + local python_summary="" + if [[ -n "$PYTHON_BIN" ]] && command -v "$PYTHON_BIN" >/dev/null 2>&1; then + local pv + pv="$("$PYTHON_BIN" -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}")' 2>/dev/null || true)" + python_summary="${PYTHON_BIN}${pv:+ ($pv)}" + elif command -v python3 >/dev/null 2>&1; then + local pv + pv="$(python3 -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}")' 2>/dev/null || true)" + python_summary="python3${pv:+ ($pv)}" + fi + ui_kv "Python" "$python_summary" + + ui_section "Installation plan" ui_kv "Flink Agents version" "$FLINK_AGENTS_VERSION" ui_kv "Install Flink" "$INSTALL_FLINK" - ui_kv "Install directory" "$INSTALL_DIR" + if [[ "$INSTALL_FLINK" == "Yes" ]]; then + ui_kv "Flink version" "$FLINK_VERSION" + ui_kv "Install directory" "$INSTALL_DIR" + else + if [[ -n "${FLINK_HOME:-}" ]]; then + ui_kv "FLINK_HOME" "$FLINK_HOME (v$FLINK_VERSION)" + fi + fi ui_kv "Enable PyFlink" "$ENABLE_PYFLINK" if [[ "$ENABLE_PYFLINK" == "Yes" ]] || [[ "$PYFLINK_ACTUALLY_ENABLED" -eq 1 ]]; then ui_kv "Venv directory" "$VENV_DIR" - if [[ -n "$PYTHON_BIN" ]]; then - ui_kv "Python interpreter" "$PYTHON_BIN" - fi - fi - if [[ -n "${FLINK_HOME:-}" ]]; then - ui_kv "FLINK_HOME" "$FLINK_HOME" - fi - if [[ -n "${JAVA_HOME:-}" ]]; then - ui_kv "JAVA_HOME" "$JAVA_HOME" fi if [[ "$DRY_RUN" == "1" ]]; then ui_kv "Dry run" "yes" diff --git a/tools/test/unit/show_install_plan.bats b/tools/test/unit/show_install_plan.bats new file mode 100644 index 000000000..0a304a2f7 --- /dev/null +++ b/tools/test/unit/show_install_plan.bats @@ -0,0 +1,74 @@ +#!/usr/bin/env bats + +setup() { + load '../helpers/load' + load_install_sh + reset_install_sh_state + OS="linux" +} + +@test "show_install_plan: Environment section always present" { + INSTALL_FLINK=Yes + INSTALL_DIR="/opt/flink" + FLINK_HOME="/opt/flink/flink-2.2.0" + run show_install_plan + [ "$status" -eq 0 ] + case "$output" in *"Environment (read-only)"*) ;; *) false ;; esac + case "$output" in *"OS:"*) ;; *) false ;; esac + case "$output" in *"Java:"*) ;; *) false ;; esac + case "$output" in *"JAVA_HOME:"*) ;; *) false ;; esac + case "$output" in *"Python:"*) ;; *) false ;; esac +} + +@test "show_install_plan: INSTALL_FLINK=Yes shows version + install directory, not FLINK_HOME explicit" { + INSTALL_FLINK=Yes + INSTALL_DIR="/opt/flink" + FLINK_VERSION="2.2.0" + FLINK_HOME="/opt/flink/flink-2.2.0" + run show_install_plan + [ "$status" -eq 0 ] + case "$output" in *"Flink version"*) ;; *) false ;; esac + case "$output" in *"Install directory"*) ;; *) false ;; esac +} + +@test "show_install_plan: INSTALL_FLINK=No hides Install directory, shows FLINK_HOME with version" { + INSTALL_FLINK=No + INSTALL_DIR="/should/not/show" + FLINK_VERSION="2.1.1" + FLINK_HOME="/usr/local/flink" + run show_install_plan + [ "$status" -eq 0 ] + case "$output" in *"FLINK_HOME"*) ;; *) false ;; esac + case "$output" in *"v2.1.1"*) ;; *) false ;; esac + case "$output" in *"Install directory"*) false ;; *) ;; esac + case "$output" in *"/should/not/show"*) false ;; *) ;; esac +} + +@test "show_install_plan: JAVA_HOME unset shows hint" { + INSTALL_FLINK=Yes + INSTALL_DIR="/opt/flink" + FLINK_HOME="/opt/flink/flink-2.2.0" + unset JAVA_HOME + run show_install_plan + [ "$status" -eq 0 ] + case "$output" in *""*) ;; *) false ;; esac +} + +@test "show_install_plan: Flink Agents version line is present" { + INSTALL_FLINK=Yes + INSTALL_DIR="/opt/flink" + FLINK_HOME="/opt/flink/flink-2.2.0" + FLINK_AGENTS_VERSION="0.2.1" + run show_install_plan + [ "$status" -eq 0 ] + case "$output" in *"Flink Agents version"*"0.2.1"*) ;; *) false ;; esac +} + +@test "show_install_plan: no redundant 'Install flink-agents JARs' row" { + INSTALL_FLINK=Yes + INSTALL_DIR="/opt/flink" + FLINK_HOME="/opt/flink/flink-2.2.0" + run show_install_plan + [ "$status" -eq 0 ] + case "$output" in *"Install flink-agents JARs"*) false ;; *) ;; esac +} From 9be4ff3d42f20afa231bda7fade8416ff5675584 Mon Sep 17 00:00:00 2001 From: WenjinXie Date: Wed, 20 May 2026 21:29:55 +0800 Subject: [PATCH 25/35] [tools] install.sh: add Flink Agents version picker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reviewer feedback: "I didn't see where to choose which flink-agents version to install." Only FLINK_VERSION had an interactive picker; FLINK_AGENTS_VERSION was hardcoded. Add a parallel picker that mirrors https://flink.apache.org/downloads/#apache-flink-agents: FLINK_AGENTS_SUPPORTED_VERSIONS=("0.2.1" "0.2.0" "0.1.1" "0.1.0") FLINK_AGENTS_RECOMMENDED_VERSION="0.2.1" - prompt_flink_agents_version_interactive() — same UX as the Flink version picker (gum choose + numbered-menu fallback, "(recommended)" marker). - plan_flink_agents() runs the picker if the version wasn't supplied explicitly; called from main right after plan_flink so users see it as part of "Planning Flink installation". - mark_explicit FLINK_AGENTS_VERSION + --flink-agents-version flag (also adds --flink-version which was missing too) so non-interactive invocations and env vars opt out of the picker. - Updated print_usage with both flags and reset_install_sh_state with the supported-versions array so tests stay deterministic. Integration test (dry-run plan) confirms "Flink Agents version: 0.2.1" appears in the plan output. Co-Authored-By: Claude Opus 4.7 --- tools/install.sh | 94 +++++++++++++++++++++++ tools/test/helpers/load.bash | 3 + tools/test/integration/dry_run_extra.bats | 42 ++++++++++ 3 files changed, 139 insertions(+) create mode 100644 tools/test/integration/dry_run_extra.bats diff --git a/tools/install.sh b/tools/install.sh index 763205f7c..acaeffbc0 100755 --- a/tools/install.sh +++ b/tools/install.sh @@ -364,6 +364,7 @@ mark_explicit() { } mark_explicit FLINK_VERSION +mark_explicit FLINK_AGENTS_VERSION mark_explicit INSTALL_DIR mark_explicit VENV_DIR @@ -381,6 +382,13 @@ FLINK_AGENTS_CHECKSUM_BASE_URL="${FLINK_AGENTS_CHECKSUM_BASE_URL:-https://downlo FLINK_SUPPORTED_VERSIONS=("2.2.0" "2.1.1" "2.0.1" "1.20.3") FLINK_RECOMMENDED_VERSION="2.2.0" +# Mirrors https://flink.apache.org/downloads/#apache-flink-agents +# (latest first). Note: 0.1.x only ships JARs for Flink 1.20, while 0.2.x +# ships JARs for Flink 1.20 / 2.0 / 2.1 / 2.2 — see the download page for +# the exact compatibility matrix. +FLINK_AGENTS_SUPPORTED_VERSIONS=("0.2.1" "0.2.0" "0.1.1" "0.1.0") +FLINK_AGENTS_RECOMMENDED_VERSION="0.2.1" + INSTALL_FLINK="${INSTALL_FLINK:-Ask}" ENABLE_PYFLINK="${ENABLE_PYFLINK:-Ask}" INSTALL_DIR="${INSTALL_DIR:-$HOME/.local/flink}" @@ -409,6 +417,9 @@ Options: --dry-run Print install plan without making changes --verify Run post-install verification checks --python Path to a Python3 interpreter (overrides PATH lookup) + --flink-version Apache Flink version (e.g. 2.2.0); overrides the interactive picker + --flink-agents-version + Flink Agents version (default: ${FLINK_AGENTS_RECOMMENDED_VERSION}); overrides the picker --help, -h Show this help Environment variables: @@ -563,6 +574,56 @@ prompt_flink_version_interactive() { return 0 } +prompt_flink_agents_version_interactive() { + if ! is_promptable; then + return 1 + fi + + local labels=() + local v + for v in "${FLINK_AGENTS_SUPPORTED_VERSIONS[@]}"; do + if [[ "$v" == "$FLINK_AGENTS_RECOMMENDED_VERSION" ]]; then + labels+=("$v (recommended)") + else + labels+=("$v") + fi + done + + local selection="" + if [[ -n "$GUM" ]] && gum_is_tty; then + local _rc=0 + selection="$("$GUM" choose \ + --header "Select Flink Agents version" \ + --cursor-prefix "❯ " \ + "${labels[@]}" < /dev/tty)" || _rc=$? + (( _rc == 0 )) || die_cancelled + selection="${selection%% *}" + else + printf 'Select Flink Agents version:\n' > /dev/tty + local i=1 + for v in "${FLINK_AGENTS_SUPPORTED_VERSIONS[@]}"; do + local suffix="" + [[ "$v" == "$FLINK_AGENTS_RECOMMENDED_VERSION" ]] && suffix=" (recommended)" + printf ' %d) %s%s\n' "$i" "$v" "$suffix" > /dev/tty + i=$((i+1)) + done + local answer="" + printf 'Enter choice [1-%d, default %s]: ' \ + "${#FLINK_AGENTS_SUPPORTED_VERSIONS[@]}" "$FLINK_AGENTS_RECOMMENDED_VERSION" > /dev/tty + read -r answer < /dev/tty || die_cancelled + if [[ "$answer" =~ ^[0-9]+$ ]] \ + && (( answer >= 1 && answer <= ${#FLINK_AGENTS_SUPPORTED_VERSIONS[@]} )); then + selection="${FLINK_AGENTS_SUPPORTED_VERSIONS[$((answer-1))]}" + fi + fi + + if [[ -z "$selection" ]]; then + selection="$FLINK_AGENTS_RECOMMENDED_VERSION" + fi + FLINK_AGENTS_VERSION="$selection" + return 0 +} + # Populate FLINK_VERSION from an existing FLINK_HOME. Tries to parse # `lib/flink-dist-.jar` first because it's instant; falls back to # `bin/flink --version`, which is authoritative but spins up a JVM and can @@ -660,6 +721,12 @@ plan_flink() { FLINK_MAJOR_MINOR="${FLINK_VERSION%.*}" } +plan_flink_agents() { + if [[ "$FLINK_AGENTS_VERSION_EXPLICIT" -eq 0 ]]; then + prompt_flink_agents_version_interactive || true + fi +} + plan_pyflink() { case "$ENABLE_PYFLINK" in Yes|No) @@ -1427,6 +1494,32 @@ parse_args() { PYTHON_BIN="${1#*=}" shift ;; + --flink-agents-version) + if [[ $# -lt 2 ]]; then + die "--flink-agents-version requires a version argument" + fi + FLINK_AGENTS_VERSION="$2" + FLINK_AGENTS_VERSION_EXPLICIT=1 + shift 2 + ;; + --flink-agents-version=*) + FLINK_AGENTS_VERSION="${1#*=}" + FLINK_AGENTS_VERSION_EXPLICIT=1 + shift + ;; + --flink-version) + if [[ $# -lt 2 ]]; then + die "--flink-version requires a version argument" + fi + FLINK_VERSION="$2" + FLINK_VERSION_EXPLICIT=1 + shift 2 + ;; + --flink-version=*) + FLINK_VERSION="${1#*=}" + FLINK_VERSION_EXPLICIT=1 + shift + ;; --help|-h) HELP=1 shift @@ -1464,6 +1557,7 @@ main() { ui_stage "Planning Flink installation" plan_flink + plan_flink_agents ui_stage "Planning Python environment" plan_pyflink diff --git a/tools/test/helpers/load.bash b/tools/test/helpers/load.bash index b46e2b707..dc2f3a1ae 100644 --- a/tools/test/helpers/load.bash +++ b/tools/test/helpers/load.bash @@ -39,6 +39,9 @@ reset_install_sh_state() { VENV_DIR=".flink-agents-env" GUM_VERSION="0.17.0" FLINK_VERSION_EXPLICIT=0 + FLINK_AGENTS_VERSION_EXPLICIT=0 INSTALL_DIR_EXPLICIT=0 VENV_DIR_EXPLICIT=0 + FLINK_AGENTS_SUPPORTED_VERSIONS=("0.2.1" "0.2.0" "0.1.1" "0.1.0") + FLINK_AGENTS_RECOMMENDED_VERSION="0.2.1" } diff --git a/tools/test/integration/dry_run_extra.bats b/tools/test/integration/dry_run_extra.bats new file mode 100644 index 000000000..1e7efac05 --- /dev/null +++ b/tools/test/integration/dry_run_extra.bats @@ -0,0 +1,42 @@ +#!/usr/bin/env bats + +setup() { + load '../helpers/load' + load '../helpers/shim' + shim_setup + export FLINK_HOME="$BATS_TEST_TMPDIR/flink-home" + mkdir -p "$FLINK_HOME/lib" "$FLINK_HOME/bin" + # Seed a fake flink-dist jar so detect_flink_version_from_home succeeds + # and the plan reflects the on-disk version (review feedback #2 + #3). + : > "$FLINK_HOME/lib/flink-dist-2.1.1.jar" + shim_bin_script java " +case \"\$1\" in + -version) echo 'openjdk version \"17.0.2\"' >&2 ;; +esac +" +} + +@test "dry-run: plan shows Flink Agents version (review #1 — JARs are implicit)" { + run bash "${BATS_TEST_DIRNAME}/../../install.sh" --dry-run --non-interactive + [ "$status" -eq 0 ] + case "$output" in *"Flink Agents version"*"0.2.1"*) ;; *) false ;; esac +} + +@test "dry-run: existing FLINK_HOME — plan shows detected version, not default (review #2)" { + run bash "${BATS_TEST_DIRNAME}/../../install.sh" --dry-run --non-interactive + [ "$status" -eq 0 ] + case "$output" in *"v2.1.1"*) ;; *) false ;; esac +} + +@test "dry-run: INSTALL_FLINK=No suppresses Install directory line (review #3)" { + run bash "${BATS_TEST_DIRNAME}/../../install.sh" --dry-run --non-interactive + [ "$status" -eq 0 ] + case "$output" in *"Install directory"*) false ;; *) ;; esac +} + +@test "dry-run: Environment section is shown separately from Plan (review #5)" { + run bash "${BATS_TEST_DIRNAME}/../../install.sh" --dry-run --non-interactive + [ "$status" -eq 0 ] + case "$output" in *"Environment (read-only)"*) ;; *) false ;; esac + case "$output" in *"Installation plan"*) ;; *) false ;; esac +} From 6cf7bce9eeebe341ab63633d5ab7aca509197811 Mon Sep 17 00:00:00 2001 From: WenjinXie Date: Wed, 20 May 2026 21:32:29 +0800 Subject: [PATCH 26/35] [tools] install.sh: enable readline tab completion on path inputs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reviewer feedback (#6): path entry had no autocomplete in either branch — gum input is a plain text box and the fallback used plain \`read -r\`, so users typing FLINK_HOME / INSTALL_DIR / VENV_DIR couldn't Tab-complete. Replace both with \`read -e -r -p\` so readline's filename completion is available everywhere. The gum input box is dropped (it offered no value beyond the visual style and blocked completion); gum choose is still used for menu selections. The previous "placeholder" argument is woven into the prompt label as "(example: …)" since readline doesn't surface placeholders the same way. Co-Authored-By: Claude Opus 4.7 --- tools/install.sh | 32 ++++++++++---------------------- 1 file changed, 10 insertions(+), 22 deletions(-) diff --git a/tools/install.sh b/tools/install.sh index acaeffbc0..c38293d5a 100755 --- a/tools/install.sh +++ b/tools/install.sh @@ -996,19 +996,16 @@ install_flink_if_needed() { prompt_path_input() { local header="$1" - local placeholder="$2" + # Placeholder kept for backward compatibility with the previous signature; + # readline does not surface placeholders the way `gum input` did, so we + # weave the hint into the prompt label instead. + local placeholder="${2:-}" local input="" - if [[ -n "$GUM" ]] && gum_is_tty; then - local _rc=0 - input="$("$GUM" input \ - --header "$header" \ - --placeholder "$placeholder" \ - --width 70 < /dev/tty)" || _rc=$? - (( _rc == 0 )) || die_cancelled - else - printf '%s: ' "$header" > /dev/tty - read -r input < /dev/tty || die_cancelled + printf '%s\n' "$header" > /dev/tty + if [[ -n "$placeholder" ]]; then + printf ' (example: %s)\n' "$placeholder" > /dev/tty fi + IFS= read -e -r -p " path> " input < /dev/tty || die_cancelled printf '%s' "$(normalize_path "$input")" } @@ -1052,17 +1049,8 @@ prompt_path_choice_interactive() { fi local input="" - if [[ -n "$GUM" ]] && gum_is_tty; then - local _rc=0 - input="$("$GUM" input \ - --header "$header" \ - --placeholder "$placeholder" \ - --width 70 < /dev/tty)" || _rc=$? - (( _rc == 0 )) || die_cancelled - else - printf 'Enter custom path: ' > /dev/tty - read -r input < /dev/tty || die_cancelled - fi + printf '%s\n' "$header" > /dev/tty + IFS= read -e -r -p " path> " input < /dev/tty || die_cancelled if [[ -z "$input" ]]; then ui_warn "Empty path; falling back to default: $default_path" From 3f4e81f936b26462af09d5475eaff9e59b4c1b72 Mon Sep 17 00:00:00 2001 From: WenjinXie Date: Wed, 20 May 2026 21:33:52 +0800 Subject: [PATCH 27/35] [tools] install.sh: rewrite finish-line hints as a numbered Next steps block MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reviewer feedback (#7a, venv hint visibility): the closing "export FLINK_HOME=..." line was opaque to first-time users — they couldn't tell why they needed to export it, and the venv activate hint was an afterthought tucked below it. Replace it with a "Next steps" section, numbered, where the venv hint is a peer of the FLINK_HOME export instead of being buried. Each step uses ui_success for the rationale and ui_info for the literal command so the copy/paste line is visually separable from the prose. The "make it permanent" guidance only mentions one or two rc lines depending on whether the venv is actually installed. Co-Authored-By: Claude Opus 4.7 --- tools/install.sh | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/tools/install.sh b/tools/install.sh index c38293d5a..7420b4f43 100755 --- a/tools/install.sh +++ b/tools/install.sh @@ -1577,12 +1577,19 @@ main() { echo "" ui_celebrate "Apache Flink Agents installation finished!" - ui_success "FLINK_HOME=$FLINK_HOME" + echo "" + ui_section "Next steps" + ui_success "1) Point FLINK_HOME at this install (Flink CLI and clients read it):" + ui_info " export FLINK_HOME=${FLINK_HOME}" if [[ "$PYFLINK_ACTUALLY_ENABLED" -eq 1 ]]; then - ui_info "To use Python environment in a new shell :" - ui_info " source ${VENV_DIR}/bin/activate" + ui_success "2) Activate the Python venv (PyFlink + flink-agents are installed there):" + ui_info " source ${VENV_DIR}/bin/activate" + ui_success "3) To make both permanent, append the two lines above to your shell rc" + ui_info " (~/.zshrc or ~/.bashrc)." + else + ui_success "2) To make it permanent, append the line above to your shell rc" + ui_info " (~/.zshrc or ~/.bashrc)." fi - ui_info " export FLINK_HOME=${FLINK_HOME}" show_footer_links } From cc035def42b69f5714f201baebdbe85d2e5849be Mon Sep 17 00:00:00 2001 From: WenjinXie Date: Wed, 20 May 2026 21:37:57 +0800 Subject: [PATCH 28/35] [tools] install.sh: cascade edit prompts and make ESC back out of any nested menu MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reviewer feedback (#4, #8, #9): toggling Install Flink / Enable PyFlink in the edit menu didn't ask the follow-up questions; ESC at the edit menu killed the whole installer; Ctrl+C during "Proceed with installation?" silently looped instead of cancelling. Cascade follow-ups: - Flipping install_flink from No → Yes immediately walks the user through Flink version + install dir. - Flipping install_flink from Yes → No prompts for FLINK_HOME and runs detect_flink_version_from_home. - Flipping enable_pyflink from No → Yes prompts for the venv dir right after resolving Python. ESC = back, at any nesting depth: - Run the entire edit body inside a `()` subshell. A die_cancelled / ESC from any nested prompt only kills the subshell. - On successful completion the subshell dumps mutated state via edit_plan_quote / edit_plan_dump_state; the parent sources it back. - On any non-zero subshell exit, the parent returns 0 ("back to confirm"); the state file is left untouched, so the user's previous plan is preserved. - The "← Back" menu item is gone — ESC IS back. - Critical detail: the subshell is wrapped with `|| rc=$?` so set -e doesn't propagate the subshell's `exit 130` and kill the parent. Ctrl+C handling: - Register `trap die_cancelled INT` once at script load so any read / gum that swallowed SIGINT still surfaces as a clean exit. - die_cancelled now writes its message to stderr, so a Ctrl+C inside `$(confirm_plan_action_interactive)` doesn't get captured into a variable and disappear. - confirm_install_plan checks the subshell exit code: action="$(...)" || exit 130 so when Ctrl+C lands inside the proceed/edit/cancel picker, the installer actually exits instead of looping forever on an empty selection. Tests: - 6 unit tests for edit_plan_quote / edit_plan_dump_state round-tripping (spaces, single quotes, $expansion, command substitution). - 3 unit tests for the subshell-with-set-e pattern: exit 130, exit 1, and exit 0 paths. Co-Authored-By: Claude Opus 4.7 --- tools/install.sh | 297 ++++++++++++++++++--------- tools/test/unit/edit_plan_back.bats | 69 +++++++ tools/test/unit/edit_plan_quote.bats | 88 ++++++++ 3 files changed, 355 insertions(+), 99 deletions(-) create mode 100644 tools/test/unit/edit_plan_back.bats create mode 100644 tools/test/unit/edit_plan_quote.bats diff --git a/tools/install.sh b/tools/install.sh index 7420b4f43..a33c619bb 100755 --- a/tools/install.sh +++ b/tools/install.sh @@ -36,6 +36,10 @@ cleanup_tmpfiles() { done } trap cleanup_tmpfiles EXIT +# Some interactive read combinations (notably `read -e` under stdin redirection) +# can swallow SIGINT, leaving the user pressing Ctrl+C with no effect. Install +# an explicit INT trap so Ctrl+C always lands. +trap 'die_cancelled' INT mktempfile() { local f @@ -306,7 +310,10 @@ die() { } die_cancelled() { - ui_info "Cancelled by user" + # Write to stderr so the message survives `$(...)` command substitution + # — otherwise the cancellation note would be silently captured by the + # caller's variable and the user would see nothing on Ctrl+C. + ui_info "Cancelled by user" >&2 exit 130 } @@ -812,117 +819,201 @@ edit_prompt_flink_home() { # Show a menu of currently-applicable plan fields and re-prompt for the # selected one. After the edit, recomputes FLINK_HOME / FLINK_MAJOR_MINOR # so the next show_install_plan reflects the change. +# Quote a value for safe re-sourcing by the parent shell via single-quoted +# assignment. Any embedded single quotes are escaped by closing/reopening +# the quoted string. +edit_plan_quote() { + local v="$1" + printf "'%s'" "${v//\'/\'\\\'\'}" +} + +# Write the subset of plan variables we may have modified to a sourceable +# state file. Called from inside the edit subshell on a successful action. +edit_plan_dump_state() { + local out="$1" + { + printf 'INSTALL_FLINK=%s\n' "$(edit_plan_quote "$INSTALL_FLINK")" + printf 'FLINK_VERSION=%s\n' "$(edit_plan_quote "$FLINK_VERSION")" + printf 'INSTALL_DIR=%s\n' "$(edit_plan_quote "$INSTALL_DIR")" + printf 'FLINK_HOME=%s\n' "$(edit_plan_quote "${FLINK_HOME:-}")" + printf 'FLINK_MAJOR_MINOR=%s\n' "$(edit_plan_quote "${FLINK_MAJOR_MINOR:-}")" + printf 'ENABLE_PYFLINK=%s\n' "$(edit_plan_quote "$ENABLE_PYFLINK")" + printf 'PYFLINK_ACTUALLY_ENABLED=%s\n' "$(edit_plan_quote "$PYFLINK_ACTUALLY_ENABLED")" + printf 'VENV_DIR=%s\n' "$(edit_plan_quote "$VENV_DIR")" + printf 'PYTHON_BIN=%s\n' "$(edit_plan_quote "${PYTHON_BIN:-}")" + printf 'FLINK_AGENTS_VERSION=%s\n' "$(edit_plan_quote "$FLINK_AGENTS_VERSION")" + } > "$out" +} + +# Show a menu of currently-applicable plan fields and re-prompt for the +# selected one. The entire body runs inside a `()` subshell so that an +# ESC anywhere — top-level menu or any nested sub-prompt — only terminates +# this menu (subshell exits 130, caller treats it as "back"). Successful +# edits dump their state to a file that the caller sources back. +# Ctrl+C is delivered to the whole process group; the parent's INT trap +# fires die_cancelled and exits the installer before we ever inspect $rc. edit_plan_interactive() { - local labels=() - local actions=() + local state_file + state_file="$(mktempfile)" - labels+=("Install Flink: $INSTALL_FLINK") - actions+=("install_flink") + # `set -e` is active, so we MUST catch any non-zero exit from the + # subshell with `|| rc=$?` — otherwise the script would terminate the + # moment a sub-prompt's ESC bubbles `exit 130` up through the subshell, + # and the parent would never get to interpret it as "back". + local rc=0 + ( + local labels=() + local actions=() - if [[ "$INSTALL_FLINK" == "Yes" ]]; then - labels+=("Flink version: $FLINK_VERSION") - actions+=("flink_version") - labels+=("Install directory: $INSTALL_DIR") - actions+=("install_dir") - else - labels+=("FLINK_HOME: ${FLINK_HOME:-}") - actions+=("flink_home") - fi + labels+=("Flink Agents version: $FLINK_AGENTS_VERSION") + actions+=("flink_agents_version") - labels+=("Enable PyFlink: $ENABLE_PYFLINK") - actions+=("enable_pyflink") + labels+=("Install Flink: $INSTALL_FLINK") + actions+=("install_flink") - if [[ "$ENABLE_PYFLINK" == "Yes" ]]; then - labels+=("Venv directory: $VENV_DIR") - actions+=("venv_dir") - fi + if [[ "$INSTALL_FLINK" == "Yes" ]]; then + labels+=("Flink version: $FLINK_VERSION") + actions+=("flink_version") + labels+=("Install directory: $INSTALL_DIR") + actions+=("install_dir") + else + labels+=("FLINK_HOME: ${FLINK_HOME:-}") + actions+=("flink_home") + fi - labels+=("Back") - actions+=("back") + labels+=("Enable PyFlink: $ENABLE_PYFLINK") + actions+=("enable_pyflink") - local picked_index=-1 - if [[ -n "$GUM" ]] && gum_is_tty; then - local _rc=0 - local selected="" - selected="$("$GUM" choose \ - --header "Edit a setting" \ - --cursor-prefix "❯ " \ - "${labels[@]}" < /dev/tty)" || _rc=$? - (( _rc == 0 )) || die_cancelled - local i=0 - for l in "${labels[@]}"; do - if [[ "$l" == "$selected" ]]; then - picked_index=$i - break + if [[ "$ENABLE_PYFLINK" == "Yes" ]]; then + labels+=("Venv directory: $VENV_DIR") + actions+=("venv_dir") + fi + + local picked_index=-1 + if [[ -n "$GUM" ]] && gum_is_tty; then + local _rc=0 + local selected="" + selected="$("$GUM" choose \ + --header "Edit a setting (↑/↓ navigate · Enter select · Esc to go back)" \ + --cursor-prefix "❯ " \ + "${labels[@]}" < /dev/tty)" || _rc=$? + # ESC inside this menu — exit the subshell so the caller treats + # it as "back to confirm". The exit code we use here (130) is + # what `gum` returns on Esc; we just propagate it. + if (( _rc != 0 )); then + exit "$_rc" + fi + local i=0 + for l in "${labels[@]}"; do + if [[ "$l" == "$selected" ]]; then + picked_index=$i + break + fi + i=$((i+1)) + done + else + printf 'Edit a setting (blank or "b" to go back):\n' > /dev/tty + local i=1 + for l in "${labels[@]}"; do + printf ' %d) %s\n' "$i" "$l" > /dev/tty + i=$((i+1)) + done + local answer="" + printf 'Enter choice [1-%d]: ' "${#labels[@]}" > /dev/tty + read -r answer < /dev/tty || die_cancelled + case "$answer" in + ""|b|B|back|0) exit 130 ;; + esac + if [[ "$answer" =~ ^[0-9]+$ ]] && (( answer >= 1 && answer <= ${#labels[@]} )); then + picked_index=$((answer - 1)) fi - i=$((i+1)) - done - else - printf 'Edit a setting:\n' > /dev/tty - local i=1 - for l in "${labels[@]}"; do - printf ' %d) %s\n' "$i" "$l" > /dev/tty - i=$((i+1)) - done - local answer="" - printf 'Enter choice [1-%d]: ' "${#labels[@]}" > /dev/tty - read -r answer < /dev/tty || die_cancelled - if [[ "$answer" =~ ^[0-9]+$ ]] && (( answer >= 1 && answer <= ${#labels[@]} )); then - picked_index=$((answer - 1)) fi - fi - (( picked_index < 0 )) && return 0 + if (( picked_index < 0 )); then + # No valid selection — same as back. + exit 130 + fi - local action="${actions[$picked_index]}" - case "$action" in - install_flink) - if choose_install_method_interactive "Install Flink?"; then - INSTALL_FLINK=Yes + local action="${actions[$picked_index]}" + case "$action" in + flink_agents_version) + prompt_flink_agents_version_interactive || true + ;; + install_flink) + if choose_install_method_interactive "Install Flink?"; then + if [[ "$INSTALL_FLINK" != "Yes" ]]; then + INSTALL_FLINK=Yes + # Coming from No → Yes: caller had no version/dir, so + # walk them through both immediately instead of + # forcing two more menu trips. + prompt_flink_version_interactive || true + INSTALL_DIR="$(prompt_path_choice_interactive \ + "Choose Flink install directory" \ + "$INSTALL_DIR" \ + "/path/to/flink-install-dir")" + INSTALL_DIR="$(normalize_path "$INSTALL_DIR")" + fi + FLINK_HOME="${INSTALL_DIR}/flink-${FLINK_VERSION}" + else + INSTALL_FLINK=No + edit_prompt_flink_home + detect_flink_version_from_home || ui_warn "Could not auto-detect Flink version; keeping ${FLINK_VERSION}" + fi + FLINK_MAJOR_MINOR="${FLINK_VERSION%.*}" + ;; + flink_version) + prompt_flink_version_interactive || true FLINK_HOME="${INSTALL_DIR}/flink-${FLINK_VERSION}" - else - INSTALL_FLINK=No + FLINK_MAJOR_MINOR="${FLINK_VERSION%.*}" + ;; + install_dir) + INSTALL_DIR="$(prompt_path_choice_interactive \ + "Choose Flink install directory" \ + "$INSTALL_DIR" \ + "/path/to/flink-install-dir")" + INSTALL_DIR="$(normalize_path "$INSTALL_DIR")" + FLINK_HOME="${INSTALL_DIR}/flink-${FLINK_VERSION}" + ;; + flink_home) edit_prompt_flink_home - fi - FLINK_MAJOR_MINOR="${FLINK_VERSION%.*}" - ;; - flink_version) - prompt_flink_version_interactive || true - FLINK_HOME="${INSTALL_DIR}/flink-${FLINK_VERSION}" - FLINK_MAJOR_MINOR="${FLINK_VERSION%.*}" - ;; - install_dir) - INSTALL_DIR="$(prompt_path_choice_interactive \ - "Choose Flink install directory" \ - "$INSTALL_DIR" \ - "/path/to/flink-install-dir")" - INSTALL_DIR="$(normalize_path "$INSTALL_DIR")" - FLINK_HOME="${INSTALL_DIR}/flink-${FLINK_VERSION}" - ;; - flink_home) - edit_prompt_flink_home - ;; - enable_pyflink) - if choose_install_method_interactive "Enable PyFlink?"; then - if [[ "$ENABLE_PYFLINK" != "Yes" ]]; then - ENABLE_PYFLINK=Yes - PYFLINK_ACTUALLY_ENABLED=1 - resolve_python + ;; + enable_pyflink) + if choose_install_method_interactive "Enable PyFlink?"; then + if [[ "$ENABLE_PYFLINK" != "Yes" ]]; then + ENABLE_PYFLINK=Yes + PYFLINK_ACTUALLY_ENABLED=1 + resolve_python + VENV_DIR="$(prompt_path_choice_interactive \ + "Choose Python venv directory" \ + "$VENV_DIR" \ + "/path/to/venv")" + VENV_DIR="$(normalize_path "$VENV_DIR")" + fi + else + ENABLE_PYFLINK=No fi - else - ENABLE_PYFLINK=No - fi - ;; - venv_dir) - VENV_DIR="$(prompt_path_choice_interactive \ - "Choose Python venv directory" \ - "$VENV_DIR" \ - "/path/to/venv")" - VENV_DIR="$(normalize_path "$VENV_DIR")" - ;; - back|*) - ;; - esac + ;; + venv_dir) + VENV_DIR="$(prompt_path_choice_interactive \ + "Choose Python venv directory" \ + "$VENV_DIR" \ + "/path/to/venv")" + VENV_DIR="$(normalize_path "$VENV_DIR")" + ;; + esac + + edit_plan_dump_state "$state_file" + ) || rc=$? + + if (( rc != 0 )); then + # Any non-zero exit from the subshell means we never reached the + # state dump — treat it as "back to confirm". (Ctrl+C would have + # killed the parent before this point via the INT trap.) + return 0 + fi + + # shellcheck disable=SC1090 + source "$state_file" } confirm_install_plan() { @@ -930,7 +1021,15 @@ confirm_install_plan() { return 0 fi while true; do - case "$(confirm_plan_action_interactive)" in + local action + # Capture stdout AND honor the child's exit code. The child runs in a + # command-substitution subshell, which does NOT inherit the parent's + # `trap die_cancelled INT`. When Ctrl+C lands inside `gum choose`, + # the subshell exits 130 but its stdout is empty — without this `||`, + # the case-statement falls through to the default arm and we'd loop + # forever re-prompting instead of exiting. + action="$(confirm_plan_action_interactive)" || exit 130 + case "$action" in confirm) return 0 ;; edit) edit_plan_interactive diff --git a/tools/test/unit/edit_plan_back.bats b/tools/test/unit/edit_plan_back.bats new file mode 100644 index 000000000..40c4796bb --- /dev/null +++ b/tools/test/unit/edit_plan_back.bats @@ -0,0 +1,69 @@ +#!/usr/bin/env bats + +setup() { + load '../helpers/load' + load_install_sh + reset_install_sh_state +} + +# When a sub-prompt's ESC propagates `exit 130` out of edit_plan_interactive's +# subshell, the surrounding `set -e` MUST NOT kill the installer. The wrapper +# should swallow the non-zero exit and return 0 ("back to confirm"). + +@test "edit_plan_interactive: subshell exit 130 is treated as back, not as installer kill" { + # Replace edit_plan_interactive with a stand-in that runs a subshell + # which exits 130 the same way the real ESC path does, and uses the + # exact `|| rc=$?` pattern we ship. + fake_edit() { + local rc=0 + ( + exit 130 + ) || rc=$? + if (( rc != 0 )); then + return 0 + fi + return 99 # we should never get here + } + + # If `set -e` were still propagating the 130, the test process would + # die here. Reaching the assertion means the pattern works. + run fake_edit + [ "$status" -eq 0 ] +} + +@test "edit_plan_interactive: subshell exit 1 (gum ESC) also becomes back" { + fake_edit() { + local rc=0 + ( exit 1 ) || rc=$? + if (( rc != 0 )); then + return 0 + fi + return 99 + } + run fake_edit + [ "$status" -eq 0 ] +} + +@test "edit_plan_interactive: subshell exit 0 sources state and propagates changes" { + local state_file="$BATS_TEST_TMPDIR/state" + cat > "$state_file" < Date: Wed, 20 May 2026 21:41:17 +0800 Subject: [PATCH 29/35] [tools] install.sh: persist auto-downloaded gum across runs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reviewer feedback: rerunning the installer always printed "Installing gum v0.17.0, please wait..." because the bootstrap path downloaded gum into a mktemp dir, used it, then EXIT-cleanup removed it. Every rerun paid the download cost again. Cache the verified gum binary under a stable, versioned path: GUM_CACHE_ROOT="${FLINK_AGENTS_GUM_CACHE_DIR:-${XDG_CACHE_HOME:-$HOME/.cache}/flink-agents/gum}" cache_dir="$GUM_CACHE_ROOT/$GUM_VERSION" cached_gum="$cache_dir/gum" Lookup order in bootstrap_gum_temp: 1. system gum on PATH -> GUM_STATUS=found 2. cached binary at $cached_gum -> GUM_STATUS=cached (new) 3. download + verify + promote into cache -> GUM_STATUS=installed A new status "cached" surfaces a friendly "gum loaded from cache" line in print_gum_status. If promoting into the cache fails (read-only HOME, etc.) the bootstrap falls back to using the freshly extracted binary for the current run and warns once via GUM_REASON — the existing TMPFILES cleanup still handles it, costing only a re-download next run. Tests: - reset_install_sh_state defaults GUM_CACHE_ROOT to BATS_TEST_TMPDIR so we never write to a developer's real $HOME/.cache during local runs. - new "cached binary on disk is reused" test confirms no curl/wget call happens when the cache is pre-seeded. - new "successful install promotes binary into the cache" test confirms the extracted binary lands at the stable cache path and GUM points at it. Co-Authored-By: Claude Opus 4.7 --- tools/install.sh | 38 +++++++++++++--- tools/test/helpers/load.bash | 3 ++ .../test/integration/bootstrap_gum_temp.bats | 44 +++++++++++++++++++ 3 files changed, 78 insertions(+), 7 deletions(-) diff --git a/tools/install.sh b/tools/install.sh index a33c619bb..d2b95c3e9 100755 --- a/tools/install.sh +++ b/tools/install.sh @@ -79,6 +79,9 @@ GUM_VERSION="${FLINK_AGENTS_GUM_VERSION:-0.17.0}" GUM="" GUM_STATUS="skipped" GUM_REASON="" +# Persistent cache for the auto-downloaded gum binary so reruns don't +# re-download it. Honors XDG_CACHE_HOME, falls back to ~/.cache. +GUM_CACHE_ROOT="${FLINK_AGENTS_GUM_CACHE_DIR:-${XDG_CACHE_HOME:-$HOME/.cache}/flink-agents/gum}" is_non_interactive_shell() { if [[ "${NO_PROMPT:-0}" == "1" ]]; then @@ -160,12 +163,21 @@ bootstrap_gum_temp() { return 0 fi + local cache_dir="${GUM_CACHE_ROOT}/${GUM_VERSION}" + local cached_gum="${cache_dir}/gum" + if [[ -x "$cached_gum" ]]; then + GUM="$cached_gum" + GUM_STATUS="cached" + GUM_REASON="reused from ${cache_dir}" + return 0 + fi + if ! command -v tar >/dev/null 2>&1; then GUM_REASON="tar not found" return 1 fi - local os arch asset base gum_tmpdir gum_path + local os arch asset base gum_tmpdir extracted_path os="$(gum_detect_os)" arch="$(gum_detect_arch)" if [[ "$os" == "unsupported" || "$arch" == "unknown" ]]; then @@ -201,21 +213,30 @@ bootstrap_gum_temp() { return 1 fi - gum_path="$(find "$gum_tmpdir" -type f -name gum 2>/dev/null | head -n1 || true)" - if [[ -z "$gum_path" ]]; then + extracted_path="$(find "$gum_tmpdir" -type f -name gum 2>/dev/null | head -n1 || true)" + if [[ -z "$extracted_path" ]]; then GUM_REASON="gum binary missing after extract" return 1 fi - chmod +x "$gum_path" >/dev/null 2>&1 || true - if [[ ! -x "$gum_path" ]]; then + chmod +x "$extracted_path" >/dev/null 2>&1 || true + if [[ ! -x "$extracted_path" ]]; then GUM_REASON="gum binary is not executable" return 1 fi - GUM="$gum_path" + # Promote into the persistent cache so subsequent runs skip the download. + if mkdir -p "$cache_dir" 2>/dev/null && mv "$extracted_path" "$cached_gum" 2>/dev/null; then + GUM="$cached_gum" + GUM_REASON="cached at ${cache_dir}" + else + # Cache write failed (read-only HOME etc.) — fall back to using the + # extracted binary directly. It'll get cleaned up at EXIT, costing a + # re-download next run, but the current run still works. + GUM="$extracted_path" + GUM_REASON="temp, verified (cache unavailable)" + fi GUM_STATUS="installed" - GUM_REASON="temp, verified" return 0 } @@ -224,6 +245,9 @@ print_gum_status() { found) ui_success "gum available (${GUM_REASON})" ;; + cached) + ui_success "gum loaded from cache (v${GUM_VERSION})" + ;; installed) ui_success "gum bootstrapped (${GUM_REASON}, v${GUM_VERSION})" ;; diff --git a/tools/test/helpers/load.bash b/tools/test/helpers/load.bash index dc2f3a1ae..308d450da 100644 --- a/tools/test/helpers/load.bash +++ b/tools/test/helpers/load.bash @@ -38,6 +38,9 @@ reset_install_sh_state() { INSTALL_DIR="$HOME/.local/flink" VENV_DIR=".flink-agents-env" GUM_VERSION="0.17.0" + # Default the bootstrap cache to a per-test directory so we never touch + # the developer's real $HOME/.cache when running locally. + GUM_CACHE_ROOT="${BATS_TEST_TMPDIR:-/tmp}/gum-cache" FLINK_VERSION_EXPLICIT=0 FLINK_AGENTS_VERSION_EXPLICIT=0 INSTALL_DIR_EXPLICIT=0 diff --git a/tools/test/integration/bootstrap_gum_temp.bats b/tools/test/integration/bootstrap_gum_temp.bats index 43fdbb2ae..c35ab9ffd 100644 --- a/tools/test/integration/bootstrap_gum_temp.bats +++ b/tools/test/integration/bootstrap_gum_temp.bats @@ -119,3 +119,47 @@ fi [ -n "$GUM" ] [ -x "$GUM" ] } + +@test "bootstrap_gum_temp: cached binary on disk is reused without downloading" { + DOWNLOADER=curl + # Pre-seed the persistent cache with an executable gum. + mkdir -p "${GUM_CACHE_ROOT}/${GUM_VERSION}" + printf '#!/bin/bash\n' > "${GUM_CACHE_ROOT}/${GUM_VERSION}/gum" + chmod +x "${GUM_CACHE_ROOT}/${GUM_VERSION}/gum" + # Any downloader call would be a regression — fail loudly if it happens. + shim_bin curl 22 + shim_bin wget 22 + + bootstrap_gum_temp + [ "$GUM_STATUS" = "cached" ] + [ "$GUM" = "${GUM_CACHE_ROOT}/${GUM_VERSION}/gum" ] + [ -x "$GUM" ] + [ "$(shim_call_count curl)" = "0" ] + [ "$(shim_call_count wget)" = "0" ] +} + +@test "bootstrap_gum_temp: successful install promotes binary into the cache" { + DOWNLOADER=curl + shim_bin_script curl ' +out=""; prev="" +for a in "$@"; do + [[ "$prev" == "-o" ]] && out="$a" + prev="$a" +done +[[ -n "$out" ]] && printf "fake" > "$out" +' + shim_bin sha256sum + shim_bin shasum + shim_bin_script tar ' +if [[ "$1" == "-xzf" ]]; then + dest="$4" + mkdir -p "$dest" + printf "#!/bin/bash\n" > "$dest/gum" + chmod +x "$dest/gum" +fi +' + bootstrap_gum_temp + [ "$GUM_STATUS" = "installed" ] + [ -x "${GUM_CACHE_ROOT}/${GUM_VERSION}/gum" ] + [ "$GUM" = "${GUM_CACHE_ROOT}/${GUM_VERSION}/gum" ] +} From f8b710a6c86a966320cd74dac7cb2522493dcf98 Mon Sep 17 00:00:00 2001 From: WenjinXie Date: Thu, 21 May 2026 13:54:15 +0800 Subject: [PATCH 30/35] [tools] install.sh: replace bash 4+ ${var,,} with tr for bash 3.2 compatibility MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reviewer reported: ./install.sh: line 1328: ${expected,,}: bad substitution Root cause: the SHA512 case-insensitive compare used ${var,,}, which is bash 4+ syntax. macOS still ships /bin/bash 3.2 (GPLv3 hold-out), and when the shebang resolves to /bin/bash, the script reaches stage 4 ("Installing Flink Agents") before the bad-substitution surfaces — so the user has already waited through the Flink + Python downloads by the time it dies. Fix: case-fold via `tr '[:upper:]' '[:lower:]'` instead — POSIX, works on bash 3.2. A grep for other bash 4+ constructs (${var^^}, mapfile / readarray, declare -A, local -n, [[ -v ]], globstar, coproc) came up clean, so the installer now runs end-to-end on macOS's /bin/bash 3.2 without anyone needing to `brew install bash` first. Verified manually with `/bin/bash install.sh --help` and both --dry-run paths. Co-Authored-By: Claude Opus 4.7 --- tools/install.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tools/install.sh b/tools/install.sh index d2b95c3e9..41c3b1ad7 100755 --- a/tools/install.sh +++ b/tools/install.sh @@ -1325,7 +1325,12 @@ verify_flink_agents_jar_sha512() { return 0 fi - if [[ "${expected,,}" != "${actual,,}" ]]; then + # Use `tr` for case-folding instead of `${var,,}` — the latter is + # bash 4+ only, and macOS still ships /bin/bash 3.2. + local expected_lc actual_lc + expected_lc="$(printf '%s' "$expected" | tr '[:upper:]' '[:lower:]')" + actual_lc="$(printf '%s' "$actual" | tr '[:upper:]' '[:lower:]')" + if [[ "$expected_lc" != "$actual_lc" ]]; then die "SHA512 mismatch for $(basename "$jar") (expected ${expected:0:16}…, got ${actual:0:16}…)" fi ui_success "SHA512 verified" From c7387f82c599d72bc91b6c28407d8ddb615986ed Mon Sep 17 00:00:00 2001 From: WenjinXie Date: Thu, 21 May 2026 14:09:37 +0800 Subject: [PATCH 31/35] [tools] install.sh: accept SNAPSHOT / -rc / X.Y versions in Flink detection Reviewer pointed at a locally compiled /Users/.../flink-dist/target/flink-2.2-SNAPSHOT-bin/flink-2.2-SNAPSHOT and the installer reported Could not auto-detect Flink version from Two problems: 1. detect_flink_version_from_home() only matched `flink-dist-X.Y.Z.jar`, so a source-build artifact called `flink-dist-2.2-SNAPSHOT.jar` fell through both paths. 2. Every place that derived FLINK_MAJOR_MINOR did `${FLINK_VERSION%.*}`, which silently produced "2" for "2.2-SNAPSHOT" (it strips the last dot-segment, which here is the entire "2-SNAPSHOT"). Fix: - Use a single shared regex `[0-9]+\.[0-9]+(\.[0-9]+)?(-[A-Za-z0-9._]+)?` in detect_flink_version_from_home() for both the JAR-filename fast path and the `bin/flink --version` slow path. Now accepts 2.2-SNAPSHOT, 2.1.0-SNAPSHOT, 2.0.0-rc1, etc. - Introduce flink_major_minor() that pulls the leading X.Y out of any such version string and replace every `${FLINK_VERSION%.*}` site with it. So FLINK_MAJOR_MINOR for "2.2-SNAPSHOT" is "2.2", which is what the ASF mirror keys agents-JAR releases on. Tests cover X.Y-SNAPSHOT (filename path), X.Y.Z-SNAPSHOT, and X.Y.Z-rc1 (CLI path), plus a small table for flink_major_minor including a "garbage in, empty out" case. Manually verified: a local snapshot FLINK_HOME now reports `(v2.2-SNAPSHOT)` in the plan and would resolve the correct mirror URL for the agents JAR. Co-Authored-By: Claude Opus 4.7 --- tools/install.sh | 38 +++++++++++++------ .../unit/detect_flink_version_from_home.bats | 34 +++++++++++++++++ 2 files changed, 61 insertions(+), 11 deletions(-) diff --git a/tools/install.sh b/tools/install.sh index 41c3b1ad7..0581eca03 100755 --- a/tools/install.sh +++ b/tools/install.sh @@ -657,35 +657,51 @@ prompt_flink_agents_version_interactive() { # Populate FLINK_VERSION from an existing FLINK_HOME. Tries to parse # `lib/flink-dist-.jar` first because it's instant; falls back to +# Extract the major.minor portion of a Flink version string. +# 2.2.0 -> 2.2 +# 2.2.0-SNAPSHOT -> 2.2 +# 2.2-SNAPSHOT -> 2.2 (local source builds, no patch number) +# 2.1-rc1 -> 2.1 +# Returns empty string on no match. Note: replaces the older +# `${FLINK_VERSION%.*}` trick, which silently produced "2" for "2.2-SNAPSHOT". +flink_major_minor() { + printf '%s' "$1" | sed -E -n 's/^([0-9]+\.[0-9]+).*/\1/p' +} + # `bin/flink --version`, which is authoritative but spins up a JVM and can # take 3-10s on cold start. Returns 0 on success. detect_flink_version_from_home() { [[ -n "${FLINK_HOME:-}" && -d "$FLINK_HOME" ]] || return 1 - local major_minor_patch="" + # A Flink version on the wire is roughly: + # .(.)?(-)? + # Suffix examples: SNAPSHOT, rc1, beta-2, 20251115. Anchor it loose + # enough to handle local source builds (flink-dist-2.2-SNAPSHOT.jar). + local ver_re='[0-9]+\.[0-9]+(\.[0-9]+)?(-[A-Za-z0-9._]+)?' + local version="" # Fast path: filename inspection. Avoids JVM startup entirely. local jar for jar in "$FLINK_HOME/lib"/flink-dist-[0-9]*.jar; do [[ -f "$jar" ]] || continue - major_minor_patch="$(basename "$jar" | sed -E -n 's/^flink-dist-([0-9]+\.[0-9]+\.[0-9]+)\.jar$/\1/p')" - [[ -n "$major_minor_patch" ]] && break + version="$(basename "$jar" | sed -E -n "s/^flink-dist-(${ver_re})\.jar$/\1/p")" + [[ -n "$version" ]] && break done # Slow path: ask the Flink CLI. JVM startup means a few seconds of # silence, so signal what's happening. - if [[ -z "$major_minor_patch" && -x "$FLINK_HOME/bin/flink" ]]; then + if [[ -z "$version" && -x "$FLINK_HOME/bin/flink" ]]; then ui_info "Detecting Flink version (running '${FLINK_HOME}/bin/flink --version', this may take a few seconds)..." local out out="$("$FLINK_HOME/bin/flink" --version 2>/dev/null || true)" - major_minor_patch="$(printf '%s\n' "$out" | sed -E -n 's/.*Version:[[:space:]]*([0-9]+\.[0-9]+\.[0-9]+).*/\1/p' | head -n1)" + version="$(printf '%s\n' "$out" | sed -E -n "s/.*Version:[[:space:]]*(${ver_re}).*/\1/p" | head -n1)" fi - if [[ -z "$major_minor_patch" ]]; then + if [[ -z "$version" ]]; then return 1 fi - FLINK_VERSION="$major_minor_patch" + FLINK_VERSION="$version" return 0 } @@ -732,7 +748,7 @@ plan_flink() { ui_warn "Could not auto-detect Flink version from $FLINK_HOME; assuming ${FLINK_VERSION}." ui_warn "If JAR copy fails, set FLINK_VERSION explicitly (e.g. FLINK_VERSION=2.1.1 bash install.sh)." fi - FLINK_MAJOR_MINOR="${FLINK_VERSION%.*}" + FLINK_MAJOR_MINOR="$(flink_major_minor "$FLINK_VERSION")" return fi @@ -749,7 +765,7 @@ plan_flink() { INSTALL_DIR="$(normalize_path "$INSTALL_DIR")" FLINK_HOME="${INSTALL_DIR}/flink-${FLINK_VERSION}" - FLINK_MAJOR_MINOR="${FLINK_VERSION%.*}" + FLINK_MAJOR_MINOR="$(flink_major_minor "$FLINK_VERSION")" } plan_flink_agents() { @@ -983,12 +999,12 @@ edit_plan_interactive() { edit_prompt_flink_home detect_flink_version_from_home || ui_warn "Could not auto-detect Flink version; keeping ${FLINK_VERSION}" fi - FLINK_MAJOR_MINOR="${FLINK_VERSION%.*}" + FLINK_MAJOR_MINOR="$(flink_major_minor "$FLINK_VERSION")" ;; flink_version) prompt_flink_version_interactive || true FLINK_HOME="${INSTALL_DIR}/flink-${FLINK_VERSION}" - FLINK_MAJOR_MINOR="${FLINK_VERSION%.*}" + FLINK_MAJOR_MINOR="$(flink_major_minor "$FLINK_VERSION")" ;; install_dir) INSTALL_DIR="$(prompt_path_choice_interactive \ diff --git a/tools/test/unit/detect_flink_version_from_home.bats b/tools/test/unit/detect_flink_version_from_home.bats index d78894ed8..546bb51ad 100644 --- a/tools/test/unit/detect_flink_version_from_home.bats +++ b/tools/test/unit/detect_flink_version_from_home.bats @@ -78,3 +78,37 @@ EOF [ "$status" -eq 0 ] case "$output" in *"Detecting Flink version"*) false ;; *) ;; esac } + +@test "detect_flink_version_from_home: accepts X.Y-SNAPSHOT from local source builds" { + : > "$FLINK_HOME/lib/flink-dist-2.2-SNAPSHOT.jar" + FLINK_VERSION="" + detect_flink_version_from_home + [ "$FLINK_VERSION" = "2.2-SNAPSHOT" ] +} + +@test "detect_flink_version_from_home: accepts X.Y.Z-SNAPSHOT" { + : > "$FLINK_HOME/lib/flink-dist-2.1.0-SNAPSHOT.jar" + FLINK_VERSION="" + detect_flink_version_from_home + [ "$FLINK_VERSION" = "2.1.0-SNAPSHOT" ] +} + +@test "detect_flink_version_from_home: accepts -rc suffix from CLI output" { + cat > "$FLINK_HOME/bin/flink" <<'EOF' +#!/usr/bin/env bash +echo "Version: 2.0.0-rc1, Commit ID: abc" +EOF + chmod +x "$FLINK_HOME/bin/flink" + FLINK_VERSION="" + detect_flink_version_from_home + [ "$FLINK_VERSION" = "2.0.0-rc1" ] +} + +@test "flink_major_minor: derives X.Y from various version strings" { + [ "$(flink_major_minor "2.2.0")" = "2.2" ] + [ "$(flink_major_minor "2.2.0-SNAPSHOT")" = "2.2" ] + [ "$(flink_major_minor "2.2-SNAPSHOT")" = "2.2" ] + [ "$(flink_major_minor "1.20.3")" = "1.20" ] + [ "$(flink_major_minor "2.1-rc1")" = "2.1" ] + [ "$(flink_major_minor "garbage")" = "" ] +} From fcee67df95d28dd153df5e63b8da4dee17776895 Mon Sep 17 00:00:00 2001 From: WenjinXie Date: Thu, 21 May 2026 15:48:31 +0800 Subject: [PATCH 32/35] [tools] install.sh: fix normalize_path producing /tmp/x\ on bash 3.2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reproduced on macOS /bin/bash 3.2.57 with PWD=/tmp/test-wizard: normalize_path("./") → "/tmp/test-wizard\" Root cause: the "/./" → "/" fold used p="${p//\/.\//\/}" Bash 3.2 doesn't strip the backslash from an escaped slash in the *replacement* half of a `${var//pat/repl}` expansion, so the literal `\/` ends up in the result. Bash 4+ unescapes it, hiding the bug — which is why we missed it locally until a reviewer (and then us) actually ran the installer through macOS's default /bin/bash. Fix: rewrite the four folding steps as a single sed invocation: sed -E 's|/+|/|g; s|/\./|/|g; s|/\.$||; s|/+$||' The "~" expansion and relative→absolute anchor still happen in bash, but the slash arithmetic is now handled by sed and avoids the parameter-expansion quirk entirely. Verified on both /bin/bash 3.2.57 and /opt/homebrew/bin/bash 5+: same output for "./", "", "/", "~/foo", "bar", "/x//y/", "/x/./y/", "/x/./y/.", "//x//y//.//". Drive-by: bump curl/wget --max-time 600 → 900, since slow ASF mirror hops were brushing up against the 10-minute ceiling on the Flink tarball. download_file.bats expected-argv assertions updated to match. Co-Authored-By: Claude Opus 4.7 --- tools/install.sh | 25 ++++++++++------------- tools/test/integration/download_file.bats | 4 ++-- 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/tools/install.sh b/tools/install.sh index 0581eca03..f658af057 100755 --- a/tools/install.sh +++ b/tools/install.sh @@ -69,10 +69,10 @@ download_file() { detect_downloader fi if [[ "$DOWNLOADER" == "curl" ]]; then - curl -fL --progress-bar --proto '=https' --tlsv1.2 --retry 3 --max-time 600 --retry-delay 1 --retry-connrefused -o "$output" "$url" + curl -fL --progress-bar --proto '=https' --tlsv1.2 --retry 3 --max-time 900 --retry-delay 1 --retry-connrefused -o "$output" "$url" return fi - wget -q --show-progress --https-only --secure-protocol=TLSv1_2 --tries=3 --timeout=600 -O "$output" "$url" + wget -q --show-progress --https-only --secure-protocol=TLSv1_2 --tries=3 --timeout=900 -O "$output" "$url" } GUM_VERSION="${FLINK_AGENTS_GUM_VERSION:-0.17.0}" @@ -489,6 +489,13 @@ require_cmd() { # - "/./" segments and trailing "/." are folded away # - trailing slashes are stripped, except for the root "/" # - runs of '/' collapse to a single '/' +# +# Implementation note: the folding steps use sed instead of bash +# `${var//pattern/replacement}`. Bash 3.2 (macOS /bin/bash) does not +# strip the backslash from an escaped slash in the replacement, so +# `${p//\/.\//\/}` leaves a literal `\/` in the result (concretely: +# normalize_path "./" under PWD=/tmp/x produced /tmp/x\ on macOS). +# sed sidesteps the parameter-expansion quirk entirely. normalize_path() { local p="$1" if [[ -z "$p" ]]; then @@ -499,18 +506,8 @@ normalize_path() { if [[ "$p" != /* ]]; then p="$PWD/$p" fi - while [[ "$p" == *//* ]]; do - p="${p//\/\//\/}" - done - while [[ "$p" == */./* ]]; do - p="${p//\/.\//\/}" - done - while [[ "$p" == */. ]]; do - p="${p%/.}" - done - while [[ "${#p}" -gt 1 && "$p" == */ ]]; do - p="${p%/}" - done + p="$(printf '%s' "$p" | sed -E 's|/+|/|g; s|/\./|/|g; s|/\.$||; s|/+$||')" + [[ -z "$p" ]] && p='/' printf '%s' "$p" } diff --git a/tools/test/integration/download_file.bats b/tools/test/integration/download_file.bats index c4e6a5572..2b02d0759 100644 --- a/tools/test/integration/download_file.bats +++ b/tools/test/integration/download_file.bats @@ -18,7 +18,7 @@ setup() { local got got="$(shim_calls curl)" local expected - expected=$'-fL\t--progress-bar\t--proto\t=https\t--tlsv1.2\t--retry\t3\t--max-time\t600\t--retry-delay\t1\t--retry-connrefused\t-o\t'"$BATS_TEST_TMPDIR/out"$'\thttps://example.test/x.tgz' + expected=$'-fL\t--progress-bar\t--proto\t=https\t--tlsv1.2\t--retry\t3\t--max-time\t900\t--retry-delay\t1\t--retry-connrefused\t-o\t'"$BATS_TEST_TMPDIR/out"$'\thttps://example.test/x.tgz' [ "$got" = "$expected" ] } @@ -57,7 +57,7 @@ setup() { local got got="$(shim_calls wget)" local expected - expected=$'-q\t--show-progress\t--https-only\t--secure-protocol=TLSv1_2\t--tries=3\t--timeout=600\t-O\t'"$BATS_TEST_TMPDIR/out"$'\thttps://example.test/x.tgz' + expected=$'-q\t--show-progress\t--https-only\t--secure-protocol=TLSv1_2\t--tries=3\t--timeout=900\t-O\t'"$BATS_TEST_TMPDIR/out"$'\thttps://example.test/x.tgz' [ "$got" = "$expected" ] } From 23ecdec34406061f0870ed6c00adfb1adad718c1 Mon Sep 17 00:00:00 2001 From: WenjinXie Date: Thu, 21 May 2026 16:34:58 +0800 Subject: [PATCH 33/35] [tools] install.sh: reject foreign directories as VENV_DIR MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reviewer reported: typing "./" at the Choose Python venv directory prompt, when the cwd held unrelated files, was silently accepted — plan announced "Reusing existing virtual environment", and the script later either died trying to `source $VENV_DIR/bin/activate` (after already downloading Flink), or worse, scribbled venv files alongside the user's project. Add validate_venv_dir() which classifies a candidate path as one of new / empty / venv / nonempty / file. Only the first three are safe. The venv marker is strict: pyvenv.cfg must be present (a stray `bin/activate` doesn't make a directory a venv). Apply the classifier in two places: - Interactive (plan_pyflink + the venv_dir / enable_pyflink edits in edit_plan_interactive) re-prompts on nonempty/file with an explanatory warning. Hoisted the prompt+validate loop into a shared prompt_and_validate_venv_dir() helper. - Non-interactive (--non-interactive / NO_PROMPT / explicit VENV_DIR= env var) dies with a clear message BEFORE stage 3 downloads Flink — the integration test guards that no curl/wget call happens on the failure path. Reordered plan_pyflink to validate VENV_DIR before resolve_python: bad user input should fail fast, not after the user has just gone to install a Python interpreter. Tests: 8 unit cases covering each classifier branch + 2 integration cases (foreign dir aborts before download, real venv with pyvenv.cfg is accepted). 133 → 143 BATS, all green. Co-Authored-By: Claude Opus 4.7 --- tools/install.sh | 109 +++++++++++++++--- .../test/integration/venv_dir_validation.bats | 61 ++++++++++ tools/test/unit/validate_venv_dir.bats | 83 +++++++++++++ 3 files changed, 237 insertions(+), 16 deletions(-) create mode 100644 tools/test/integration/venv_dir_validation.bats create mode 100644 tools/test/unit/validate_venv_dir.bats diff --git a/tools/install.sh b/tools/install.sh index f658af057..036705592 100755 --- a/tools/install.sh +++ b/tools/install.sh @@ -771,6 +771,75 @@ plan_flink_agents() { fi } +# Classify a candidate VENV_DIR path. Echoes one of: +# new — path doesn't exist; caller should mkdir + python -m venv +# empty — empty directory; safe to use as venv root +# venv — already a real Python venv (pyvenv.cfg marker present) +# nonempty — directory contains foreign files; caller MUST re-prompt +# file — path is a regular file or invalid; caller MUST re-prompt +# Exit code: 0 for the safe-to-use cases, 1 for the must-re-prompt cases. +# +# We only treat pyvenv.cfg as the venv marker — `bin/activate` alone is +# not enough since arbitrary projects sometimes ship a file by that name. +validate_venv_dir() { + local p="$1" + if [[ -z "$p" ]]; then + printf 'file' + return 1 + fi + if [[ ! -e "$p" ]]; then + printf 'new' + return 0 + fi + if [[ ! -d "$p" ]]; then + printf 'file' + return 1 + fi + if [[ -f "$p/pyvenv.cfg" ]]; then + printf 'venv' + return 0 + fi + # Directory exists. Empty iff no entries including dotfiles. + local entries + entries="$(find "$p" -mindepth 1 -maxdepth 1 -print -quit 2>/dev/null || true)" + if [[ -z "$entries" ]]; then + printf 'empty' + return 0 + fi + printf 'nonempty' + return 1 +} + +# Drive the "Choose Python venv directory" picker, then loop until the +# user picks a path that's either non-existent, empty, or an actual +# Python venv. Sets the global VENV_DIR. Caller must already have +# ensured we're in an interactive shell (is_promptable). +prompt_and_validate_venv_dir() { + local seed_default="$1" + VENV_DIR="$(prompt_path_choice_interactive \ + "Choose Python venv directory" \ + "$seed_default" \ + "/path/to/venv")" + VENV_DIR="$(normalize_path "$VENV_DIR")" + local kind + while true; do + kind="$(validate_venv_dir "$VENV_DIR")" || true + case "$kind" in + new|empty|venv) return 0 ;; + nonempty) + ui_warn "${VENV_DIR} already exists and is not a Python venv. Pick a different path, or remove its contents and try again." + ;; + file|*) + ui_warn "${VENV_DIR} is not a directory. Pick a different path." + ;; + esac + VENV_DIR="$(prompt_path_input \ + "Enter Python venv directory" \ + "/path/to/venv")" + VENV_DIR="$(normalize_path "$VENV_DIR")" + done +} + plan_pyflink() { case "$ENABLE_PYFLINK" in Yes|No) @@ -792,16 +861,32 @@ plan_pyflink() { fi PYFLINK_ACTUALLY_ENABLED=1 - resolve_python + # Validate VENV_DIR before doing anything Python-related — if the + # user picked a bad path we want to bail before we make them resolve + # a Python interpreter, and certainly before stage 3 downloads Flink. if [[ "$VENV_DIR_EXPLICIT" -eq 0 ]] && is_promptable; then - VENV_DIR="$(prompt_path_choice_interactive \ - "Choose Python venv directory" \ - "$VENV_DIR" \ - "/path/to/venv")" + prompt_and_validate_venv_dir "$VENV_DIR" + else + VENV_DIR="$(normalize_path "$VENV_DIR")" + # No prompt available (--non-interactive / NO_PROMPT / explicit + # VENV_DIR). Refuse to scribble into a foreign directory; tell + # the caller exactly what's wrong instead of silently mixing the + # venv into their codebase. + local kind + kind="$(validate_venv_dir "$VENV_DIR")" || true + case "$kind" in + new|empty|venv) ;; + nonempty) + die "VENV_DIR=${VENV_DIR} already exists and is not a Python venv. Set VENV_DIR to a new or empty path, or to an existing venv." + ;; + file|*) + die "VENV_DIR=${VENV_DIR} is not a directory." + ;; + esac fi - VENV_DIR="$(normalize_path "$VENV_DIR")" + resolve_python } # Echo one of: "confirm" | "edit" | "cancel" @@ -1020,22 +1105,14 @@ edit_plan_interactive() { ENABLE_PYFLINK=Yes PYFLINK_ACTUALLY_ENABLED=1 resolve_python - VENV_DIR="$(prompt_path_choice_interactive \ - "Choose Python venv directory" \ - "$VENV_DIR" \ - "/path/to/venv")" - VENV_DIR="$(normalize_path "$VENV_DIR")" + prompt_and_validate_venv_dir "$VENV_DIR" fi else ENABLE_PYFLINK=No fi ;; venv_dir) - VENV_DIR="$(prompt_path_choice_interactive \ - "Choose Python venv directory" \ - "$VENV_DIR" \ - "/path/to/venv")" - VENV_DIR="$(normalize_path "$VENV_DIR")" + prompt_and_validate_venv_dir "$VENV_DIR" ;; esac diff --git a/tools/test/integration/venv_dir_validation.bats b/tools/test/integration/venv_dir_validation.bats new file mode 100644 index 000000000..55fc23155 --- /dev/null +++ b/tools/test/integration/venv_dir_validation.bats @@ -0,0 +1,61 @@ +#!/usr/bin/env bats + +# End-to-end behavior of VENV_DIR validation. In non-interactive mode +# (the only mode the integration runner can drive), passing a foreign +# non-empty directory as VENV_DIR must abort BEFORE downloading Flink. + +setup() { + load '../helpers/load' + load '../helpers/shim' + shim_setup + # Fake an existing Flink so plan_flink (INSTALL_FLINK=No) passes. + export FLINK_HOME="$BATS_TEST_TMPDIR/flink-home" + mkdir -p "$FLINK_HOME/lib" + : > "$FLINK_HOME/lib/flink-dist-2.2.0.jar" + # Fake java so check_java passes. + shim_bin_script java " +case \"\$1\" in + -version) echo 'openjdk version \"17.0.2\"' >&2 ;; +esac +" +} + +@test "VENV_DIR=non-empty foreign dir + --non-interactive → die before download" { + local foreign="$BATS_TEST_TMPDIR/foreign" + mkdir -p "$foreign" + : > "$foreign/unrelated.txt" + + run env VENV_DIR="$foreign" PYTHON_BIN=/no/such/python3 \ + bash "${BATS_TEST_DIRNAME}/../../install.sh" \ + --non-interactive --enable-pyflink + + [ "$status" -ne 0 ] + case "$output" in + *"already exists and is not a Python venv"*) ;; + *) false ;; + esac + # Critical: must NOT have invoked any downloader before failing. + [ "$(shim_call_count curl)" = "0" ] + [ "$(shim_call_count wget)" = "0" ] +} + +@test "VENV_DIR=existing real venv (has pyvenv.cfg) is accepted in non-interactive mode" { + local venv="$BATS_TEST_TMPDIR/real-venv" + mkdir -p "$venv/bin" + : > "$venv/pyvenv.cfg" + : > "$venv/bin/activate" + + # We don't actually want it to *install* — just to get past the + # plan_pyflink validation. Use --dry-run so plan succeeds and we + # bail before stage 3. + run env VENV_DIR="$venv" PYTHON_BIN=/no/such/python3 \ + bash "${BATS_TEST_DIRNAME}/../../install.sh" \ + --non-interactive --enable-pyflink --dry-run + # Either the dry-run printout completes (status 0) or fails for an + # unrelated reason (e.g. PYTHON_BIN missing in non-interactive resolve_python). + # The thing we explicitly want NOT to see: the VENV_DIR rejection. + case "$output" in + *"is not a Python venv"*) false ;; + *) ;; + esac +} diff --git a/tools/test/unit/validate_venv_dir.bats b/tools/test/unit/validate_venv_dir.bats new file mode 100644 index 000000000..b41dfd157 --- /dev/null +++ b/tools/test/unit/validate_venv_dir.bats @@ -0,0 +1,83 @@ +#!/usr/bin/env bats + +# validate_venv_dir classifies a candidate VENV_DIR path so the caller +# can either accept it (new / empty / real venv) or re-prompt the user +# (non-empty foreign directory or file). +# +# Echoes one of: new | empty | venv | nonempty | file +# Exit code 0 for new/empty/venv (caller proceeds), 1 for nonempty/file +# (caller re-prompts). + +setup() { + load '../helpers/load' + load_install_sh + reset_install_sh_state +} + +@test "validate_venv_dir: path that does not exist → 'new'" { + local p="$BATS_TEST_TMPDIR/will-be-created" + run validate_venv_dir "$p" + [ "$status" -eq 0 ] + [ "$output" = "new" ] +} + +@test "validate_venv_dir: empty directory → 'empty'" { + local p="$BATS_TEST_TMPDIR/empty" + mkdir -p "$p" + run validate_venv_dir "$p" + [ "$status" -eq 0 ] + [ "$output" = "empty" ] +} + +@test "validate_venv_dir: directory with pyvenv.cfg → 'venv'" { + local p="$BATS_TEST_TMPDIR/avenv" + mkdir -p "$p/bin" + : > "$p/pyvenv.cfg" + : > "$p/bin/activate" + run validate_venv_dir "$p" + [ "$status" -eq 0 ] + [ "$output" = "venv" ] +} + +@test "validate_venv_dir: non-empty foreign directory → 'nonempty' (rc=1)" { + local p="$BATS_TEST_TMPDIR/foreign" + mkdir -p "$p" + : > "$p/some-unrelated-file" + : > "$p/README.md" + run validate_venv_dir "$p" + [ "$status" -eq 1 ] + [ "$output" = "nonempty" ] +} + +@test "validate_venv_dir: path is a regular file → 'file' (rc=1)" { + local p="$BATS_TEST_TMPDIR/just-a-file" + : > "$p" + run validate_venv_dir "$p" + [ "$status" -eq 1 ] + [ "$output" = "file" ] +} + +@test "validate_venv_dir: directory with only a hidden file is still 'nonempty'" { + local p="$BATS_TEST_TMPDIR/dot" + mkdir -p "$p" + : > "$p/.hiddenfile" + run validate_venv_dir "$p" + [ "$status" -eq 1 ] + [ "$output" = "nonempty" ] +} + +@test "validate_venv_dir: bin/activate without pyvenv.cfg is still 'nonempty'" { + # A user could have a random project with a bin/activate file. Treat + # only the strict pyvenv.cfg marker as a real venv. + local p="$BATS_TEST_TMPDIR/halfbaked" + mkdir -p "$p/bin" + : > "$p/bin/activate" + run validate_venv_dir "$p" + [ "$status" -eq 1 ] + [ "$output" = "nonempty" ] +} + +@test "validate_venv_dir: empty argument → 'file' (rc=1) (treated as invalid)" { + run validate_venv_dir "" + [ "$status" -eq 1 ] +} From 2b0fedf4449a6749c5d26bdece911893954918a4 Mon Sep 17 00:00:00 2001 From: WenjinXie Date: Thu, 21 May 2026 16:44:06 +0800 Subject: [PATCH 34/35] [tools] install.sh: fall back to apache-flink~=X.Y.0 for SNAPSHOT builds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reviewer ran the installer against a source-built Flink 2.1-SNAPSHOT and pip blew up at stage 4: ERROR: No matching distribution found for apache-flink==2.1-SNAPSHOT PyPI only ships finished release wheels for apache-flink — there's no "2.1-SNAPSHOT" upload — so an `==` pin against any pre-release Flink version is doomed. Switch to PEP 440's compatible-release operator (~=) when the Flink version carries a pre-release suffix: FLINK_VERSION=2.1-SNAPSHOT → apache-flink~=2.1.0 FLINK_VERSION=2.0.0-rc1 → apache-flink~=2.0.0 FLINK_VERSION=2.2.0 → apache-flink==2.2.0 (unchanged) FLINK_VERSION=1.20.3 → apache-flink==1.20.3 (unchanged) A new is_snapshot_version() helper decides the branch (any "-suffix" is treated as a pre-release; pure X.Y.Z stays on the exact pin). When we take the fallback we warn the user that PyFlink will come from the nearest released minor on PyPI — their Java JARs are still the source build, but the Python bridge may not match exactly. That tradeoff is better than a silent install failure after Flink has already been downloaded. Tests: 4 new unit cases for is_snapshot_version (SNAPSHOT, rc/beta/dev, plain releases, empty); 143 → 147 BATS, all green. Co-Authored-By: Claude Opus 4.7 --- tools/install.sh | 23 ++++++++++++- tools/test/unit/is_snapshot_version.bats | 43 ++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 1 deletion(-) create mode 100644 tools/test/unit/is_snapshot_version.bats diff --git a/tools/install.sh b/tools/install.sh index 036705592..752b0b6a2 100755 --- a/tools/install.sh +++ b/tools/install.sh @@ -665,6 +665,15 @@ flink_major_minor() { printf '%s' "$1" | sed -E -n 's/^([0-9]+\.[0-9]+).*/\1/p' } +# True (rc=0) when the version string carries a pre-release suffix +# (-SNAPSHOT, -rc1, -dev, -beta-2, ...). PyPI only carries finished +# release wheels for apache-flink, so we need to know when to fall +# back from `==exact` to `~=X.Y.0` (compatible release). +is_snapshot_version() { + local v="${1:-}" + [[ -n "$v" && "$v" == *-* ]] +} + # `bin/flink --version`, which is authoritative but spins up a JVM and can # take 3-10s on cold start. Returns 0 on success. detect_flink_version_from_home() { @@ -1366,10 +1375,22 @@ setup_python_env() { export PIP_NO_COLOR=1 export PIP_NO_INPUT=1 + # PyPI only ships finished release wheels for apache-flink, so a + # source-built FLINK_VERSION like "2.1-SNAPSHOT" / "2.0.0-rc1" has + # no matching distribution. Fall back to the compatible-release + # operator (~= X.Y.0 ↔ >=X.Y.0, Date: Thu, 21 May 2026 17:02:59 +0800 Subject: [PATCH 35/35] [tools] install.sh: print a stage-aware failure banner on set -e exits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reviewer reproductions (Flink download 404, pip install for SNAPSHOT, tar exit non-zero) all ended with the script silently dying after a wall of pip/curl/tar output — no marker for "the installer failed", no hint of which stage we were in, no line number to grep for. Install an ERR trap that fires whenever set -e tears the script down on an unhandled non-zero command. The banner prints: ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ ✗ Installation failed at stage 3/5 (Installing Apache Flink). Command: curl -fL --progress-bar ... -o "$output" "$url" Source: install.sh:75 Exit code: 56 Re-run with --verbose for full output, or report at: https://github.com/apache/flink-agents/issues ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ Implementation notes: - Enable `set -E` (errtrace) so the trap survives into functions and command substitutions — the actual fail sites are all nested. - ui_stage() now remembers INSTALL_STAGE_TITLE so the banner can name the stage in addition to its number. - The banner writes to stderr and self-disables (trap - ERR) before its own echo lines so a failure inside the handler doesn't recurse. - die() / die_cancelled() use plain `exit`, which doesn't trip ERR, so their existing single-line messages stay unchanged — banner is reserved for the genuinely unexpected failures. - Script exits with the failing command's original rc (curl 56, pip 1, tar 2, etc.) so report-an-issue threads carry a real signal. Tests: 2 integration cases — set -e tear-down lights up the banner with the right stage / line / exit code, and a die() path stays a single-line warning. 147 → 149 BATS, all green. Co-Authored-By: Claude Opus 4.7 --- tools/install.sh | 47 +++++++++++++++++++++++++++- tools/test/integration/err_trap.bats | 39 +++++++++++++++++++++++ 2 files changed, 85 insertions(+), 1 deletion(-) create mode 100644 tools/test/integration/err_trap.bats diff --git a/tools/install.sh b/tools/install.sh index 752b0b6a2..c71e3fa31 100755 --- a/tools/install.sh +++ b/tools/install.sh @@ -17,7 +17,10 @@ # limitations under the License. ################################################################################ -set -euo pipefail +# -E (errtrace) propagates the ERR trap into functions and command +# substitutions so the failure banner fires from anywhere in the script, +# not only at the top level. +set -Eeuo pipefail BOLD='\033[1m' ACCENT='\033[38;2;255;77;77m' # coral-bright @@ -341,8 +344,46 @@ die_cancelled() { exit 130 } +# Fires on any command that exits non-zero under set -e — the cases that +# would otherwise dump a tail of pip / curl / tar noise and silently exit. +# `die`/`die_cancelled` use plain `exit` (not a non-zero command), so they +# do NOT trip this trap; they print their own friendly message and exit +# straight away. That keeps the banner reserved for genuinely unexpected +# failures. +on_error() { + local rc=$1 + local line=$2 + local cmd=$3 + # Suppress ourselves if we re-enter (e.g. echo failing under set -e + # inside the handler itself). + trap - ERR + { + echo "" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + if (( INSTALL_STAGE_CURRENT > 0 )); then + printf '%b Installation failed at stage %d/%d (%s).\n' \ + "${ERROR}✗${NC}" \ + "$INSTALL_STAGE_CURRENT" "$INSTALL_STAGE_TOTAL" \ + "$INSTALL_STAGE_TITLE" + else + printf '%b Installation failed.\n' "${ERROR}✗${NC}" + fi + echo "" + echo " Command: ${cmd}" + echo " Source: install.sh:${line}" + echo " Exit code: ${rc}" + echo "" + echo " Re-run with --verbose for full output, or report at:" + echo " https://github.com/apache/flink-agents/issues" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + } >&2 + exit "$rc" +} +trap 'on_error $? $LINENO "$BASH_COMMAND"' ERR + INSTALL_STAGE_TOTAL=5 INSTALL_STAGE_CURRENT=0 +INSTALL_STAGE_TITLE="" ui_section() { local title="$1" @@ -357,6 +398,10 @@ ui_section() { ui_stage() { local title="$1" INSTALL_STAGE_CURRENT=$((INSTALL_STAGE_CURRENT + 1)) + # Remember the stage title so on_error can include it in the + # failure banner ("Installation failed at stage 3/5 (Installing + # Apache Flink)"). + INSTALL_STAGE_TITLE="$title" ui_section "[${INSTALL_STAGE_CURRENT}/${INSTALL_STAGE_TOTAL}] ${title}" } diff --git a/tools/test/integration/err_trap.bats b/tools/test/integration/err_trap.bats new file mode 100644 index 000000000..bc553c53f --- /dev/null +++ b/tools/test/integration/err_trap.bats @@ -0,0 +1,39 @@ +#!/usr/bin/env bats + +# When a command fails under `set -e` (i.e. NOT through die()/die_cancelled), +# the ERR trap should print a banner that names the stage, the command, +# the install.sh line, and the exit code. die() must keep its existing +# single-line message and not trip the banner. + +@test "on_error: set -e failure triggers a stage-aware banner" { + # Use a real failure mode: pretend a Flink download 404'd. The script + # walks through plan_flink with INSTALL_FLINK=Yes (default INSTALL_DIR, + # but we override it to /tmp) and then dies on curl in stage 3. + local tmp="$BATS_TEST_TMPDIR/demo" + mkdir -p "$tmp" + run env INSTALL_DIR="$tmp" FLINK_BASE_URL="https://dlcdn.apache.org/flink" \ + FLINK_VERSION=99.99.0 \ + bash "${BATS_TEST_DIRNAME}/../../install.sh" \ + --install-flink --non-interactive + [ "$status" -ne 0 ] + # Banner must name the stage by title. + case "$output" in *"Installation failed at stage 3/5"*) ;; *) false ;; esac + case "$output" in *"Installing Apache Flink"*) ;; *) false ;; esac + # Banner must include source line + exit code keywords. + case "$output" in *"Source:"*"install.sh:"*) ;; *) false ;; esac + case "$output" in *"Exit code:"*) ;; *) false ;; esac + # Banner must not point at the trap line itself, which would be a + # regression — we want the line where the failing command lives. + case "$output" in *"install.sh:0"*) false ;; *) ;; esac +} + +@test "die(): single-line message, NO banner duplication" { + # die() uses `exit`, which doesn't trigger ERR. Reach it by giving + # plan_flink a non-existent FLINK_HOME under --non-interactive. + run env FLINK_HOME="" \ + bash "${BATS_TEST_DIRNAME}/../../install.sh" --non-interactive + [ "$status" -ne 0 ] + case "$output" in *"FLINK_HOME is not set"*) ;; *) false ;; esac + # Banner box characters must NOT appear in a die() path. + case "$output" in *"Installation failed at stage"*) false ;; *) ;; esac +}