From 6d6a13e576b79d69d0fb1775b00ea04f73829346 Mon Sep 17 00:00:00 2001 From: yuzelin Date: Wed, 20 Nov 2024 10:13:14 +0800 Subject: [PATCH 1/5] [ci] Update license/notice for project and inner python-java-bridge module and add ci for java module license check --- .github/workflows/check-licensing.yml | 57 +++ LICENSE | 20 + NOTICE | 267 +++++++++++- .../paimon-python-java-bridge/pom.xml | 95 ++++- .../src/main/resources/META-INF/NOTICE | 21 + .../tools/ci/log4j.properties | 43 ++ .../tools/ci/paimon-ci-tools/pom.xml | 78 ++++ .../tools/ci/licensecheck/JarFileChecker.java | 330 +++++++++++++++ .../tools/ci/licensecheck/LicenseChecker.java | 53 +++ .../ci/licensecheck/NoticeFileChecker.java | 387 ++++++++++++++++++ .../ci/suffixcheck/ScalaSuffixChecker.java | 260 ++++++++++++ .../ci/utils/dependency/DependencyParser.java | 224 ++++++++++ .../tools/ci/utils/deploy/DeployParser.java | 71 ++++ .../tools/ci/utils/notice/NoticeContents.java | 42 ++ .../tools/ci/utils/notice/NoticeParser.java | 92 +++++ .../tools/ci/utils/shade/ShadeParser.java | 110 +++++ .../tools/ci/utils/shared/Dependency.java | 135 ++++++ .../tools/ci/utils/shared/DependencyTree.java | 123 ++++++ .../tools/ci/utils/shared/ParserUtils.java | 74 ++++ .../src/main/resources/log4j2.properties | 25 ++ ...es-defining-excess-dependencies.modulelist | 20 + 21 files changed, 2510 insertions(+), 17 deletions(-) create mode 100644 .github/workflows/check-licensing.yml create mode 100644 paimon_python_java/paimon-python-java-bridge/src/main/resources/META-INF/NOTICE create mode 100644 paimon_python_java/paimon-python-java-bridge/tools/ci/log4j.properties create mode 100644 paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/pom.xml create mode 100644 paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/licensecheck/JarFileChecker.java create mode 100644 paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/licensecheck/LicenseChecker.java create mode 100644 paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/licensecheck/NoticeFileChecker.java create mode 100644 paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/suffixcheck/ScalaSuffixChecker.java create mode 100644 paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/utils/dependency/DependencyParser.java create mode 100644 paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/utils/deploy/DeployParser.java create mode 100644 paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/utils/notice/NoticeContents.java create mode 100644 paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/utils/notice/NoticeParser.java create mode 100644 paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/utils/shade/ShadeParser.java create mode 100644 paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/utils/shared/Dependency.java create mode 100644 paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/utils/shared/DependencyTree.java create mode 100644 paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/utils/shared/ParserUtils.java create mode 100644 paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/resources/log4j2.properties create mode 100644 paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/resources/modules-defining-excess-dependencies.modulelist diff --git a/.github/workflows/check-licensing.yml b/.github/workflows/check-licensing.yml new file mode 100644 index 0000000..53cc5c0 --- /dev/null +++ b/.github/workflows/check-licensing.yml @@ -0,0 +1,57 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Check licensing + +on: [push, pull_request] + +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.number || github.run_id }} + cancel-in-progress: true + +jobs: + build: + runs-on: ubuntu-latest + env: + MVN_COMMON_OPTIONS: -U -B --no-transfer-progress + MVN_BUILD_OUTPUT_FILE: "/tmp/mvn_build_output.out" + MVN_VALIDATION_DIR: "/tmp/paimon-validation-deployment" + + steps: + - uses: actions/checkout@v2 + + - name: Set JDK + uses: actions/setup-java@v2 + with: + java-version: 8 + distribution: 'adopt' + - name: Build + run: | + set -o pipefail + cd paimon_python_java/paimon-python-java-bridge + mvn clean deploy ${{ env.MVN_COMMON_OPTIONS }} -DskipTests \ + -DaltDeploymentRepository=validation_repository::default::file:${{ env.MVN_VALIDATION_DIR }} \ + | tee ${{ env.MVN_BUILD_OUTPUT_FILE }} + cd tools/ci/paimon-ci-tools + mvn clean install -DskipTests + + - name: Check licensing + run: | + cd paimon_python_java/paimon-python-java-bridge + mvn ${{ env.MVN_COMMON_OPTIONS }} exec:java@check-licensing -N \ + -Dexec.args="${{ env.MVN_BUILD_OUTPUT_FILE }} $(pwd) ${{ env.MVN_VALIDATION_DIR }}" \ + -Dlog4j.configurationFile=file://$(pwd)/tools/ci/log4j.properties + env: + MAVEN_OPTS: -Xmx4096m \ No newline at end of file diff --git a/LICENSE b/LICENSE index 261eeb9..f51b94d 100644 --- a/LICENSE +++ b/LICENSE @@ -199,3 +199,23 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + +------------------------------------------------------------------------------------ + +This product bundles various third-party components under other open source licenses. +This section summarizes those components and their licenses. See licenses/ +for text of these licenses. + +Apache Software Foundation License 2.0 +-------------------------------------- + +paimon_python_java/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/FileLock.java +paimon_python_java/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/NetUtils +paimon_python_java/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/PythonEnvUtils +paimon_python_java/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/PythonGatewayServer + +paimon_python_java/gateway_server.py +paimon_python_java/java_gateway.py +paimon_python_java/util/exceptions.py + +from http://flink.apache.org/ version 1.20.0 diff --git a/NOTICE b/NOTICE index b666c9f..81978c0 100644 --- a/NOTICE +++ b/NOTICE @@ -2,4 +2,269 @@ Apache Paimon Python Copyright 2024 The Apache Software Foundation This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). \ No newline at end of file +The Apache Software Foundation (http://www.apache.org/). + + +This project bundles the following dependencies under the under the MIT license +- org.slf4j:slf4j-api:1.7.32 + +This project bundles the following dependencies under the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt) +- org.apache.logging.log4j:log4j-1.2-api:2.17.1 +- org.apache.arrow:arrow-vector:14.0.0 +- org.apache.arrow:arrow-format:14.0.0 +- org.apache.arrow:arrow-memory-core:14.0.0 +- org.apache.arrow:arrow-memory-unsafe:14.0.0 +- org.apache.arrow:arrow-c-data:14.0.0 +- com.google.flatbuffers:flatbuffers-java:1.12.0 + +This project bundles the following dependencies under the 2-Clause BSD License +- net.sf.py4j:py4j:0.10.9.7 + + +Apache Paimon-shade (incubating) +Copyright 2023-2024 The Apache Software Foundation + +Paimon : Bundle +Copyright 2023-2024 The Apache Software Foundation + +paimon-common +Copyright 2023-2024 The Apache Software Foundation + +This project bundles the following dependencies under the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt) +- org.roaringbitmap:RoaringBitmap:1.0.5 +- org.apache.datasketches:datasketches-java:4.2.0 +- org.apache.datasketches:datasketches-memory:2.2.0 + +This project bundles the following dependencies under the BSD 3-clause license. +You find them under licenses/LICENSE.antlr-runtime and licenses/LICENSE.janino. + +- org.antlr:antlr4-runtime:4.9.3 +- org.codehaus.janino:janino:3.0.11 +- org.codehaus.janino:commons-compiler:3.0.11 +- it.unimi.dsi:fastutil:8.5.12 +- net.openhft:zero-allocation-hashing:0.16 +- com.github.davidmoten:hilbert-curve:0.2.2 +- com.github.davidmoten:guava-mini:0.1.3 + +datasketches-java +Copyright 2015-2022 The Apache Software Foundation + +Apache DataSketches Memory +Copyright 2022 - The Apache Software Foundation + +Copyright 2015-2018 Yahoo Inc. +Copyright 2019-2020 Verizon Media +Copyright 2021 Yahoo Inc. + +Prior to moving to ASF, the software for this project was developed at +Yahoo Inc. (https://developer.yahoo.com). + +Paimon : Core +Copyright 2023-2024 The Apache Software Foundation + +Paimon : Code Gen Loader +Copyright 2023-2024 The Apache Software Foundation + +paimon-format +Copyright 2023-2024 The Apache Software Foundation + +This project bundles the following dependencies under the Apache Software License 2.0. (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- org.apache.orc:orc-core:1.9.2 +- org.apache.orc:orc-shims:1.9.2 +- org.apache.hive:hive-storage-api:2.8.1 +- io.airlift:aircompressor:0.27 +- commons-lang:commons-lang:2.6 +- org.apache.commons:commons-lang3:3.12.0 + +- org.apache.avro:avro:1.11.3 +- com.fasterxml.jackson.core:jackson-core:2.14.2 +- com.fasterxml.jackson.core:jackson-databind:2.14.2 +- com.fasterxml.jackson.core:jackson-annotations:2.14.2 +- org.apache.commons:commons-compress:1.22 + +- org.apache.parquet:parquet-hadoop:1.13.1 +- org.apache.parquet:parquet-column:1.13.1 +- org.apache.parquet:parquet-common:1.13.1 +- org.apache.parquet:parquet-encoding:1.13.1 +- org.apache.parquet:parquet-format-structures:1.13.1 +- org.apache.parquet:parquet-jackson:1.13.1 +- commons-pool:commons-pool:1.6 + +This project bundles the following dependencies under the BSD license. +You find it under licenses/LICENSE.protobuf, licenses/LICENSE.zstd-jni +and licenses/LICENSE.threeten-extra + +- com.google.protobuf:protobuf-java:3.19.6 +- com.github.luben:zstd-jni:1.5.5-11 +- org.threeten:threeten-extra:1.7.1 + +# Jackson JSON processor + +Jackson is a high-performance, Free/Open Source JSON processing library. +It was originally written by Tatu Saloranta (tatu.saloranta@iki.fi), and has +been in development since 2007. +It is currently developed by a community of developers. + +## Licensing + +Jackson 2.x core and extension components are licensed under Apache License 2.0 +To find the details that apply to this artifact see the accompanying LICENSE file. + +## Credits + +A list of contributors may be found from CREDITS(-2.x) file, which is included +in some artifacts (usually source distributions); but is always available +from the source code management (SCM) system project uses. + +Apache Commons Lang +Copyright 2001-2011 The Apache Software Foundation + +This product includes software developed by +The Apache Software Foundation (http://www.apache.org/). + +ORC Core +Copyright 2013-2023 The Apache Software Foundation + +ORC Shims +Copyright 2013-2023 The Apache Software Foundation + +Apache Commons Lang +Copyright 2001-2021 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (https://www.apache.org/). + +Hive Storage API +Copyright 2020 The Apache Software Foundation + +Apache Avro +Copyright 2009-2023 The Apache Software Foundation + +Apache Commons Compress +Copyright 2002-2022 The Apache Software Foundation + +--- + +The files in the package org.apache.commons.compress.archivers.sevenz +were derived from the LZMA SDK, version 9.20 (C/ and CPP/7zip/), +which has been placed in the public domain: + +"LZMA SDK is placed in the public domain." (http://www.7-zip.org/sdk.html) + +The test file lbzip2_32767.bz2 has been copied from libbzip2's source +repository: + +This program, "bzip2", the associated library "libbzip2", and all +documentation, are copyright (C) 1996-2019 Julian R Seward. All +rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. The origin of this software must not be misrepresented; you must + not claim that you wrote the original software. If you use this + software in a product, an acknowledgment in the product + documentation would be appreciated but is not required. + +3. Altered source versions must be plainly marked as such, and must + not be misrepresented as being the original software. + +4. The name of the author may not be used to endorse or promote + products derived from this software without specific prior written + permission. + +THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS +OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE +GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Julian Seward, jseward@acm.org + +Apache Commons Pool +Copyright 2001-2012 The Apache Software Foundation + +Paimon : Hive Catalog +Copyright 2023-2024 The Apache Software Foundation + +Paimon : Hive Common +Copyright 2023-2024 The Apache Software Foundation + +paimon-shade-jackson-2 +Copyright 2023-2024 The Apache Software Foundation + +This project includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +This project bundles the following dependencies under the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- com.fasterxml.jackson.core:jackson-annotations:2.14.2 +- com.fasterxml.jackson.core:jackson-core:2.14.2 +- com.fasterxml.jackson.core:jackson-databind:2.14.2 +- com.fasterxml.jackson.dataformat:jackson-dataformat-csv:2.14.2 +- com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.14.2 +- com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.14.2 +- com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.14.2 +- org.yaml:snakeyaml:1.33 + +Jackson is a high-performance, Free/Open Source JSON processing library. +It was originally written by Tatu Saloranta (tatu.saloranta@iki.fi), and has +been in development since 2007. +It is currently developed by a community of developers, as well as supported +commercially by FasterXML.com. + +Jackson core and extension components may be licensed under different licenses. +To find the details that apply to this artifact see the accompanying LICENSE file. +For more information, including possible other licensing options, contact +FasterXML.com (http://fasterxml.com). + +A list of contributors may be found from CREDITS file, which is included +in some artifacts (usually source distributions); but is always available +from the source code management (SCM) system project uses. + +paimon-shade-guava-30 +Copyright 2023-2024 The Apache Software Foundation + +- com.google.guava:guava:30.1.1-jre +- com.google.guava:failureaccess:1.0.1 + +paimon-shade-caffeine-2 +Copyright 2023-2024 The Apache Software Foundation + +- com.github.ben-manes.caffeine:caffeine:2.9.3 + +Paimon : Arrow +Copyright 2023-2024 The Apache Software Foundation + +Apache Log4j 1.x Compatibility API +Copyright 1999-1969 The Apache Software Foundation + +Arrow Vectors +Copyright 2023 The Apache Software Foundation + +Arrow Format +Copyright 2023 The Apache Software Foundation + +Arrow Memory - Core +Copyright 2023 The Apache Software Foundation + +Arrow Memory - Unsafe +Copyright 2023 The Apache Software Foundation + +Arrow Java C Data Interface +Copyright 2023 The Apache Software Foundation + +Apache Flink +Copyright 2014-2024 The Apache Software Foundation + diff --git a/paimon_python_java/paimon-python-java-bridge/pom.xml b/paimon_python_java/paimon-python-java-bridge/pom.xml index 52bc061..5927b08 100644 --- a/paimon_python_java/paimon-python-java-bridge/pom.xml +++ b/paimon_python_java/paimon-python-java-bridge/pom.xml @@ -27,6 +27,7 @@ Paimon : Python-Java Bridge jar + 2024 0.9.0 @@ -132,21 +133,21 @@ - org.apache.maven.plugins - maven-compiler-plugin - 3.8.0 - - ${target.java.version} - ${target.java.version} - - false - - - -Xpkginfo:always - -Xlint:deprecation - - - + org.apache.maven.plugins + maven-compiler-plugin + 3.8.0 + + ${target.java.version} + ${target.java.version} + + false + + + -Xpkginfo:always + -Xlint:deprecation + + + org.apache.maven.plugins @@ -159,6 +160,9 @@ shade + false + false + true org.apache.paimon:paimon-bundle @@ -174,6 +178,18 @@ net.sf.py4j:py4j + + + + + + Apache Paimon + ${project.inceptionYear} + UTF-8 + + com.fasterxml.jackson @@ -184,7 +200,54 @@ + + + org.apache.maven.plugins + maven-clean-plugin + + + + ${project.basedir} + + dependency-reduced-pom.xml + + + + + - + + + + org.codehaus.mojo + exec-maven-plugin + 3.1.0 + false + + + check-license + + none + + java + + + + + org.apache.paimon.tools.ci.licensecheck.LicenseChecker + true + false + + + + org.apache.paimon + paimon-ci-tools + ${project.version} + + + + + + diff --git a/paimon_python_java/paimon-python-java-bridge/src/main/resources/META-INF/NOTICE b/paimon_python_java/paimon-python-java-bridge/src/main/resources/META-INF/NOTICE new file mode 100644 index 0000000..db5800c --- /dev/null +++ b/paimon_python_java/paimon-python-java-bridge/src/main/resources/META-INF/NOTICE @@ -0,0 +1,21 @@ +paimon-python-java-bridge +Copyright 2024 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +This project bundles the following dependencies under the under the MIT license +- org.slf4j:slf4j-api:1.7.32 + +This project bundles the following dependencies under the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt) +- org.apache.logging.log4j:log4j-1.2-api:2.17.1 +- org.apache.arrow:arrow-vector:14.0.0 +- org.apache.arrow:arrow-format:14.0.0 +- org.apache.arrow:arrow-memory-core:14.0.0 +- org.apache.arrow:arrow-memory-unsafe:14.0.0 +- org.apache.arrow:arrow-c-data:14.0.0 +- com.google.flatbuffers:flatbuffers-java:1.12.0 + +This project bundles the following dependencies under the 2-Clause BSD License +- net.sf.py4j:py4j:0.10.9.7 + diff --git a/paimon_python_java/paimon-python-java-bridge/tools/ci/log4j.properties b/paimon_python_java/paimon-python-java-bridge/tools/ci/log4j.properties new file mode 100644 index 0000000..7daf1c3 --- /dev/null +++ b/paimon_python_java/paimon-python-java-bridge/tools/ci/log4j.properties @@ -0,0 +1,43 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +rootLogger.level = INFO +rootLogger.appenderRef.out.ref = ConsoleAppender + +# ----------------------------------------------------------------------------- +# Console (use 'console') +# ----------------------------------------------------------------------------- + +appender.console.name = ConsoleAppender +appender.console.type = CONSOLE +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{HH:mm:ss,SSS} [%20t] %-5p %-60c %x - %m%n + +# ----------------------------------------------------------------------------- +# File (use 'file') +# ----------------------------------------------------------------------------- +appender.file.name = FileAppender +appender.file.type = FILE +appender.file.fileName = ${sys:log.dir}/mvn-${sys:mvn.forkNumber:-output}.log +appender.file.layout.type = PatternLayout +appender.file.layout.pattern = %d{HH:mm:ss,SSS} [%20t] %-5p %-60c %x - %m%n +appender.file.createOnDemand = true + +# suppress the irrelevant (wrong) warnings from the netty channel handler +logger.netty.name = org.jboss.netty.channel.DefaultChannelPipeline +logger.netty.level = ERROR diff --git a/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/pom.xml b/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/pom.xml new file mode 100644 index 0000000..7305e28 --- /dev/null +++ b/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/pom.xml @@ -0,0 +1,78 @@ + + + + + 4.0.0 + + org.apache.paimon + paimon-ci-tools + 0.1-SNAPSHOT + Paimon : Tools : CI : Java + + jar + + + true + 30.1.1-jre + 0.6.0-incubating + 2.17.1 + + + + + org.apache.paimon + paimon-shade-guava-30 + ${paimon.shade.guava.version}-${paimon.shade.version} + + + + org.apache.logging.log4j + log4j-slf4j-impl + ${log4j.version} + compile + + + org.apache.logging.log4j + log4j-api + ${log4j.version} + compile + + + org.apache.logging.log4j + log4j-core + ${log4j.version} + compile + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 8 + 8 + + + + + diff --git a/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/licensecheck/JarFileChecker.java b/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/licensecheck/JarFileChecker.java new file mode 100644 index 0000000..b829435 --- /dev/null +++ b/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/licensecheck/JarFileChecker.java @@ -0,0 +1,330 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.tools.ci.licensecheck; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.net.URI; +import java.nio.charset.MalformedInputException; +import java.nio.charset.StandardCharsets; +import java.nio.file.FileSystem; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** Checks the Jar files created by the build process. */ +public class JarFileChecker { + private static final Logger LOG = LoggerFactory.getLogger(JarFileChecker.class); + + public static int checkPath(Path path) throws Exception { + List files = getBuildJars(path); + LOG.info("Checking directory {} with a total of {} jar files.", path, files.size()); + + int severeIssues = 0; + for (Path file : files) { + severeIssues += checkJar(file); + } + + return severeIssues; + } + + private static List getBuildJars(Path path) throws IOException { + return Files.walk(path) + .filter(file -> file.toString().endsWith(".jar")) + .collect(Collectors.toList()); + } + + static int checkJar(Path file) throws Exception { + final URI uri = file.toUri(); + + int numSevereIssues = 0; + try (final FileSystem fileSystem = + FileSystems.newFileSystem( + new URI("jar:file", uri.getHost(), uri.getPath(), uri.getFragment()), + Collections.emptyMap())) { + if (isTestJarAndEmpty(file, fileSystem.getPath("/"))) { + return 0; + } + if (!noticeFileExistsAndIsValid(fileSystem.getPath("META-INF", "NOTICE"), file)) { + numSevereIssues++; + } + if (!licenseFileExistsAndIsValid(fileSystem.getPath("META-INF", "LICENSE"), file)) { + numSevereIssues++; + } + + numSevereIssues += + getNumLicenseFilesOutsideMetaInfDirectory(file, fileSystem.getPath("/")); + + numSevereIssues += getFilesWithIncompatibleLicenses(file, fileSystem.getPath("/")); + } + return numSevereIssues; + } + + private static boolean isTestJarAndEmpty(Path jar, Path jarRoot) throws IOException { + if (jar.getFileName().toString().endsWith("-tests.jar")) { + try (Stream files = Files.walk(jarRoot)) { + long numClassFiles = + files.filter(path -> !path.equals(jarRoot)) + .filter(path -> path.getFileName().toString().endsWith(".class")) + .count(); + if (numClassFiles == 0) { + return true; + } + } + } + + return false; + } + + private static boolean noticeFileExistsAndIsValid(Path noticeFile, Path jar) + throws IOException { + if (!Files.exists(noticeFile)) { + LOG.error("Missing META-INF/NOTICE in {}", jar); + return false; + } + + final String noticeFileContents = readFile(noticeFile); + if (!noticeFileContents.contains("The Apache Software Foundation")) { + LOG.error("The notice file in {} does not contain the expected entries.", jar); + return false; + } + + return true; + } + + private static boolean licenseFileExistsAndIsValid(Path licenseFile, Path jar) + throws IOException { + if (!Files.exists(licenseFile)) { + LOG.error("Missing META-INF/LICENSE in {}", jar); + return false; + } + + final String licenseFileContents = readFile(licenseFile); + if (!licenseFileContents.contains("Apache License") + || !licenseFileContents.contains("Version 2.0, January 2004")) { + LOG.error("The license file in {} does not contain the expected entries.", jar); + return false; + } + + return true; + } + + private static int getFilesWithIncompatibleLicenses(Path jar, Path jarRoot) throws IOException { + // patterns are based on https://www.apache.org/legal/resolved.html#category-x + return findNonBinaryFilesContainingText( + jar, + jarRoot, + asPatterns( + "Binary Code License", + "Intel Simplified Software License", + "JSR 275", + "Microsoft Limited Public License", + "Amazon Software License", + // Java SDK for Satori RTM license + "as necessary for your use of Satori services", + "REDIS SOURCE AVAILABLE LICENSE", + "Booz Allen Public License", + "Confluent Community License Agreement Version 1.0", + // “Commons Clause” License Condition v1.0 + "the License does not grant to you, the right to Sell the Software.", + "Sun Community Source License Version 3.0", + "GNU General Public License", + "GNU Affero General Public License", + "GNU Lesser General Public License", + "Q Public License", + "Sleepycat License", + "Server Side Public License", + "Code Project Open License", + // BSD 4-Clause + " All advertising materials mentioning features or use of this software must display the following acknowledgement", + // Facebook Patent clause v1 + "The license granted hereunder will terminate, automatically and without notice, for anyone that makes any claim", + // Facebook Patent clause v2 + "The license granted hereunder will terminate, automatically and without notice, if you (or any of your subsidiaries, corporate affiliates or agents) initiate directly or indirectly, or take a direct financial interest in, any Patent Assertion: (i) against Facebook", + "Netscape Public License", + "SOLIPSISTIC ECLIPSE PUBLIC LICENSE", + // DON'T BE A DICK PUBLIC LICENSE + "Do whatever you like with the original work, just don't be a dick.", + // JSON License + "The Software shall be used for Good, not Evil.", + // can sometimes be found in "funny" licenses + "Don’t be evil")); + } + + private static Collection asPatterns(String... texts) { + return Stream.of(texts) + .map(JarFileChecker::asPatternWithPotentialLineBreaks) + .collect(Collectors.toList()); + } + + private static Pattern asPatternWithPotentialLineBreaks(String text) { + // allows word sequences to be separated by whitespace, line-breaks and comments(//, #) + return Pattern.compile(text.toLowerCase(Locale.ROOT).replaceAll(" ", " ?\\\\R?[\\\\s/#]*")); + } + + private static int findNonBinaryFilesContainingText( + Path jar, Path jarRoot, Collection forbidden) throws IOException { + try (Stream files = Files.walk(jarRoot)) { + return files.filter(path -> !path.equals(jarRoot)) + .filter(path -> !Files.isDirectory(path)) + .filter(JarFileChecker::isNoClassFile) + // frequent false-positives due to dual-licensing; generated by maven + .filter(path -> !getFileName(path).equals("dependencies")) + // false-positives due to dual-licensing; use startsWith to cover .txt/.md files + .filter(path -> !getFileName(path).startsWith("license")) + // false-positives due to optional components; startsWith covers .txt/.md files + .filter(path -> !getFileName(path).startsWith("notice")) + // dual-licensed under GPL 2 and CDDL 1.1 + // contained in hadoop/presto S3 FS and paimon-dist + .filter( + path -> + !path.toString() + .contains("/META-INF/versions/11/javax/xml/bind")) + .filter(path -> !isJavaxManifest(jar, path)) + // dual-licensed under GPL 2 and EPL 2.0 + // contained in sql-avro-confluent-registry + .filter(path -> !pathStartsWith(path, "/org/glassfish/jersey/internal")) + .map( + path -> { + try { + final String fileContents; + try { + fileContents = readFile(path).toLowerCase(Locale.ROOT); + } catch (MalformedInputException mie) { + // binary file + return 0; + } + + int violations = 0; + for (Pattern text : forbidden) { + if (text.matcher(fileContents).find()) { + // do not count individual violations because it can be + // confusing when checking with aliases for the same + // license + violations = 1; + LOG.error( + "File '{}' in jar '{}' contains match with forbidden regex '{}'.", + path, + jar, + text); + } + } + return violations; + } catch (IOException e) { + throw new RuntimeException( + String.format( + "Could not read contents of file '%s' in jar '%s'.", + path, jar), + e); + } + }) + .reduce(Integer::sum) + .orElse(0); + } + } + + private static int getNumLicenseFilesOutsideMetaInfDirectory(Path jar, Path jarRoot) + throws IOException { + try (Stream files = Files.walk(jarRoot)) { + /* + * LICENSE or NOTICE files found outside of the META-INF directory are most likely shading mistakes (we are including the files from other dependencies, thus providing an invalid LICENSE file) + * + *

In such a case, we recommend updating the shading exclusions, and adding the license file to META-INF/licenses. + */ + final List filesWithIssues = + files.filter(path -> !path.equals(jarRoot)) + .filter( + path -> + getFileName(path).contains("license") + || getFileName(path).contains("notice")) + .filter( + path -> + !Files.isDirectory( + path)) // ignore directories, e.g. "license/" + .filter(JarFileChecker::isNoClassFile) // some class files contain + // LICENSE in their name + .filter( + path -> + !getFileName(path) + .endsWith(".ftl")) // a false positive in + // python + .map(Path::toString) + .filter( + path -> + !path.contains( + "META-INF")) // license files in META-INF are + // expected + .filter( + path -> + !path.endsWith( + "web/3rdpartylicenses.txt")) // a false positive + // in + // web + .collect(Collectors.toList()); + for (String fileWithIssue : filesWithIssues) { + LOG.error( + "Jar file {} contains a LICENSE file in an unexpected location: {}", + jar, + fileWithIssue); + } + return filesWithIssues.size(); + } + } + + private static String getFileName(Path path) { + return path.getFileName().toString().toLowerCase(); + } + + private static boolean pathStartsWith(Path file, String path) { + return file.startsWith(file.getFileSystem().getPath(path)); + } + + private static boolean equals(Path file, String path) { + return file.equals(file.getFileSystem().getPath(path)); + } + + private static boolean isNoClassFile(Path file) { + return !getFileName(file).endsWith(".class"); + } + + private static boolean isJavaxManifest(Path jar, Path potentialManifestFile) { + try { + return equals(potentialManifestFile, "/META-INF/versions/11/META-INF/MANIFEST.MF") + && readFile(potentialManifestFile).contains("Specification-Title: jaxb-api"); + } catch (IOException e) { + throw new RuntimeException( + String.format( + "Error while reading file %s from jar %s.", potentialManifestFile, jar), + e); + } + } + + private static String readFile(Path file) throws IOException { + return new String(Files.readAllBytes(file), StandardCharsets.UTF_8); + } +} diff --git a/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/licensecheck/LicenseChecker.java b/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/licensecheck/LicenseChecker.java new file mode 100644 index 0000000..9142f10 --- /dev/null +++ b/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/licensecheck/LicenseChecker.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.tools.ci.licensecheck; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.nio.file.Paths; + +/** Utility for checking all things related to License and Notice files. */ +public class LicenseChecker { + // ---------------------------------------- Launcher ---------------------------------------- // + + private static final Logger LOG = LoggerFactory.getLogger(LicenseChecker.class); + + public static void main(String[] args) throws Exception { + if (args.length < 2) { + System.out.println( + "Usage: LicenseChecker "); + System.exit(1); + } + LOG.warn( + "THIS UTILITY IS ONLY CHECKING FOR COMMON LICENSING MISTAKES. A MANUAL CHECK OF THE NOTICE FILES, DEPLOYED ARTIFACTS, ETC. IS STILL NEEDED!"); + + int severeIssueCount = NoticeFileChecker.run(new File(args[0]), Paths.get(args[1])); + + severeIssueCount += JarFileChecker.checkPath(Paths.get(args[2])); + + if (severeIssueCount > 0) { + LOG.warn("Found a total of {} severe license issues", severeIssueCount); + + System.exit(1); + } + LOG.info("License check completed without severe issues."); + } +} diff --git a/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/licensecheck/NoticeFileChecker.java b/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/licensecheck/NoticeFileChecker.java new file mode 100644 index 0000000..ebba7ce --- /dev/null +++ b/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/licensecheck/NoticeFileChecker.java @@ -0,0 +1,387 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.tools.ci.licensecheck; + +import org.apache.paimon.tools.ci.utils.dependency.DependencyParser; +import org.apache.paimon.tools.ci.utils.deploy.DeployParser; +import org.apache.paimon.tools.ci.utils.notice.NoticeContents; +import org.apache.paimon.tools.ci.utils.notice.NoticeParser; +import org.apache.paimon.tools.ci.utils.shade.ShadeParser; +import org.apache.paimon.tools.ci.utils.shared.Dependency; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.Nullable; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** Utility class checking for proper NOTICE files based on the maven build output. */ +public class NoticeFileChecker { + + private static final Logger LOG = LoggerFactory.getLogger(NoticeFileChecker.class); + + private static final List MODULES_DEFINING_EXCESS_DEPENDENCIES = + loadFromResources("modules-defining-excess-dependencies.modulelist"); + + // Examples: + // "- org.apache.htrace:htrace-core:3.1.0-incubating" + // or + // "This project bundles "net.jcip:jcip-annotations:1.0". + private static final Pattern NOTICE_DEPENDENCY_PATTERN = + Pattern.compile( + "- ([^ :]+):([^:]+):([^ ]+)($| )|.*bundles \"([^:]+):([^:]+):([^\"]+)\".*"); + + static int run(File buildResult, Path root) throws IOException { + // parse included dependencies from build output + final Map> modulesWithBundledDependencies = + combineAndFilterPaimonDependencies( + ShadeParser.parseShadeOutput(buildResult.toPath()), + DependencyParser.parseDependencyCopyOutput(buildResult.toPath())); + + final Set deployedModules = DeployParser.parseDeployOutput(buildResult); + + LOG.info( + "Extracted " + + deployedModules.size() + + " modules that were deployed and " + + modulesWithBundledDependencies.keySet().size() + + " modules which bundle dependencies with a total of " + + modulesWithBundledDependencies.values().size() + + " dependencies"); + + // find modules producing a shaded-jar + List noticeFiles = findNoticeFiles(root); + LOG.info("Found {} NOTICE files to check", noticeFiles.size()); + + final Map> moduleToNotice = + noticeFiles.stream() + .collect( + Collectors.toMap( + NoticeFileChecker::getModuleFromNoticeFile, + noticeFile -> { + try { + return NoticeParser.parseNoticeFile(noticeFile); + } catch (IOException e) { + // some machine issue + throw new RuntimeException(e); + } + })); + + return run(modulesWithBundledDependencies, deployedModules, moduleToNotice); + } + + static int run( + Map> modulesWithBundledDependencies, + Set deployedModules, + Map> noticeFiles) + throws IOException { + int severeIssueCount = 0; + + final Set modulesSkippingDeployment = + new HashSet<>(modulesWithBundledDependencies.keySet()); + modulesSkippingDeployment.removeAll(deployedModules); + + LOG.debug( + "The following {} modules are skipping deployment: {}", + modulesSkippingDeployment.size(), + modulesSkippingDeployment.stream() + .sorted() + .collect(Collectors.joining("\n\t", "\n\t", ""))); + + for (String moduleSkippingDeployment : modulesSkippingDeployment) { + // TODO: this doesn't work for modules requiring a NOTICE that are bundled indirectly + // TODO: via another non-deployed module + boolean bundledByDeployedModule = + modulesWithBundledDependencies.entrySet().stream() + .filter( + entry -> + entry.getValue().stream() + .map(Dependency::getArtifactId) + .anyMatch( + artifactId -> + artifactId.equals( + moduleSkippingDeployment))) + .anyMatch(entry -> !modulesSkippingDeployment.contains(entry.getKey())); + + if (!bundledByDeployedModule) { + modulesWithBundledDependencies.remove(moduleSkippingDeployment); + } else { + LOG.debug( + "Including module {} in license checks, despite not being deployed, because it is bundled by another deployed module.", + moduleSkippingDeployment); + } + } + + // check that all required NOTICE files exists + severeIssueCount += + ensureRequiredNoticeFiles(modulesWithBundledDependencies, noticeFiles.keySet()); + + // check each NOTICE file + for (Map.Entry> noticeFile : noticeFiles.entrySet()) { + severeIssueCount += + checkNoticeFileAndLogProblems( + modulesWithBundledDependencies, + noticeFile.getKey(), + noticeFile.getValue().orElse(null)); + } + + return severeIssueCount; + } + + private static Map> combineAndFilterPaimonDependencies( + Map> modulesWithBundledDependencies, + Map> modulesWithCopiedDependencies) { + + final Map> combinedAndFiltered = new LinkedHashMap<>(); + + Stream.concat( + modulesWithBundledDependencies.entrySet().stream(), + modulesWithCopiedDependencies.entrySet().stream()) + .forEach( + (entry) -> { + final Set dependencies = + combinedAndFiltered.computeIfAbsent( + entry.getKey(), ignored -> new LinkedHashSet<>()); + + for (Dependency dependency : entry.getValue()) { + if (!dependency.getGroupId().contains("org.apache.paimon")) { + dependencies.add(dependency); + } + } + }); + + return combinedAndFiltered; + } + + private static int ensureRequiredNoticeFiles( + Map> modulesWithShadedDependencies, + Collection modulesWithNoticeFile) { + int severeIssueCount = 0; + Set shadingModules = new HashSet<>(modulesWithShadedDependencies.keySet()); + shadingModules.removeAll(modulesWithNoticeFile); + for (String moduleWithoutNotice : shadingModules) { + if (modulesWithShadedDependencies.get(moduleWithoutNotice).stream() + .anyMatch(dependency -> !dependency.getGroupId().equals("org.apache.paimon"))) { + LOG.error( + "Module {} is missing a NOTICE file. It has shaded dependencies: {}", + moduleWithoutNotice, + modulesWithShadedDependencies.get(moduleWithoutNotice).stream() + .map(Dependency::toString) + .collect(Collectors.joining("\n\t", "\n\t", ""))); + severeIssueCount++; + } + } + return severeIssueCount; + } + + private static String getModuleFromNoticeFile(Path noticeFile) { + Path moduleDirectory = + noticeFile + .getParent() // META-INF + .getParent() // resources + .getParent() // main + .getParent() // src + .getParent(); // <-- module name + return moduleDirectory.getFileName().toString(); + } + + private static int checkNoticeFileAndLogProblems( + Map> modulesWithShadedDependencies, + String moduleName, + @Nullable NoticeContents noticeContents) + throws IOException { + + final Map> problemsBySeverity = + checkNoticeFile(modulesWithShadedDependencies, moduleName, noticeContents); + + final List severeProblems = + problemsBySeverity.getOrDefault(Severity.CRITICAL, Collections.emptyList()); + + if (!problemsBySeverity.isEmpty()) { + final List toleratedProblems = + problemsBySeverity.getOrDefault(Severity.TOLERATED, Collections.emptyList()); + final List expectedProblems = + problemsBySeverity.getOrDefault(Severity.SUPPRESSED, Collections.emptyList()); + + LOG.info( + "Problems were detected for a NOTICE file.\n" + "\t{}:\n" + "{}{}{}", + moduleName, + convertProblemsToIndentedString( + severeProblems, + "These issue are legally problematic and MUST be fixed:"), + convertProblemsToIndentedString( + toleratedProblems, + "These issues are mistakes that aren't legally problematic. They SHOULD be fixed at some point, but we don't have to:"), + convertProblemsToIndentedString( + expectedProblems, "These issues are assumed to be false-positives:")); + } + + return severeProblems.size(); + } + + static Map> checkNoticeFile( + Map> modulesWithShadedDependencies, + String moduleName, + @Nullable NoticeContents noticeContents) { + + final Map> problemsBySeverity = new HashMap<>(); + + if (noticeContents == null) { + addProblem(problemsBySeverity, Severity.CRITICAL, "The NOTICE file was empty."); + } else { + // first line must be the module name. + if (!noticeContents.getNoticeModuleName().equals(moduleName)) { + addProblem( + problemsBySeverity, + Severity.TOLERATED, + String.format( + "First line does not start with module name. firstLine=%s", + noticeContents.getNoticeModuleName())); + } + + // collect all declared dependencies from NOTICE file + Set declaredDependencies = new HashSet<>(); + for (Dependency declaredDependency : noticeContents.getDeclaredDependencies()) { + if (!declaredDependencies.add(declaredDependency)) { + addProblem( + problemsBySeverity, + Severity.CRITICAL, + String.format("Dependency %s is declared twice.", declaredDependency)); + } + } + + // find all dependencies missing from NOTICE file + Collection expectedDependencies = + modulesWithShadedDependencies.getOrDefault(moduleName, Collections.emptySet()) + .stream() + .filter( + dependency -> + !dependency.getGroupId().equals("org.apache.paimon")) + .collect(Collectors.toList()); + + for (Dependency expectedDependency : expectedDependencies) { + if (!declaredDependencies.contains(expectedDependency)) { + addProblem( + problemsBySeverity, + Severity.CRITICAL, + String.format("Dependency %s is not listed.", expectedDependency)); + } + } + + boolean moduleDefinesExcessDependencies = + MODULES_DEFINING_EXCESS_DEPENDENCIES.contains(moduleName); + + // find all dependencies defined in NOTICE file, which were not expected + for (Dependency declaredDependency : declaredDependencies) { + if (!expectedDependencies.contains(declaredDependency)) { + final Severity severity = + moduleDefinesExcessDependencies + ? Severity.SUPPRESSED + : Severity.TOLERATED; + addProblem( + problemsBySeverity, + severity, + String.format( + "Dependency %s is not bundled, but listed.", + declaredDependency)); + } + } + } + + return problemsBySeverity; + } + + private static void addProblem( + Map> problemsBySeverity, Severity severity, String problem) { + problemsBySeverity.computeIfAbsent(severity, ignored -> new ArrayList<>()).add(problem); + } + + enum Severity { + /** Issues that a legally problematic which must be fixed. */ + CRITICAL, + /** Issues that affect the correctness but aren't legally problematic. */ + TOLERATED, + /** Issues where we intentionally break the rules. */ + SUPPRESSED + } + + private static String convertProblemsToIndentedString(List problems, String header) { + return problems.isEmpty() + ? "" + : problems.stream() + .map(s -> "\t\t\t" + s) + .collect(Collectors.joining("\n", "\t\t " + header + " \n", "\n")); + } + + private static List findNoticeFiles(Path root) throws IOException { + return Files.walk(root) + .filter( + file -> { + int nameCount = file.getNameCount(); + return file.getName(nameCount - 3).toString().equals("resources") + && file.getName(nameCount - 2).toString().equals("META-INF") + && file.getName(nameCount - 1).toString().equals("NOTICE"); + }) + .collect(Collectors.toList()); + } + + private static List loadFromResources(String fileName) { + try { + try (BufferedReader bufferedReader = + new BufferedReader( + new InputStreamReader( + Objects.requireNonNull( + NoticeFileChecker.class.getResourceAsStream( + "/" + fileName))))) { + + List result = + bufferedReader + .lines() + .filter(line -> !line.startsWith("#") && !line.isEmpty()) + .collect(Collectors.toList()); + LOG.debug("Loaded {} items from resource {}", result.size(), fileName); + return result; + } + } catch (Throwable e) { + // wrap anything in a RuntimeException to be callable from the static initializer + throw new RuntimeException("Error while loading resource", e); + } + } +} diff --git a/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/suffixcheck/ScalaSuffixChecker.java b/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/suffixcheck/ScalaSuffixChecker.java new file mode 100644 index 0000000..792dc3c --- /dev/null +++ b/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/suffixcheck/ScalaSuffixChecker.java @@ -0,0 +1,260 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.tools.ci.suffixcheck; + +import org.apache.paimon.tools.ci.utils.dependency.DependencyParser; +import org.apache.paimon.tools.ci.utils.shared.Dependency; +import org.apache.paimon.tools.ci.utils.shared.DependencyTree; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** Utility for checking the presence/absence of scala-suffixes. */ +public class ScalaSuffixChecker { + private static final Logger LOG = LoggerFactory.getLogger(ScalaSuffixChecker.class); + + // [INFO] --- maven-dependency-plugin:3.1.1:tree (default-cli) @ paimon-annotations --- + private static final Pattern moduleNamePattern = + Pattern.compile(".* --- maven-dependency-plugin.* @ (.*) ---.*"); + + // [INFO] +- junit:junit:jar:4.13.2:test + // [INFO] | \- org.hamcrest:hamcrest-core:jar:1.3:test + // [INFO] \- org.apache.logging.log4j:log4j-1.2-api:jar:2.14.1:test + private static final Pattern blockPattern = Pattern.compile(".* [+|\\\\].*"); + + // [INFO] +- org.scala-lang:scala-reflect:jar:2.11.12:test + private static final Pattern scalaSuffixPattern = Pattern.compile("_2.1[0-9]"); + + private static final Set EXCLUDED_MODULES = new HashSet<>(Arrays.asList()); + + public static void main(String[] args) throws IOException { + if (args.length < 2) { + System.out.println("Usage: ScalaSuffixChecker "); + System.exit(1); + } + + final Path mavenOutputPath = Paths.get(args[0]); + final Path paimonRootPath = Paths.get(args[1]); + + final ParseResult parseResult = parseMavenOutput(mavenOutputPath); + if (parseResult.getCleanModules().isEmpty()) { + LOG.error("Parsing found 0 scala-free modules; the parsing is likely broken."); + System.exit(1); + } + if (parseResult.getInfectedModules().isEmpty()) { + LOG.error("Parsing found 0 scala-dependent modules; the parsing is likely broken."); + System.exit(1); + } + + final Collection violations = checkScalaSuffixes(parseResult, paimonRootPath); + + if (!violations.isEmpty()) { + LOG.error( + "Violations found:{}", + violations.stream().collect(Collectors.joining("\n\t", "\n\t", ""))); + System.exit(1); + } + } + + private static ParseResult parseMavenOutput(final Path path) throws IOException { + final Set cleanModules = new HashSet<>(); + final Set infectedModules = new HashSet<>(); + + final Map dependenciesByModule = + DependencyParser.parseDependencyTreeOutput(path); + + for (String module : dependenciesByModule.keySet()) { + final String moduleName = stripScalaSuffix(module); + if (isExcluded(moduleName)) { + continue; + } + LOG.trace("Processing module '{}'.", moduleName); + + final List dependencies = + dependenciesByModule.get(module).flatten().collect(Collectors.toList()); + + boolean infected = false; + for (Dependency dependency : dependencies) { + final boolean dependsOnScala = dependsOnScala(dependency); + final boolean isTestDependency = dependency.getScope().get().equals("test"); + final boolean isExcluded = isExcluded(dependency.getArtifactId()); + LOG.trace("\tdependency:{}", dependency); + LOG.trace("\t\tdepends-on-scala:{}", dependsOnScala); + LOG.trace("\t\tis-test-dependency:{}", isTestDependency); + LOG.trace("\t\tis-excluded:{}", isExcluded); + if (dependsOnScala && !isTestDependency && !isExcluded) { + LOG.trace("\t\tOutbreak detected at {}!", moduleName); + infected = true; + } + } + + if (infected) { + infectedModules.add(moduleName); + } else { + cleanModules.add(moduleName); + } + } + + return new ParseResult(cleanModules, infectedModules); + } + + private static String stripScalaSuffix(final String moduleName) { + final int i = moduleName.indexOf("_2."); + return i > 0 ? moduleName.substring(0, i) : moduleName; + } + + private static boolean dependsOnScala(final Dependency dependency) { + return dependency.getGroupId().contains("org.scala-lang") + || scalaSuffixPattern.matcher(dependency.getArtifactId()).find(); + } + + private static Collection checkScalaSuffixes( + final ParseResult parseResult, Path paimonRootPath) throws IOException { + final Collection violations = new ArrayList<>(); + + // exclude e2e modules and paimon-docs for convenience as they + // a) are not deployed during a release + // b) exist only for dev purposes + // c) no-one should depend on them + final Collection excludedModules = new ArrayList<>(); + excludedModules.add("paimon-docs"); + excludedModules.addAll(getEndToEndTestModules(paimonRootPath)); + + for (String excludedModule : excludedModules) { + parseResult.getCleanModules().remove(excludedModule); + parseResult.getInfectedModules().remove(excludedModule); + } + + violations.addAll(checkCleanModules(parseResult.getCleanModules(), paimonRootPath)); + violations.addAll(checkInfectedModules(parseResult.getInfectedModules(), paimonRootPath)); + + return violations; + } + + private static Collection getEndToEndTestModules(Path paimonRootPath) + throws IOException { + try (Stream pathStream = Files.walk(paimonRootPath.resolve("paimon-e2e-tests"), 5)) { + return pathStream + .filter(path -> path.getFileName().toString().equals("pom.xml")) + .map(path -> path.getParent().getFileName().toString()) + .collect(Collectors.toList()); + } + } + + private static Collection checkCleanModules( + Collection modules, Path paimonRootPath) throws IOException { + return checkModules( + modules, + paimonRootPath, + "_${scala.binary.version}", + "Scala-free module '%s' is referenced with scala suffix in '%s'."); + } + + private static Collection checkInfectedModules( + Collection modules, Path paimonRootPath) throws IOException { + return checkModules( + modules, + paimonRootPath, + "", + "Scala-dependent module '%s' is referenced without scala suffix in '%s'."); + } + + private static Collection checkModules( + Collection modules, + Path paimonRootPath, + String moduleSuffix, + String violationTemplate) + throws IOException { + + final ArrayList sortedModules = new ArrayList<>(modules); + sortedModules.sort(String::compareTo); + + final Collection violations = new ArrayList<>(); + for (String module : sortedModules) { + int numPreviousViolations = violations.size(); + try (Stream pathStream = Files.walk(paimonRootPath, 3)) { + final List pomFiles = + pathStream + .filter(path -> path.getFileName().toString().equals("pom.xml")) + .collect(Collectors.toList()); + + for (Path pomFile : pomFiles) { + try (Stream lines = Files.lines(pomFile, StandardCharsets.UTF_8)) { + final boolean existsCleanReference = + lines.anyMatch( + line -> + line.contains( + module + moduleSuffix + "")); + + if (existsCleanReference) { + violations.add( + String.format( + violationTemplate, + module, + paimonRootPath.relativize(pomFile))); + } + } + } + } + if (numPreviousViolations == violations.size()) { + LOG.info("OK {}", module); + } + } + return violations; + } + + private static boolean isExcluded(String line) { + return EXCLUDED_MODULES.stream().anyMatch(line::contains); + } + + private static class ParseResult { + + private final Set cleanModules; + private final Set infectedModules; + + private ParseResult(Set cleanModules, Set infectedModules) { + this.cleanModules = cleanModules; + this.infectedModules = infectedModules; + } + + public Set getCleanModules() { + return cleanModules; + } + + public Set getInfectedModules() { + return infectedModules; + } + } +} diff --git a/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/utils/dependency/DependencyParser.java b/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/utils/dependency/DependencyParser.java new file mode 100644 index 0000000..46db112 --- /dev/null +++ b/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/utils/dependency/DependencyParser.java @@ -0,0 +1,224 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.tools.ci.utils.dependency; + +import org.apache.paimon.tools.ci.utils.shared.Dependency; +import org.apache.paimon.tools.ci.utils.shared.DependencyTree; +import org.apache.paimon.tools.ci.utils.shared.ParserUtils; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Iterator; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.Stack; +import java.util.function.Function; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Stream; + +/** Parsing utils for the Maven dependency plugin. */ +public class DependencyParser { + + private static final Pattern DEPENDENCY_COPY_NEXT_MODULE_PATTERN = + Pattern.compile( + ".*maven-dependency-plugin:[^:]+:copy .* @ (?[^ _]+)(?:_[0-9.]+)? --.*"); + + private static final Pattern DEPENDENCY_TREE_NEXT_MODULE_PATTERN = + Pattern.compile( + ".*maven-dependency-plugin:[^:]+:tree .* @ (?[^ _]+)(?:_[0-9.]+)? --.*"); + + /** See {@link DependencyParserTreeTest} for examples. */ + private static final Pattern DEPENDENCY_TREE_ITEM_PATTERN = + Pattern.compile( + ".* +" + + "(?.*?):" + + "(?.*?):" + + "(?.*?):" + + "(?:(?.*?):)?" + + "(?.*?):" + + "(?[^ ]*)" + + "(? \\(optional\\))?"); + + /** See {@link DependencyParserCopyTest} for examples. */ + private static final Pattern DEPENDENCY_COPY_ITEM_PATTERN = + Pattern.compile( + ".* Configured Artifact: +" + + "(?.*?):" + + "(?.*?):" + + "(?:(?.*?):)?" + + "(?.*?):" + + "(?:\\?:)?" // unknown cause; e.g.: javax.xml.bind:jaxb-api:?:jar + + "(?.*)"); + + /** + * Parses the output of a Maven build where {@code dependency:copy} was used, and returns a set + * of copied dependencies for each module. + * + *

The returned dependencies will NEVER contain the scope or optional flag. + */ + public static Map> parseDependencyCopyOutput(Path buildOutput) + throws IOException { + return processLines(buildOutput, DependencyParser::parseDependencyCopyOutput); + } + + /** + * Parses the output of a Maven build where {@code dependency:tree} was used, and returns a set + * of dependencies for each module. + */ + public static Map parseDependencyTreeOutput(Path buildOutput) + throws IOException { + return processLines(buildOutput, DependencyParser::parseDependencyTreeOutput); + } + + private static X processLines(Path buildOutput, Function, X> processor) + throws IOException { + try (Stream lines = Files.lines(buildOutput)) { + return processor.apply(lines.filter(line -> line.contains("[INFO]"))); + } + } + + static Map> parseDependencyCopyOutput(Stream lines) { + return ParserUtils.parsePluginOutput( + lines, + DEPENDENCY_COPY_NEXT_MODULE_PATTERN, + DependencyParser::parseCopyDependencyBlock); + } + + static Map parseDependencyTreeOutput(Stream lines) { + return ParserUtils.parsePluginOutput( + lines, + DEPENDENCY_TREE_NEXT_MODULE_PATTERN, + DependencyParser::parseTreeDependencyBlock); + } + + private static Set parseCopyDependencyBlock(Iterator block) { + final Set dependencies = new LinkedHashSet<>(); + + Optional parsedDependency = parseCopyDependency(block.next()); + while (parsedDependency.isPresent()) { + dependencies.add(parsedDependency.get()); + + if (block.hasNext()) { + parsedDependency = parseCopyDependency(block.next()); + } else { + parsedDependency = Optional.empty(); + } + } + + return dependencies; + } + + private static DependencyTree parseTreeDependencyBlock(Iterator block) { + // discard one line, which only contains the current module name + block.next(); + + if (!block.hasNext()) { + throw new IllegalStateException("Expected more output from the dependency-plugin."); + } + + final DependencyTree dependencies = new DependencyTree(); + + final Stack parentStack = new Stack<>(); + final Stack treeDepthStack = new Stack<>(); + String line = block.next(); + Optional parsedDependency = parseTreeDependency(line); + while (parsedDependency.isPresent()) { + int treeDepth = getDepth(line); + + while (!treeDepthStack.isEmpty() && treeDepth <= treeDepthStack.peek()) { + parentStack.pop(); + treeDepthStack.pop(); + } + + final Dependency dependency = parsedDependency.get(); + + if (parentStack.isEmpty()) { + dependencies.addDirectDependency(dependency); + } else { + dependencies.addTransitiveDependencyTo(dependency, parentStack.peek()); + } + + if (treeDepthStack.isEmpty() || treeDepth > treeDepthStack.peek()) { + treeDepthStack.push(treeDepth); + parentStack.push(dependency); + } + + if (block.hasNext()) { + line = block.next(); + parsedDependency = parseTreeDependency(line); + } else { + parsedDependency = Optional.empty(); + } + } + + return dependencies; + } + + static Optional parseCopyDependency(String line) { + Matcher dependencyMatcher = DEPENDENCY_COPY_ITEM_PATTERN.matcher(line); + if (!dependencyMatcher.find()) { + return Optional.empty(); + } + + return Optional.of( + Dependency.create( + dependencyMatcher.group("groupId"), + dependencyMatcher.group("artifactId"), + dependencyMatcher.group("version"), + dependencyMatcher.group("classifier"))); + } + + static Optional parseTreeDependency(String line) { + Matcher dependencyMatcher = DEPENDENCY_TREE_ITEM_PATTERN.matcher(line); + if (!dependencyMatcher.find()) { + return Optional.empty(); + } + + return Optional.of( + Dependency.create( + dependencyMatcher.group("groupId"), + dependencyMatcher.group("artifactId"), + dependencyMatcher.group("version"), + dependencyMatcher.group("classifier"), + dependencyMatcher.group("scope"), + dependencyMatcher.group("optional") != null)); + } + + /** + * The depths returned by this method do NOT return a continuous sequence. + * + * 

+     * +- org.apache.paimon:...
+     * |  +- org.apache.paimon:...
+     * |  |  \- org.apache.paimon:...
+     * ...
+     *
+ */ + private static int getDepth(String line) { + final int level = line.indexOf('+'); + if (level != -1) { + return level; + } + return line.indexOf('\\'); + } +} diff --git a/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/utils/deploy/DeployParser.java b/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/utils/deploy/DeployParser.java new file mode 100644 index 0000000..2975f92 --- /dev/null +++ b/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/utils/deploy/DeployParser.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.tools.ci.utils.deploy; + +import org.apache.paimon.tools.ci.utils.shared.ParserUtils; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** Parsing utils for the Maven deploy plugin. */ +public class DeployParser { + + // Examples: + // + // Deployment on CI with alternative repo + // [INFO] --- maven-deploy-plugin:2.8.2:deploy (default-deploy) @ paimon-parent --- + // [INFO] Using alternate deployment repository.../tmp/paimon-validation-deployment + // + // Skipped deployment: + // [INFO] --- maven-deploy-plugin:2.8.2:deploy (default-deploy) @ paimon-parent --- + // [INFO] Skipping artifact deployment + private static final Pattern DEPLOY_MODULE_PATTERN = + Pattern.compile( + ".maven-deploy-plugin:.*:deploy .* @ (?[^ _]+)(?:_[0-9.]+)? --.*"); + + /** + * Parses the output of a Maven build where {@code deploy:deploy} was used, and returns a set of + * deployed modules. + */ + public static Set parseDeployOutput(File buildResult) throws IOException { + try (Stream linesStream = Files.lines(buildResult.toPath())) { + return parseDeployOutput(linesStream); + } + } + + static Set parseDeployOutput(Stream lines) { + return ParserUtils.parsePluginOutput( + lines, DEPLOY_MODULE_PATTERN, DeployParser::parseDeployBlock) + .entrySet().stream() + .filter(Map.Entry::getValue) + .map(Map.Entry::getKey) + .collect(Collectors.toSet()); + } + + private static boolean parseDeployBlock(Iterator block) { + return block.hasNext() && !block.next().contains("Skipping artifact deployment"); + } +} diff --git a/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/utils/notice/NoticeContents.java b/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/utils/notice/NoticeContents.java new file mode 100644 index 0000000..8dadaa3 --- /dev/null +++ b/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/utils/notice/NoticeContents.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.tools.ci.utils.notice; + +import org.apache.paimon.tools.ci.utils.shared.Dependency; + +import java.util.Collection; + +/** Represents the parsed contents of a NOTICE file. */ +public class NoticeContents { + private final String noticeModuleName; + private final Collection declaredDependencies; + + public NoticeContents(String noticeModuleName, Collection declaredDependencies) { + this.noticeModuleName = noticeModuleName; + this.declaredDependencies = declaredDependencies; + } + + public String getNoticeModuleName() { + return noticeModuleName; + } + + public Collection getDeclaredDependencies() { + return declaredDependencies; + } +} diff --git a/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/utils/notice/NoticeParser.java b/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/utils/notice/NoticeParser.java new file mode 100644 index 0000000..370c46b --- /dev/null +++ b/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/utils/notice/NoticeParser.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.tools.ci.utils.notice; + +import org.apache.paimon.tools.ci.utils.shared.Dependency; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Optional; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** Parsing utils for NOTICE files. */ +public class NoticeParser { + + // "- org.apache.htrace:htrace-core:3.1.0-incubating" + private static final Pattern NOTICE_DEPENDENCY_PATTERN = + Pattern.compile( + "- " + + "(?[^ ]*?):" + + "(?[^ ]*?):" + + "(?:(?[^ ]*?):)?" + + "(?[^ ]*?)" + + "($| )"); + // "This project bundles "net.jcip:jcip-annotations:1.0". + private static final Pattern NOTICE_BUNDLES_DEPENDENCY_PATTERN = + Pattern.compile( + ".*bundles \"" + + "(?[^ ]*?):" + + "(?[^ ]*?):" + + "(?:(?[^ ]*?):)?" + + "(?[^ ]*?)" + + "\".*"); + + public static Optional parseNoticeFile(Path noticeFile) throws IOException { + // 1st line contains module name + final List noticeContents = Files.readAllLines(noticeFile); + + return parseNoticeFile(noticeContents); + } + + static Optional parseNoticeFile(List noticeContents) { + if (noticeContents.isEmpty()) { + return Optional.empty(); + } + + final String noticeModuleName = noticeContents.get(0); + + Collection declaredDependencies = new ArrayList<>(); + for (String line : noticeContents) { + Optional dependency = tryParsing(NOTICE_DEPENDENCY_PATTERN, line); + if (!dependency.isPresent()) { + dependency = tryParsing(NOTICE_BUNDLES_DEPENDENCY_PATTERN, line); + } + dependency.ifPresent(declaredDependencies::add); + } + + return Optional.of(new NoticeContents(noticeModuleName, declaredDependencies)); + } + + private static Optional tryParsing(Pattern pattern, String line) { + Matcher matcher = pattern.matcher(line); + if (matcher.find()) { + String groupId = matcher.group("groupId"); + String artifactId = matcher.group("artifactId"); + String version = matcher.group("version"); + String classifier = matcher.group("classifier"); + return Optional.of(Dependency.create(groupId, artifactId, version, classifier)); + } + return Optional.empty(); + } +} diff --git a/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/utils/shade/ShadeParser.java b/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/utils/shade/ShadeParser.java new file mode 100644 index 0000000..f5a6056 --- /dev/null +++ b/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/utils/shade/ShadeParser.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.tools.ci.utils.shade; + +import org.apache.paimon.tools.ci.utils.shared.Dependency; +import org.apache.paimon.tools.ci.utils.shared.ParserUtils; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Iterator; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Stream; + +/** Utils for parsing the shade-plugin output. */ +public final class ShadeParser { + + private static final Pattern SHADE_NEXT_MODULE_PATTERN = + Pattern.compile( + ".*:shade \\((?:shade-paimon|shade-dist|default)\\) @ (?[^ _]+)(?:_[0-9.]+)? --.*"); + + private static final Pattern SHADE_INCLUDE_MODULE_PATTERN = + Pattern.compile( + ".* " + + "(?.*?):" + + "(?.*?):" + + "(?.*?):" + + "(?:(?.*?):)?" + + "(?.*?)" + + " in the shaded jar"); + + /** + * Parses the output of a Maven build where {@code shade:shade} was used, and returns a set of + * bundled dependencies for each module. + * + *

The returned dependencies will NEVER contain the scope or optional flag. + * + *

This method only considers the {@code shade-paimon} and {@code shade-dist} executions, + * because all artifacts we produce that are either published or referenced are created by these + * executions. In other words, all artifacts from other executions are only used internally by + * the module that created them. + */ + public static Map> parseShadeOutput(Path buildOutput) + throws IOException { + try (Stream lines = Files.lines(buildOutput)) { + return parseShadeOutput(lines); + } + } + + static Map> parseShadeOutput(Stream lines) { + return ParserUtils.parsePluginOutput( + lines.filter(line -> !line.contains(" Excluding ")), + SHADE_NEXT_MODULE_PATTERN, + ShadeParser::parseBlock); + } + + private static Set parseBlock(Iterator block) { + final Set dependencies = new LinkedHashSet<>(); + + Optional parsedDependency = parseDependency(block.next()); + while (parsedDependency.isPresent()) { + dependencies.add(parsedDependency.get()); + + if (block.hasNext()) { + parsedDependency = parseDependency(block.next()); + } else { + parsedDependency = Optional.empty(); + } + } + + return dependencies; + } + + static Optional parseDependency(String line) { + Matcher dependencyMatcher = SHADE_INCLUDE_MODULE_PATTERN.matcher(line); + if (!dependencyMatcher.find()) { + return Optional.empty(); + } + + return Optional.of( + Dependency.create( + dependencyMatcher.group("groupId"), + dependencyMatcher.group("artifactId"), + dependencyMatcher.group("version"), + dependencyMatcher.group("classifier"))); + } + + private ShadeParser() {} +} diff --git a/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/utils/shared/Dependency.java b/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/utils/shared/Dependency.java new file mode 100644 index 0000000..fad1ac2 --- /dev/null +++ b/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/utils/shared/Dependency.java @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.tools.ci.utils.shared; + +import javax.annotation.Nullable; + +import java.util.Objects; +import java.util.Optional; + +/** + * Represents a dependency. + * + *

For some properties we return an {@link Optional}, for those that, depending on the plugin + * goal, may not be determinable. For example, {@code dependency:copy} never prints the scope or + * optional flag. + */ +public final class Dependency { + + private final String groupId; + private final String artifactId; + private final String version; + @Nullable private final String classifier; + @Nullable private final String scope; + @Nullable private final Boolean isOptional; + + private Dependency( + String groupId, + String artifactId, + String version, + @Nullable String classifier, + @Nullable String scope, + @Nullable Boolean isOptional) { + this.groupId = Objects.requireNonNull(groupId); + this.artifactId = Objects.requireNonNull(artifactId); + this.version = Objects.requireNonNull(version); + this.classifier = classifier; + this.scope = scope; + this.isOptional = isOptional; + } + + public static Dependency create( + String groupId, + String artifactId, + String version, + String classifier, + String scope, + boolean isOptional) { + return new Dependency( + groupId, + artifactId, + version, + classifier, + Objects.requireNonNull(scope), + isOptional); + } + + public static Dependency create( + String groupId, String artifactId, String version, String classifier) { + return new Dependency(groupId, artifactId, version, classifier, null, null); + } + + public String getGroupId() { + return groupId; + } + + public String getArtifactId() { + return artifactId; + } + + public String getVersion() { + return version; + } + + public Optional getClassifier() { + return Optional.ofNullable(classifier); + } + + public Optional getScope() { + return Optional.ofNullable(scope); + } + + public Optional isOptional() { + return Optional.ofNullable(isOptional); + } + + @Override + public String toString() { + return groupId + + ":" + + artifactId + + ":" + + version + + (classifier != null ? ":" + classifier : "") + + (scope != null ? ":" + scope : "") + + (isOptional != null && isOptional ? " (optional)" : ""); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + Dependency that = (Dependency) o; + return Objects.equals(groupId, that.groupId) + && Objects.equals(artifactId, that.artifactId) + && Objects.equals(version, that.version) + && Objects.equals(classifier, that.classifier) + && Objects.equals(scope, that.scope) + && Objects.equals(isOptional, that.isOptional); + } + + @Override + public int hashCode() { + return Objects.hash(groupId, artifactId, version, classifier, scope, isOptional); + } +} diff --git a/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/utils/shared/DependencyTree.java b/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/utils/shared/DependencyTree.java new file mode 100644 index 0000000..b7afe11 --- /dev/null +++ b/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/utils/shared/DependencyTree.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.tools.ci.utils.shared; + +import org.apache.paimon.shade.guava30.com.google.common.graph.Traverser; + +import javax.annotation.Nullable; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; + +/** + * Represents a dependency tree. + * + *

Every dependency can only occur exactly once. + */ +public class DependencyTree { + + private final Map lookup = new LinkedHashMap<>(); + private final List directDependencies = new ArrayList<>(); + + public void addDirectDependency(Dependency dependency) { + final String key = getKey(dependency); + if (lookup.containsKey(key)) { + return; + } + final Node node = new Node(dependency, null); + + lookup.put(key, node); + directDependencies.add(node); + } + + public void addTransitiveDependencyTo(Dependency transitiveDependency, Dependency parent) { + final String key = getKey(transitiveDependency); + if (lookup.containsKey(key)) { + return; + } + final Node node = lookup.get(getKey(parent)).addTransitiveDependency(transitiveDependency); + + lookup.put(key, node); + } + + private static final class Node { + private final Dependency dependency; + @Nullable private final Node parent; + private final List children = new ArrayList<>(); + + private Node(Dependency dependency, @Nullable Node parent) { + this.dependency = dependency; + this.parent = parent; + } + + public Node addTransitiveDependency(Dependency dependency) { + final Node node = new Node(dependency, this); + this.children.add(node); + return node; + } + + private boolean isRoot() { + return parent == null; + } + } + + public List getPathTo(Dependency dependency) { + final LinkedList path = new LinkedList<>(); + + Node node = lookup.get(getKey(dependency)); + path.addFirst(node.dependency); + while (!node.isRoot()) { + node = node.parent; + path.addFirst(node.dependency); + } + + return path; + } + + public Stream flatten() { + return StreamSupport.stream( + Traverser.forTree(node -> node.children) + .depthFirstPreOrder(directDependencies) + .spliterator(), + false) + .map(node -> node.dependency); + } + + /** + * We don't use the {@link Dependency} as a key because we don't want lookups to be dependent on + * scope or the optional flag. + * + * @param dependency + * @return + */ + static String getKey(Dependency dependency) { + return dependency.getGroupId() + + ":" + + dependency.getArtifactId() + + ":" + + dependency.getVersion() + + ":" + + dependency.getClassifier().orElse("(no-classifier)"); + } +} diff --git a/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/utils/shared/ParserUtils.java b/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/utils/shared/ParserUtils.java new file mode 100644 index 0000000..3354668 --- /dev/null +++ b/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/utils/shared/ParserUtils.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.tools.ci.utils.shared; + +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.function.Function; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Stream; + +/** Parsing utils. */ +public class ParserUtils { + + /** + * Iterates over the given lines, identifying plugin execution blocks with the given pattern and + * parses the plugin output with the given parser. + * + *

This method assumes that the given pattern matches at most once for each module. + * + *

The given pattern must include a {@code module} group that captures the module that the + * plugin runs on (without the scala suffix!). + * + * @param lines maven output lines + * @param executionLinePattern pattern that matches plugin executions + * @param blockParser parser for the plugin block + * @return map containing the parser result for each module + * @param block parser output + */ + public static Map parsePluginOutput( + Stream lines, + Pattern executionLinePattern, + Function, D> blockParser) { + final Map result = new LinkedHashMap<>(); + + final Iterator iterator = lines.iterator(); + + while (iterator.hasNext()) { + Matcher moduleMatcher = executionLinePattern.matcher(iterator.next()); + while (!moduleMatcher.find()) { + if (iterator.hasNext()) { + moduleMatcher = executionLinePattern.matcher(iterator.next()); + } else { + return result; + } + } + final String currentModule = moduleMatcher.group("module"); + + if (!iterator.hasNext()) { + throw new IllegalStateException("Expected more output from the plugin."); + } + + result.put(currentModule, blockParser.apply(iterator)); + } + return result; + } +} diff --git a/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/resources/log4j2.properties b/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/resources/log4j2.properties new file mode 100644 index 0000000..b0b222b --- /dev/null +++ b/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/resources/log4j2.properties @@ -0,0 +1,25 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +rootLogger.level = DEBUG +rootLogger.appenderRef.console.ref = ConsoleAppender + +appender.console.name = ConsoleAppender +appender.console.type = CONSOLE +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{HH:mm:ss,SSS} %-5p %-60c %x - %m%n diff --git a/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/resources/modules-defining-excess-dependencies.modulelist b/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/resources/modules-defining-excess-dependencies.modulelist new file mode 100644 index 0000000..c496e6d --- /dev/null +++ b/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/resources/modules-defining-excess-dependencies.modulelist @@ -0,0 +1,20 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +# This file lists modules which define additional dependencies, not shown by the maven shade plugin output. +# These are usually undeclared shaded (or otherwise included) dependencies from transitive dependencies. From 976060411cfd5d328a7155b30437502d2105daf6 Mon Sep 17 00:00:00 2001 From: yuzelin Date: Wed, 20 Nov 2024 20:45:44 +0800 Subject: [PATCH 2/5] fix --- .github/workflows/check-licensing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check-licensing.yml b/.github/workflows/check-licensing.yml index 53cc5c0..3904510 100644 --- a/.github/workflows/check-licensing.yml +++ b/.github/workflows/check-licensing.yml @@ -54,4 +54,4 @@ jobs: -Dexec.args="${{ env.MVN_BUILD_OUTPUT_FILE }} $(pwd) ${{ env.MVN_VALIDATION_DIR }}" \ -Dlog4j.configurationFile=file://$(pwd)/tools/ci/log4j.properties env: - MAVEN_OPTS: -Xmx4096m \ No newline at end of file + MAVEN_OPTS: -Xmx4096m From 98b3cf9855eac114a81f2621097515c7b69190a9 Mon Sep 17 00:00:00 2001 From: yuzelin Date: Wed, 20 Nov 2024 20:49:44 +0800 Subject: [PATCH 3/5] fix --- .../{check-licensing.yml => check-java-bridge-licensing.yml} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename .github/workflows/{check-licensing.yml => check-java-bridge-licensing.yml} (98%) diff --git a/.github/workflows/check-licensing.yml b/.github/workflows/check-java-bridge-licensing.yml similarity index 98% rename from .github/workflows/check-licensing.yml rename to .github/workflows/check-java-bridge-licensing.yml index 3904510..ae1faeb 100644 --- a/.github/workflows/check-licensing.yml +++ b/.github/workflows/check-java-bridge-licensing.yml @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -name: Check licensing +name: Check Java Bridge Licensing on: [push, pull_request] From d46730cfcb6147a7f3abfd77a86e699abcd13005 Mon Sep 17 00:00:00 2001 From: yuzelin Date: Thu, 21 Nov 2024 13:43:27 +0800 Subject: [PATCH 4/5] remove paimon-ci-tools --- .../workflows/check-java-bridge-licensing.yml | 2 - .../paimon-python-java-bridge/pom.xml | 3 +- .../tools/ci/paimon-ci-tools/pom.xml | 78 ---- .../tools/ci/licensecheck/JarFileChecker.java | 330 --------------- .../tools/ci/licensecheck/LicenseChecker.java | 53 --- .../ci/licensecheck/NoticeFileChecker.java | 387 ------------------ .../ci/suffixcheck/ScalaSuffixChecker.java | 260 ------------ .../ci/utils/dependency/DependencyParser.java | 224 ---------- .../tools/ci/utils/deploy/DeployParser.java | 71 ---- .../tools/ci/utils/notice/NoticeContents.java | 42 -- .../tools/ci/utils/notice/NoticeParser.java | 92 ----- .../tools/ci/utils/shade/ShadeParser.java | 110 ----- .../tools/ci/utils/shared/Dependency.java | 135 ------ .../tools/ci/utils/shared/DependencyTree.java | 123 ------ .../tools/ci/utils/shared/ParserUtils.java | 74 ---- .../src/main/resources/log4j2.properties | 25 -- ...es-defining-excess-dependencies.modulelist | 20 - 17 files changed, 2 insertions(+), 2027 deletions(-) delete mode 100644 paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/pom.xml delete mode 100644 paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/licensecheck/JarFileChecker.java delete mode 100644 paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/licensecheck/LicenseChecker.java delete mode 100644 paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/licensecheck/NoticeFileChecker.java delete mode 100644 paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/suffixcheck/ScalaSuffixChecker.java delete mode 100644 paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/utils/dependency/DependencyParser.java delete mode 100644 paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/utils/deploy/DeployParser.java delete mode 100644 paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/utils/notice/NoticeContents.java delete mode 100644 paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/utils/notice/NoticeParser.java delete mode 100644 paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/utils/shade/ShadeParser.java delete mode 100644 paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/utils/shared/Dependency.java delete mode 100644 paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/utils/shared/DependencyTree.java delete mode 100644 paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/utils/shared/ParserUtils.java delete mode 100644 paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/resources/log4j2.properties delete mode 100644 paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/resources/modules-defining-excess-dependencies.modulelist diff --git a/.github/workflows/check-java-bridge-licensing.yml b/.github/workflows/check-java-bridge-licensing.yml index ae1faeb..1072532 100644 --- a/.github/workflows/check-java-bridge-licensing.yml +++ b/.github/workflows/check-java-bridge-licensing.yml @@ -44,8 +44,6 @@ jobs: mvn clean deploy ${{ env.MVN_COMMON_OPTIONS }} -DskipTests \ -DaltDeploymentRepository=validation_repository::default::file:${{ env.MVN_VALIDATION_DIR }} \ | tee ${{ env.MVN_BUILD_OUTPUT_FILE }} - cd tools/ci/paimon-ci-tools - mvn clean install -DskipTests - name: Check licensing run: | diff --git a/paimon_python_java/paimon-python-java-bridge/pom.xml b/paimon_python_java/paimon-python-java-bridge/pom.xml index 5927b08..e825956 100644 --- a/paimon_python_java/paimon-python-java-bridge/pom.xml +++ b/paimon_python_java/paimon-python-java-bridge/pom.xml @@ -38,6 +38,7 @@ package 14.0.0 1.8 + 0.9.0 @@ -243,7 +244,7 @@ org.apache.paimon paimon-ci-tools - ${project.version} + ${paimon.ci.tools.version} diff --git a/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/pom.xml b/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/pom.xml deleted file mode 100644 index 7305e28..0000000 --- a/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/pom.xml +++ /dev/null @@ -1,78 +0,0 @@ - - - - - 4.0.0 - - org.apache.paimon - paimon-ci-tools - 0.1-SNAPSHOT - Paimon : Tools : CI : Java - - jar - - - true - 30.1.1-jre - 0.6.0-incubating - 2.17.1 - - - - - org.apache.paimon - paimon-shade-guava-30 - ${paimon.shade.guava.version}-${paimon.shade.version} - - - - org.apache.logging.log4j - log4j-slf4j-impl - ${log4j.version} - compile - - - org.apache.logging.log4j - log4j-api - ${log4j.version} - compile - - - org.apache.logging.log4j - log4j-core - ${log4j.version} - compile - - - - - - - org.apache.maven.plugins - maven-compiler-plugin - - 8 - 8 - - - - - diff --git a/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/licensecheck/JarFileChecker.java b/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/licensecheck/JarFileChecker.java deleted file mode 100644 index b829435..0000000 --- a/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/licensecheck/JarFileChecker.java +++ /dev/null @@ -1,330 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.paimon.tools.ci.licensecheck; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.net.URI; -import java.nio.charset.MalformedInputException; -import java.nio.charset.StandardCharsets; -import java.nio.file.FileSystem; -import java.nio.file.FileSystems; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Locale; -import java.util.regex.Pattern; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -/** Checks the Jar files created by the build process. */ -public class JarFileChecker { - private static final Logger LOG = LoggerFactory.getLogger(JarFileChecker.class); - - public static int checkPath(Path path) throws Exception { - List files = getBuildJars(path); - LOG.info("Checking directory {} with a total of {} jar files.", path, files.size()); - - int severeIssues = 0; - for (Path file : files) { - severeIssues += checkJar(file); - } - - return severeIssues; - } - - private static List getBuildJars(Path path) throws IOException { - return Files.walk(path) - .filter(file -> file.toString().endsWith(".jar")) - .collect(Collectors.toList()); - } - - static int checkJar(Path file) throws Exception { - final URI uri = file.toUri(); - - int numSevereIssues = 0; - try (final FileSystem fileSystem = - FileSystems.newFileSystem( - new URI("jar:file", uri.getHost(), uri.getPath(), uri.getFragment()), - Collections.emptyMap())) { - if (isTestJarAndEmpty(file, fileSystem.getPath("/"))) { - return 0; - } - if (!noticeFileExistsAndIsValid(fileSystem.getPath("META-INF", "NOTICE"), file)) { - numSevereIssues++; - } - if (!licenseFileExistsAndIsValid(fileSystem.getPath("META-INF", "LICENSE"), file)) { - numSevereIssues++; - } - - numSevereIssues += - getNumLicenseFilesOutsideMetaInfDirectory(file, fileSystem.getPath("/")); - - numSevereIssues += getFilesWithIncompatibleLicenses(file, fileSystem.getPath("/")); - } - return numSevereIssues; - } - - private static boolean isTestJarAndEmpty(Path jar, Path jarRoot) throws IOException { - if (jar.getFileName().toString().endsWith("-tests.jar")) { - try (Stream files = Files.walk(jarRoot)) { - long numClassFiles = - files.filter(path -> !path.equals(jarRoot)) - .filter(path -> path.getFileName().toString().endsWith(".class")) - .count(); - if (numClassFiles == 0) { - return true; - } - } - } - - return false; - } - - private static boolean noticeFileExistsAndIsValid(Path noticeFile, Path jar) - throws IOException { - if (!Files.exists(noticeFile)) { - LOG.error("Missing META-INF/NOTICE in {}", jar); - return false; - } - - final String noticeFileContents = readFile(noticeFile); - if (!noticeFileContents.contains("The Apache Software Foundation")) { - LOG.error("The notice file in {} does not contain the expected entries.", jar); - return false; - } - - return true; - } - - private static boolean licenseFileExistsAndIsValid(Path licenseFile, Path jar) - throws IOException { - if (!Files.exists(licenseFile)) { - LOG.error("Missing META-INF/LICENSE in {}", jar); - return false; - } - - final String licenseFileContents = readFile(licenseFile); - if (!licenseFileContents.contains("Apache License") - || !licenseFileContents.contains("Version 2.0, January 2004")) { - LOG.error("The license file in {} does not contain the expected entries.", jar); - return false; - } - - return true; - } - - private static int getFilesWithIncompatibleLicenses(Path jar, Path jarRoot) throws IOException { - // patterns are based on https://www.apache.org/legal/resolved.html#category-x - return findNonBinaryFilesContainingText( - jar, - jarRoot, - asPatterns( - "Binary Code License", - "Intel Simplified Software License", - "JSR 275", - "Microsoft Limited Public License", - "Amazon Software License", - // Java SDK for Satori RTM license - "as necessary for your use of Satori services", - "REDIS SOURCE AVAILABLE LICENSE", - "Booz Allen Public License", - "Confluent Community License Agreement Version 1.0", - // “Commons Clause” License Condition v1.0 - "the License does not grant to you, the right to Sell the Software.", - "Sun Community Source License Version 3.0", - "GNU General Public License", - "GNU Affero General Public License", - "GNU Lesser General Public License", - "Q Public License", - "Sleepycat License", - "Server Side Public License", - "Code Project Open License", - // BSD 4-Clause - " All advertising materials mentioning features or use of this software must display the following acknowledgement", - // Facebook Patent clause v1 - "The license granted hereunder will terminate, automatically and without notice, for anyone that makes any claim", - // Facebook Patent clause v2 - "The license granted hereunder will terminate, automatically and without notice, if you (or any of your subsidiaries, corporate affiliates or agents) initiate directly or indirectly, or take a direct financial interest in, any Patent Assertion: (i) against Facebook", - "Netscape Public License", - "SOLIPSISTIC ECLIPSE PUBLIC LICENSE", - // DON'T BE A DICK PUBLIC LICENSE - "Do whatever you like with the original work, just don't be a dick.", - // JSON License - "The Software shall be used for Good, not Evil.", - // can sometimes be found in "funny" licenses - "Don’t be evil")); - } - - private static Collection asPatterns(String... texts) { - return Stream.of(texts) - .map(JarFileChecker::asPatternWithPotentialLineBreaks) - .collect(Collectors.toList()); - } - - private static Pattern asPatternWithPotentialLineBreaks(String text) { - // allows word sequences to be separated by whitespace, line-breaks and comments(//, #) - return Pattern.compile(text.toLowerCase(Locale.ROOT).replaceAll(" ", " ?\\\\R?[\\\\s/#]*")); - } - - private static int findNonBinaryFilesContainingText( - Path jar, Path jarRoot, Collection forbidden) throws IOException { - try (Stream files = Files.walk(jarRoot)) { - return files.filter(path -> !path.equals(jarRoot)) - .filter(path -> !Files.isDirectory(path)) - .filter(JarFileChecker::isNoClassFile) - // frequent false-positives due to dual-licensing; generated by maven - .filter(path -> !getFileName(path).equals("dependencies")) - // false-positives due to dual-licensing; use startsWith to cover .txt/.md files - .filter(path -> !getFileName(path).startsWith("license")) - // false-positives due to optional components; startsWith covers .txt/.md files - .filter(path -> !getFileName(path).startsWith("notice")) - // dual-licensed under GPL 2 and CDDL 1.1 - // contained in hadoop/presto S3 FS and paimon-dist - .filter( - path -> - !path.toString() - .contains("/META-INF/versions/11/javax/xml/bind")) - .filter(path -> !isJavaxManifest(jar, path)) - // dual-licensed under GPL 2 and EPL 2.0 - // contained in sql-avro-confluent-registry - .filter(path -> !pathStartsWith(path, "/org/glassfish/jersey/internal")) - .map( - path -> { - try { - final String fileContents; - try { - fileContents = readFile(path).toLowerCase(Locale.ROOT); - } catch (MalformedInputException mie) { - // binary file - return 0; - } - - int violations = 0; - for (Pattern text : forbidden) { - if (text.matcher(fileContents).find()) { - // do not count individual violations because it can be - // confusing when checking with aliases for the same - // license - violations = 1; - LOG.error( - "File '{}' in jar '{}' contains match with forbidden regex '{}'.", - path, - jar, - text); - } - } - return violations; - } catch (IOException e) { - throw new RuntimeException( - String.format( - "Could not read contents of file '%s' in jar '%s'.", - path, jar), - e); - } - }) - .reduce(Integer::sum) - .orElse(0); - } - } - - private static int getNumLicenseFilesOutsideMetaInfDirectory(Path jar, Path jarRoot) - throws IOException { - try (Stream files = Files.walk(jarRoot)) { - /* - * LICENSE or NOTICE files found outside of the META-INF directory are most likely shading mistakes (we are including the files from other dependencies, thus providing an invalid LICENSE file) - * - *

In such a case, we recommend updating the shading exclusions, and adding the license file to META-INF/licenses. - */ - final List filesWithIssues = - files.filter(path -> !path.equals(jarRoot)) - .filter( - path -> - getFileName(path).contains("license") - || getFileName(path).contains("notice")) - .filter( - path -> - !Files.isDirectory( - path)) // ignore directories, e.g. "license/" - .filter(JarFileChecker::isNoClassFile) // some class files contain - // LICENSE in their name - .filter( - path -> - !getFileName(path) - .endsWith(".ftl")) // a false positive in - // python - .map(Path::toString) - .filter( - path -> - !path.contains( - "META-INF")) // license files in META-INF are - // expected - .filter( - path -> - !path.endsWith( - "web/3rdpartylicenses.txt")) // a false positive - // in - // web - .collect(Collectors.toList()); - for (String fileWithIssue : filesWithIssues) { - LOG.error( - "Jar file {} contains a LICENSE file in an unexpected location: {}", - jar, - fileWithIssue); - } - return filesWithIssues.size(); - } - } - - private static String getFileName(Path path) { - return path.getFileName().toString().toLowerCase(); - } - - private static boolean pathStartsWith(Path file, String path) { - return file.startsWith(file.getFileSystem().getPath(path)); - } - - private static boolean equals(Path file, String path) { - return file.equals(file.getFileSystem().getPath(path)); - } - - private static boolean isNoClassFile(Path file) { - return !getFileName(file).endsWith(".class"); - } - - private static boolean isJavaxManifest(Path jar, Path potentialManifestFile) { - try { - return equals(potentialManifestFile, "/META-INF/versions/11/META-INF/MANIFEST.MF") - && readFile(potentialManifestFile).contains("Specification-Title: jaxb-api"); - } catch (IOException e) { - throw new RuntimeException( - String.format( - "Error while reading file %s from jar %s.", potentialManifestFile, jar), - e); - } - } - - private static String readFile(Path file) throws IOException { - return new String(Files.readAllBytes(file), StandardCharsets.UTF_8); - } -} diff --git a/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/licensecheck/LicenseChecker.java b/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/licensecheck/LicenseChecker.java deleted file mode 100644 index 9142f10..0000000 --- a/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/licensecheck/LicenseChecker.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.paimon.tools.ci.licensecheck; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.nio.file.Paths; - -/** Utility for checking all things related to License and Notice files. */ -public class LicenseChecker { - // ---------------------------------------- Launcher ---------------------------------------- // - - private static final Logger LOG = LoggerFactory.getLogger(LicenseChecker.class); - - public static void main(String[] args) throws Exception { - if (args.length < 2) { - System.out.println( - "Usage: LicenseChecker "); - System.exit(1); - } - LOG.warn( - "THIS UTILITY IS ONLY CHECKING FOR COMMON LICENSING MISTAKES. A MANUAL CHECK OF THE NOTICE FILES, DEPLOYED ARTIFACTS, ETC. IS STILL NEEDED!"); - - int severeIssueCount = NoticeFileChecker.run(new File(args[0]), Paths.get(args[1])); - - severeIssueCount += JarFileChecker.checkPath(Paths.get(args[2])); - - if (severeIssueCount > 0) { - LOG.warn("Found a total of {} severe license issues", severeIssueCount); - - System.exit(1); - } - LOG.info("License check completed without severe issues."); - } -} diff --git a/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/licensecheck/NoticeFileChecker.java b/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/licensecheck/NoticeFileChecker.java deleted file mode 100644 index ebba7ce..0000000 --- a/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/licensecheck/NoticeFileChecker.java +++ /dev/null @@ -1,387 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.paimon.tools.ci.licensecheck; - -import org.apache.paimon.tools.ci.utils.dependency.DependencyParser; -import org.apache.paimon.tools.ci.utils.deploy.DeployParser; -import org.apache.paimon.tools.ci.utils.notice.NoticeContents; -import org.apache.paimon.tools.ci.utils.notice.NoticeParser; -import org.apache.paimon.tools.ci.utils.shade.ShadeParser; -import org.apache.paimon.tools.ci.utils.shared.Dependency; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.annotation.Nullable; - -import java.io.BufferedReader; -import java.io.File; -import java.io.IOException; -import java.io.InputStreamReader; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.regex.Pattern; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -/** Utility class checking for proper NOTICE files based on the maven build output. */ -public class NoticeFileChecker { - - private static final Logger LOG = LoggerFactory.getLogger(NoticeFileChecker.class); - - private static final List MODULES_DEFINING_EXCESS_DEPENDENCIES = - loadFromResources("modules-defining-excess-dependencies.modulelist"); - - // Examples: - // "- org.apache.htrace:htrace-core:3.1.0-incubating" - // or - // "This project bundles "net.jcip:jcip-annotations:1.0". - private static final Pattern NOTICE_DEPENDENCY_PATTERN = - Pattern.compile( - "- ([^ :]+):([^:]+):([^ ]+)($| )|.*bundles \"([^:]+):([^:]+):([^\"]+)\".*"); - - static int run(File buildResult, Path root) throws IOException { - // parse included dependencies from build output - final Map> modulesWithBundledDependencies = - combineAndFilterPaimonDependencies( - ShadeParser.parseShadeOutput(buildResult.toPath()), - DependencyParser.parseDependencyCopyOutput(buildResult.toPath())); - - final Set deployedModules = DeployParser.parseDeployOutput(buildResult); - - LOG.info( - "Extracted " - + deployedModules.size() - + " modules that were deployed and " - + modulesWithBundledDependencies.keySet().size() - + " modules which bundle dependencies with a total of " - + modulesWithBundledDependencies.values().size() - + " dependencies"); - - // find modules producing a shaded-jar - List noticeFiles = findNoticeFiles(root); - LOG.info("Found {} NOTICE files to check", noticeFiles.size()); - - final Map> moduleToNotice = - noticeFiles.stream() - .collect( - Collectors.toMap( - NoticeFileChecker::getModuleFromNoticeFile, - noticeFile -> { - try { - return NoticeParser.parseNoticeFile(noticeFile); - } catch (IOException e) { - // some machine issue - throw new RuntimeException(e); - } - })); - - return run(modulesWithBundledDependencies, deployedModules, moduleToNotice); - } - - static int run( - Map> modulesWithBundledDependencies, - Set deployedModules, - Map> noticeFiles) - throws IOException { - int severeIssueCount = 0; - - final Set modulesSkippingDeployment = - new HashSet<>(modulesWithBundledDependencies.keySet()); - modulesSkippingDeployment.removeAll(deployedModules); - - LOG.debug( - "The following {} modules are skipping deployment: {}", - modulesSkippingDeployment.size(), - modulesSkippingDeployment.stream() - .sorted() - .collect(Collectors.joining("\n\t", "\n\t", ""))); - - for (String moduleSkippingDeployment : modulesSkippingDeployment) { - // TODO: this doesn't work for modules requiring a NOTICE that are bundled indirectly - // TODO: via another non-deployed module - boolean bundledByDeployedModule = - modulesWithBundledDependencies.entrySet().stream() - .filter( - entry -> - entry.getValue().stream() - .map(Dependency::getArtifactId) - .anyMatch( - artifactId -> - artifactId.equals( - moduleSkippingDeployment))) - .anyMatch(entry -> !modulesSkippingDeployment.contains(entry.getKey())); - - if (!bundledByDeployedModule) { - modulesWithBundledDependencies.remove(moduleSkippingDeployment); - } else { - LOG.debug( - "Including module {} in license checks, despite not being deployed, because it is bundled by another deployed module.", - moduleSkippingDeployment); - } - } - - // check that all required NOTICE files exists - severeIssueCount += - ensureRequiredNoticeFiles(modulesWithBundledDependencies, noticeFiles.keySet()); - - // check each NOTICE file - for (Map.Entry> noticeFile : noticeFiles.entrySet()) { - severeIssueCount += - checkNoticeFileAndLogProblems( - modulesWithBundledDependencies, - noticeFile.getKey(), - noticeFile.getValue().orElse(null)); - } - - return severeIssueCount; - } - - private static Map> combineAndFilterPaimonDependencies( - Map> modulesWithBundledDependencies, - Map> modulesWithCopiedDependencies) { - - final Map> combinedAndFiltered = new LinkedHashMap<>(); - - Stream.concat( - modulesWithBundledDependencies.entrySet().stream(), - modulesWithCopiedDependencies.entrySet().stream()) - .forEach( - (entry) -> { - final Set dependencies = - combinedAndFiltered.computeIfAbsent( - entry.getKey(), ignored -> new LinkedHashSet<>()); - - for (Dependency dependency : entry.getValue()) { - if (!dependency.getGroupId().contains("org.apache.paimon")) { - dependencies.add(dependency); - } - } - }); - - return combinedAndFiltered; - } - - private static int ensureRequiredNoticeFiles( - Map> modulesWithShadedDependencies, - Collection modulesWithNoticeFile) { - int severeIssueCount = 0; - Set shadingModules = new HashSet<>(modulesWithShadedDependencies.keySet()); - shadingModules.removeAll(modulesWithNoticeFile); - for (String moduleWithoutNotice : shadingModules) { - if (modulesWithShadedDependencies.get(moduleWithoutNotice).stream() - .anyMatch(dependency -> !dependency.getGroupId().equals("org.apache.paimon"))) { - LOG.error( - "Module {} is missing a NOTICE file. It has shaded dependencies: {}", - moduleWithoutNotice, - modulesWithShadedDependencies.get(moduleWithoutNotice).stream() - .map(Dependency::toString) - .collect(Collectors.joining("\n\t", "\n\t", ""))); - severeIssueCount++; - } - } - return severeIssueCount; - } - - private static String getModuleFromNoticeFile(Path noticeFile) { - Path moduleDirectory = - noticeFile - .getParent() // META-INF - .getParent() // resources - .getParent() // main - .getParent() // src - .getParent(); // <-- module name - return moduleDirectory.getFileName().toString(); - } - - private static int checkNoticeFileAndLogProblems( - Map> modulesWithShadedDependencies, - String moduleName, - @Nullable NoticeContents noticeContents) - throws IOException { - - final Map> problemsBySeverity = - checkNoticeFile(modulesWithShadedDependencies, moduleName, noticeContents); - - final List severeProblems = - problemsBySeverity.getOrDefault(Severity.CRITICAL, Collections.emptyList()); - - if (!problemsBySeverity.isEmpty()) { - final List toleratedProblems = - problemsBySeverity.getOrDefault(Severity.TOLERATED, Collections.emptyList()); - final List expectedProblems = - problemsBySeverity.getOrDefault(Severity.SUPPRESSED, Collections.emptyList()); - - LOG.info( - "Problems were detected for a NOTICE file.\n" + "\t{}:\n" + "{}{}{}", - moduleName, - convertProblemsToIndentedString( - severeProblems, - "These issue are legally problematic and MUST be fixed:"), - convertProblemsToIndentedString( - toleratedProblems, - "These issues are mistakes that aren't legally problematic. They SHOULD be fixed at some point, but we don't have to:"), - convertProblemsToIndentedString( - expectedProblems, "These issues are assumed to be false-positives:")); - } - - return severeProblems.size(); - } - - static Map> checkNoticeFile( - Map> modulesWithShadedDependencies, - String moduleName, - @Nullable NoticeContents noticeContents) { - - final Map> problemsBySeverity = new HashMap<>(); - - if (noticeContents == null) { - addProblem(problemsBySeverity, Severity.CRITICAL, "The NOTICE file was empty."); - } else { - // first line must be the module name. - if (!noticeContents.getNoticeModuleName().equals(moduleName)) { - addProblem( - problemsBySeverity, - Severity.TOLERATED, - String.format( - "First line does not start with module name. firstLine=%s", - noticeContents.getNoticeModuleName())); - } - - // collect all declared dependencies from NOTICE file - Set declaredDependencies = new HashSet<>(); - for (Dependency declaredDependency : noticeContents.getDeclaredDependencies()) { - if (!declaredDependencies.add(declaredDependency)) { - addProblem( - problemsBySeverity, - Severity.CRITICAL, - String.format("Dependency %s is declared twice.", declaredDependency)); - } - } - - // find all dependencies missing from NOTICE file - Collection expectedDependencies = - modulesWithShadedDependencies.getOrDefault(moduleName, Collections.emptySet()) - .stream() - .filter( - dependency -> - !dependency.getGroupId().equals("org.apache.paimon")) - .collect(Collectors.toList()); - - for (Dependency expectedDependency : expectedDependencies) { - if (!declaredDependencies.contains(expectedDependency)) { - addProblem( - problemsBySeverity, - Severity.CRITICAL, - String.format("Dependency %s is not listed.", expectedDependency)); - } - } - - boolean moduleDefinesExcessDependencies = - MODULES_DEFINING_EXCESS_DEPENDENCIES.contains(moduleName); - - // find all dependencies defined in NOTICE file, which were not expected - for (Dependency declaredDependency : declaredDependencies) { - if (!expectedDependencies.contains(declaredDependency)) { - final Severity severity = - moduleDefinesExcessDependencies - ? Severity.SUPPRESSED - : Severity.TOLERATED; - addProblem( - problemsBySeverity, - severity, - String.format( - "Dependency %s is not bundled, but listed.", - declaredDependency)); - } - } - } - - return problemsBySeverity; - } - - private static void addProblem( - Map> problemsBySeverity, Severity severity, String problem) { - problemsBySeverity.computeIfAbsent(severity, ignored -> new ArrayList<>()).add(problem); - } - - enum Severity { - /** Issues that a legally problematic which must be fixed. */ - CRITICAL, - /** Issues that affect the correctness but aren't legally problematic. */ - TOLERATED, - /** Issues where we intentionally break the rules. */ - SUPPRESSED - } - - private static String convertProblemsToIndentedString(List problems, String header) { - return problems.isEmpty() - ? "" - : problems.stream() - .map(s -> "\t\t\t" + s) - .collect(Collectors.joining("\n", "\t\t " + header + " \n", "\n")); - } - - private static List findNoticeFiles(Path root) throws IOException { - return Files.walk(root) - .filter( - file -> { - int nameCount = file.getNameCount(); - return file.getName(nameCount - 3).toString().equals("resources") - && file.getName(nameCount - 2).toString().equals("META-INF") - && file.getName(nameCount - 1).toString().equals("NOTICE"); - }) - .collect(Collectors.toList()); - } - - private static List loadFromResources(String fileName) { - try { - try (BufferedReader bufferedReader = - new BufferedReader( - new InputStreamReader( - Objects.requireNonNull( - NoticeFileChecker.class.getResourceAsStream( - "/" + fileName))))) { - - List result = - bufferedReader - .lines() - .filter(line -> !line.startsWith("#") && !line.isEmpty()) - .collect(Collectors.toList()); - LOG.debug("Loaded {} items from resource {}", result.size(), fileName); - return result; - } - } catch (Throwable e) { - // wrap anything in a RuntimeException to be callable from the static initializer - throw new RuntimeException("Error while loading resource", e); - } - } -} diff --git a/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/suffixcheck/ScalaSuffixChecker.java b/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/suffixcheck/ScalaSuffixChecker.java deleted file mode 100644 index 792dc3c..0000000 --- a/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/suffixcheck/ScalaSuffixChecker.java +++ /dev/null @@ -1,260 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.paimon.tools.ci.suffixcheck; - -import org.apache.paimon.tools.ci.utils.dependency.DependencyParser; -import org.apache.paimon.tools.ci.utils.shared.Dependency; -import org.apache.paimon.tools.ci.utils.shared.DependencyTree; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.regex.Pattern; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -/** Utility for checking the presence/absence of scala-suffixes. */ -public class ScalaSuffixChecker { - private static final Logger LOG = LoggerFactory.getLogger(ScalaSuffixChecker.class); - - // [INFO] --- maven-dependency-plugin:3.1.1:tree (default-cli) @ paimon-annotations --- - private static final Pattern moduleNamePattern = - Pattern.compile(".* --- maven-dependency-plugin.* @ (.*) ---.*"); - - // [INFO] +- junit:junit:jar:4.13.2:test - // [INFO] | \- org.hamcrest:hamcrest-core:jar:1.3:test - // [INFO] \- org.apache.logging.log4j:log4j-1.2-api:jar:2.14.1:test - private static final Pattern blockPattern = Pattern.compile(".* [+|\\\\].*"); - - // [INFO] +- org.scala-lang:scala-reflect:jar:2.11.12:test - private static final Pattern scalaSuffixPattern = Pattern.compile("_2.1[0-9]"); - - private static final Set EXCLUDED_MODULES = new HashSet<>(Arrays.asList()); - - public static void main(String[] args) throws IOException { - if (args.length < 2) { - System.out.println("Usage: ScalaSuffixChecker "); - System.exit(1); - } - - final Path mavenOutputPath = Paths.get(args[0]); - final Path paimonRootPath = Paths.get(args[1]); - - final ParseResult parseResult = parseMavenOutput(mavenOutputPath); - if (parseResult.getCleanModules().isEmpty()) { - LOG.error("Parsing found 0 scala-free modules; the parsing is likely broken."); - System.exit(1); - } - if (parseResult.getInfectedModules().isEmpty()) { - LOG.error("Parsing found 0 scala-dependent modules; the parsing is likely broken."); - System.exit(1); - } - - final Collection violations = checkScalaSuffixes(parseResult, paimonRootPath); - - if (!violations.isEmpty()) { - LOG.error( - "Violations found:{}", - violations.stream().collect(Collectors.joining("\n\t", "\n\t", ""))); - System.exit(1); - } - } - - private static ParseResult parseMavenOutput(final Path path) throws IOException { - final Set cleanModules = new HashSet<>(); - final Set infectedModules = new HashSet<>(); - - final Map dependenciesByModule = - DependencyParser.parseDependencyTreeOutput(path); - - for (String module : dependenciesByModule.keySet()) { - final String moduleName = stripScalaSuffix(module); - if (isExcluded(moduleName)) { - continue; - } - LOG.trace("Processing module '{}'.", moduleName); - - final List dependencies = - dependenciesByModule.get(module).flatten().collect(Collectors.toList()); - - boolean infected = false; - for (Dependency dependency : dependencies) { - final boolean dependsOnScala = dependsOnScala(dependency); - final boolean isTestDependency = dependency.getScope().get().equals("test"); - final boolean isExcluded = isExcluded(dependency.getArtifactId()); - LOG.trace("\tdependency:{}", dependency); - LOG.trace("\t\tdepends-on-scala:{}", dependsOnScala); - LOG.trace("\t\tis-test-dependency:{}", isTestDependency); - LOG.trace("\t\tis-excluded:{}", isExcluded); - if (dependsOnScala && !isTestDependency && !isExcluded) { - LOG.trace("\t\tOutbreak detected at {}!", moduleName); - infected = true; - } - } - - if (infected) { - infectedModules.add(moduleName); - } else { - cleanModules.add(moduleName); - } - } - - return new ParseResult(cleanModules, infectedModules); - } - - private static String stripScalaSuffix(final String moduleName) { - final int i = moduleName.indexOf("_2."); - return i > 0 ? moduleName.substring(0, i) : moduleName; - } - - private static boolean dependsOnScala(final Dependency dependency) { - return dependency.getGroupId().contains("org.scala-lang") - || scalaSuffixPattern.matcher(dependency.getArtifactId()).find(); - } - - private static Collection checkScalaSuffixes( - final ParseResult parseResult, Path paimonRootPath) throws IOException { - final Collection violations = new ArrayList<>(); - - // exclude e2e modules and paimon-docs for convenience as they - // a) are not deployed during a release - // b) exist only for dev purposes - // c) no-one should depend on them - final Collection excludedModules = new ArrayList<>(); - excludedModules.add("paimon-docs"); - excludedModules.addAll(getEndToEndTestModules(paimonRootPath)); - - for (String excludedModule : excludedModules) { - parseResult.getCleanModules().remove(excludedModule); - parseResult.getInfectedModules().remove(excludedModule); - } - - violations.addAll(checkCleanModules(parseResult.getCleanModules(), paimonRootPath)); - violations.addAll(checkInfectedModules(parseResult.getInfectedModules(), paimonRootPath)); - - return violations; - } - - private static Collection getEndToEndTestModules(Path paimonRootPath) - throws IOException { - try (Stream pathStream = Files.walk(paimonRootPath.resolve("paimon-e2e-tests"), 5)) { - return pathStream - .filter(path -> path.getFileName().toString().equals("pom.xml")) - .map(path -> path.getParent().getFileName().toString()) - .collect(Collectors.toList()); - } - } - - private static Collection checkCleanModules( - Collection modules, Path paimonRootPath) throws IOException { - return checkModules( - modules, - paimonRootPath, - "_${scala.binary.version}", - "Scala-free module '%s' is referenced with scala suffix in '%s'."); - } - - private static Collection checkInfectedModules( - Collection modules, Path paimonRootPath) throws IOException { - return checkModules( - modules, - paimonRootPath, - "", - "Scala-dependent module '%s' is referenced without scala suffix in '%s'."); - } - - private static Collection checkModules( - Collection modules, - Path paimonRootPath, - String moduleSuffix, - String violationTemplate) - throws IOException { - - final ArrayList sortedModules = new ArrayList<>(modules); - sortedModules.sort(String::compareTo); - - final Collection violations = new ArrayList<>(); - for (String module : sortedModules) { - int numPreviousViolations = violations.size(); - try (Stream pathStream = Files.walk(paimonRootPath, 3)) { - final List pomFiles = - pathStream - .filter(path -> path.getFileName().toString().equals("pom.xml")) - .collect(Collectors.toList()); - - for (Path pomFile : pomFiles) { - try (Stream lines = Files.lines(pomFile, StandardCharsets.UTF_8)) { - final boolean existsCleanReference = - lines.anyMatch( - line -> - line.contains( - module + moduleSuffix + "")); - - if (existsCleanReference) { - violations.add( - String.format( - violationTemplate, - module, - paimonRootPath.relativize(pomFile))); - } - } - } - } - if (numPreviousViolations == violations.size()) { - LOG.info("OK {}", module); - } - } - return violations; - } - - private static boolean isExcluded(String line) { - return EXCLUDED_MODULES.stream().anyMatch(line::contains); - } - - private static class ParseResult { - - private final Set cleanModules; - private final Set infectedModules; - - private ParseResult(Set cleanModules, Set infectedModules) { - this.cleanModules = cleanModules; - this.infectedModules = infectedModules; - } - - public Set getCleanModules() { - return cleanModules; - } - - public Set getInfectedModules() { - return infectedModules; - } - } -} diff --git a/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/utils/dependency/DependencyParser.java b/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/utils/dependency/DependencyParser.java deleted file mode 100644 index 46db112..0000000 --- a/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/utils/dependency/DependencyParser.java +++ /dev/null @@ -1,224 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.paimon.tools.ci.utils.dependency; - -import org.apache.paimon.tools.ci.utils.shared.Dependency; -import org.apache.paimon.tools.ci.utils.shared.DependencyTree; -import org.apache.paimon.tools.ci.utils.shared.ParserUtils; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Iterator; -import java.util.LinkedHashSet; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.Stack; -import java.util.function.Function; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import java.util.stream.Stream; - -/** Parsing utils for the Maven dependency plugin. */ -public class DependencyParser { - - private static final Pattern DEPENDENCY_COPY_NEXT_MODULE_PATTERN = - Pattern.compile( - ".*maven-dependency-plugin:[^:]+:copy .* @ (?[^ _]+)(?:_[0-9.]+)? --.*"); - - private static final Pattern DEPENDENCY_TREE_NEXT_MODULE_PATTERN = - Pattern.compile( - ".*maven-dependency-plugin:[^:]+:tree .* @ (?[^ _]+)(?:_[0-9.]+)? --.*"); - - /** See {@link DependencyParserTreeTest} for examples. */ - private static final Pattern DEPENDENCY_TREE_ITEM_PATTERN = - Pattern.compile( - ".* +" - + "(?.*?):" - + "(?.*?):" - + "(?.*?):" - + "(?:(?.*?):)?" - + "(?.*?):" - + "(?[^ ]*)" - + "(? \\(optional\\))?"); - - /** See {@link DependencyParserCopyTest} for examples. */ - private static final Pattern DEPENDENCY_COPY_ITEM_PATTERN = - Pattern.compile( - ".* Configured Artifact: +" - + "(?.*?):" - + "(?.*?):" - + "(?:(?.*?):)?" - + "(?.*?):" - + "(?:\\?:)?" // unknown cause; e.g.: javax.xml.bind:jaxb-api:?:jar - + "(?.*)"); - - /** - * Parses the output of a Maven build where {@code dependency:copy} was used, and returns a set - * of copied dependencies for each module. - * - *

The returned dependencies will NEVER contain the scope or optional flag. - */ - public static Map> parseDependencyCopyOutput(Path buildOutput) - throws IOException { - return processLines(buildOutput, DependencyParser::parseDependencyCopyOutput); - } - - /** - * Parses the output of a Maven build where {@code dependency:tree} was used, and returns a set - * of dependencies for each module. - */ - public static Map parseDependencyTreeOutput(Path buildOutput) - throws IOException { - return processLines(buildOutput, DependencyParser::parseDependencyTreeOutput); - } - - private static X processLines(Path buildOutput, Function, X> processor) - throws IOException { - try (Stream lines = Files.lines(buildOutput)) { - return processor.apply(lines.filter(line -> line.contains("[INFO]"))); - } - } - - static Map> parseDependencyCopyOutput(Stream lines) { - return ParserUtils.parsePluginOutput( - lines, - DEPENDENCY_COPY_NEXT_MODULE_PATTERN, - DependencyParser::parseCopyDependencyBlock); - } - - static Map parseDependencyTreeOutput(Stream lines) { - return ParserUtils.parsePluginOutput( - lines, - DEPENDENCY_TREE_NEXT_MODULE_PATTERN, - DependencyParser::parseTreeDependencyBlock); - } - - private static Set parseCopyDependencyBlock(Iterator block) { - final Set dependencies = new LinkedHashSet<>(); - - Optional parsedDependency = parseCopyDependency(block.next()); - while (parsedDependency.isPresent()) { - dependencies.add(parsedDependency.get()); - - if (block.hasNext()) { - parsedDependency = parseCopyDependency(block.next()); - } else { - parsedDependency = Optional.empty(); - } - } - - return dependencies; - } - - private static DependencyTree parseTreeDependencyBlock(Iterator block) { - // discard one line, which only contains the current module name - block.next(); - - if (!block.hasNext()) { - throw new IllegalStateException("Expected more output from the dependency-plugin."); - } - - final DependencyTree dependencies = new DependencyTree(); - - final Stack parentStack = new Stack<>(); - final Stack treeDepthStack = new Stack<>(); - String line = block.next(); - Optional parsedDependency = parseTreeDependency(line); - while (parsedDependency.isPresent()) { - int treeDepth = getDepth(line); - - while (!treeDepthStack.isEmpty() && treeDepth <= treeDepthStack.peek()) { - parentStack.pop(); - treeDepthStack.pop(); - } - - final Dependency dependency = parsedDependency.get(); - - if (parentStack.isEmpty()) { - dependencies.addDirectDependency(dependency); - } else { - dependencies.addTransitiveDependencyTo(dependency, parentStack.peek()); - } - - if (treeDepthStack.isEmpty() || treeDepth > treeDepthStack.peek()) { - treeDepthStack.push(treeDepth); - parentStack.push(dependency); - } - - if (block.hasNext()) { - line = block.next(); - parsedDependency = parseTreeDependency(line); - } else { - parsedDependency = Optional.empty(); - } - } - - return dependencies; - } - - static Optional parseCopyDependency(String line) { - Matcher dependencyMatcher = DEPENDENCY_COPY_ITEM_PATTERN.matcher(line); - if (!dependencyMatcher.find()) { - return Optional.empty(); - } - - return Optional.of( - Dependency.create( - dependencyMatcher.group("groupId"), - dependencyMatcher.group("artifactId"), - dependencyMatcher.group("version"), - dependencyMatcher.group("classifier"))); - } - - static Optional parseTreeDependency(String line) { - Matcher dependencyMatcher = DEPENDENCY_TREE_ITEM_PATTERN.matcher(line); - if (!dependencyMatcher.find()) { - return Optional.empty(); - } - - return Optional.of( - Dependency.create( - dependencyMatcher.group("groupId"), - dependencyMatcher.group("artifactId"), - dependencyMatcher.group("version"), - dependencyMatcher.group("classifier"), - dependencyMatcher.group("scope"), - dependencyMatcher.group("optional") != null)); - } - - /** - * The depths returned by this method do NOT return a continuous sequence. - * - * 

-     * +- org.apache.paimon:...
-     * |  +- org.apache.paimon:...
-     * |  |  \- org.apache.paimon:...
-     * ...
-     *
- */ - private static int getDepth(String line) { - final int level = line.indexOf('+'); - if (level != -1) { - return level; - } - return line.indexOf('\\'); - } -} diff --git a/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/utils/deploy/DeployParser.java b/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/utils/deploy/DeployParser.java deleted file mode 100644 index 2975f92..0000000 --- a/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/utils/deploy/DeployParser.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.paimon.tools.ci.utils.deploy; - -import org.apache.paimon.tools.ci.utils.shared.ParserUtils; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.util.Iterator; -import java.util.Map; -import java.util.Set; -import java.util.regex.Pattern; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -/** Parsing utils for the Maven deploy plugin. */ -public class DeployParser { - - // Examples: - // - // Deployment on CI with alternative repo - // [INFO] --- maven-deploy-plugin:2.8.2:deploy (default-deploy) @ paimon-parent --- - // [INFO] Using alternate deployment repository.../tmp/paimon-validation-deployment - // - // Skipped deployment: - // [INFO] --- maven-deploy-plugin:2.8.2:deploy (default-deploy) @ paimon-parent --- - // [INFO] Skipping artifact deployment - private static final Pattern DEPLOY_MODULE_PATTERN = - Pattern.compile( - ".maven-deploy-plugin:.*:deploy .* @ (?[^ _]+)(?:_[0-9.]+)? --.*"); - - /** - * Parses the output of a Maven build where {@code deploy:deploy} was used, and returns a set of - * deployed modules. - */ - public static Set parseDeployOutput(File buildResult) throws IOException { - try (Stream linesStream = Files.lines(buildResult.toPath())) { - return parseDeployOutput(linesStream); - } - } - - static Set parseDeployOutput(Stream lines) { - return ParserUtils.parsePluginOutput( - lines, DEPLOY_MODULE_PATTERN, DeployParser::parseDeployBlock) - .entrySet().stream() - .filter(Map.Entry::getValue) - .map(Map.Entry::getKey) - .collect(Collectors.toSet()); - } - - private static boolean parseDeployBlock(Iterator block) { - return block.hasNext() && !block.next().contains("Skipping artifact deployment"); - } -} diff --git a/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/utils/notice/NoticeContents.java b/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/utils/notice/NoticeContents.java deleted file mode 100644 index 8dadaa3..0000000 --- a/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/utils/notice/NoticeContents.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.paimon.tools.ci.utils.notice; - -import org.apache.paimon.tools.ci.utils.shared.Dependency; - -import java.util.Collection; - -/** Represents the parsed contents of a NOTICE file. */ -public class NoticeContents { - private final String noticeModuleName; - private final Collection declaredDependencies; - - public NoticeContents(String noticeModuleName, Collection declaredDependencies) { - this.noticeModuleName = noticeModuleName; - this.declaredDependencies = declaredDependencies; - } - - public String getNoticeModuleName() { - return noticeModuleName; - } - - public Collection getDeclaredDependencies() { - return declaredDependencies; - } -} diff --git a/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/utils/notice/NoticeParser.java b/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/utils/notice/NoticeParser.java deleted file mode 100644 index 370c46b..0000000 --- a/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/utils/notice/NoticeParser.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.paimon.tools.ci.utils.notice; - -import org.apache.paimon.tools.ci.utils.shared.Dependency; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Optional; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** Parsing utils for NOTICE files. */ -public class NoticeParser { - - // "- org.apache.htrace:htrace-core:3.1.0-incubating" - private static final Pattern NOTICE_DEPENDENCY_PATTERN = - Pattern.compile( - "- " - + "(?[^ ]*?):" - + "(?[^ ]*?):" - + "(?:(?[^ ]*?):)?" - + "(?[^ ]*?)" - + "($| )"); - // "This project bundles "net.jcip:jcip-annotations:1.0". - private static final Pattern NOTICE_BUNDLES_DEPENDENCY_PATTERN = - Pattern.compile( - ".*bundles \"" - + "(?[^ ]*?):" - + "(?[^ ]*?):" - + "(?:(?[^ ]*?):)?" - + "(?[^ ]*?)" - + "\".*"); - - public static Optional parseNoticeFile(Path noticeFile) throws IOException { - // 1st line contains module name - final List noticeContents = Files.readAllLines(noticeFile); - - return parseNoticeFile(noticeContents); - } - - static Optional parseNoticeFile(List noticeContents) { - if (noticeContents.isEmpty()) { - return Optional.empty(); - } - - final String noticeModuleName = noticeContents.get(0); - - Collection declaredDependencies = new ArrayList<>(); - for (String line : noticeContents) { - Optional dependency = tryParsing(NOTICE_DEPENDENCY_PATTERN, line); - if (!dependency.isPresent()) { - dependency = tryParsing(NOTICE_BUNDLES_DEPENDENCY_PATTERN, line); - } - dependency.ifPresent(declaredDependencies::add); - } - - return Optional.of(new NoticeContents(noticeModuleName, declaredDependencies)); - } - - private static Optional tryParsing(Pattern pattern, String line) { - Matcher matcher = pattern.matcher(line); - if (matcher.find()) { - String groupId = matcher.group("groupId"); - String artifactId = matcher.group("artifactId"); - String version = matcher.group("version"); - String classifier = matcher.group("classifier"); - return Optional.of(Dependency.create(groupId, artifactId, version, classifier)); - } - return Optional.empty(); - } -} diff --git a/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/utils/shade/ShadeParser.java b/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/utils/shade/ShadeParser.java deleted file mode 100644 index f5a6056..0000000 --- a/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/utils/shade/ShadeParser.java +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.paimon.tools.ci.utils.shade; - -import org.apache.paimon.tools.ci.utils.shared.Dependency; -import org.apache.paimon.tools.ci.utils.shared.ParserUtils; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Iterator; -import java.util.LinkedHashSet; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import java.util.stream.Stream; - -/** Utils for parsing the shade-plugin output. */ -public final class ShadeParser { - - private static final Pattern SHADE_NEXT_MODULE_PATTERN = - Pattern.compile( - ".*:shade \\((?:shade-paimon|shade-dist|default)\\) @ (?[^ _]+)(?:_[0-9.]+)? --.*"); - - private static final Pattern SHADE_INCLUDE_MODULE_PATTERN = - Pattern.compile( - ".* " - + "(?.*?):" - + "(?.*?):" - + "(?.*?):" - + "(?:(?.*?):)?" - + "(?.*?)" - + " in the shaded jar"); - - /** - * Parses the output of a Maven build where {@code shade:shade} was used, and returns a set of - * bundled dependencies for each module. - * - *

The returned dependencies will NEVER contain the scope or optional flag. - * - *

This method only considers the {@code shade-paimon} and {@code shade-dist} executions, - * because all artifacts we produce that are either published or referenced are created by these - * executions. In other words, all artifacts from other executions are only used internally by - * the module that created them. - */ - public static Map> parseShadeOutput(Path buildOutput) - throws IOException { - try (Stream lines = Files.lines(buildOutput)) { - return parseShadeOutput(lines); - } - } - - static Map> parseShadeOutput(Stream lines) { - return ParserUtils.parsePluginOutput( - lines.filter(line -> !line.contains(" Excluding ")), - SHADE_NEXT_MODULE_PATTERN, - ShadeParser::parseBlock); - } - - private static Set parseBlock(Iterator block) { - final Set dependencies = new LinkedHashSet<>(); - - Optional parsedDependency = parseDependency(block.next()); - while (parsedDependency.isPresent()) { - dependencies.add(parsedDependency.get()); - - if (block.hasNext()) { - parsedDependency = parseDependency(block.next()); - } else { - parsedDependency = Optional.empty(); - } - } - - return dependencies; - } - - static Optional parseDependency(String line) { - Matcher dependencyMatcher = SHADE_INCLUDE_MODULE_PATTERN.matcher(line); - if (!dependencyMatcher.find()) { - return Optional.empty(); - } - - return Optional.of( - Dependency.create( - dependencyMatcher.group("groupId"), - dependencyMatcher.group("artifactId"), - dependencyMatcher.group("version"), - dependencyMatcher.group("classifier"))); - } - - private ShadeParser() {} -} diff --git a/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/utils/shared/Dependency.java b/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/utils/shared/Dependency.java deleted file mode 100644 index fad1ac2..0000000 --- a/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/utils/shared/Dependency.java +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.paimon.tools.ci.utils.shared; - -import javax.annotation.Nullable; - -import java.util.Objects; -import java.util.Optional; - -/** - * Represents a dependency. - * - *

For some properties we return an {@link Optional}, for those that, depending on the plugin - * goal, may not be determinable. For example, {@code dependency:copy} never prints the scope or - * optional flag. - */ -public final class Dependency { - - private final String groupId; - private final String artifactId; - private final String version; - @Nullable private final String classifier; - @Nullable private final String scope; - @Nullable private final Boolean isOptional; - - private Dependency( - String groupId, - String artifactId, - String version, - @Nullable String classifier, - @Nullable String scope, - @Nullable Boolean isOptional) { - this.groupId = Objects.requireNonNull(groupId); - this.artifactId = Objects.requireNonNull(artifactId); - this.version = Objects.requireNonNull(version); - this.classifier = classifier; - this.scope = scope; - this.isOptional = isOptional; - } - - public static Dependency create( - String groupId, - String artifactId, - String version, - String classifier, - String scope, - boolean isOptional) { - return new Dependency( - groupId, - artifactId, - version, - classifier, - Objects.requireNonNull(scope), - isOptional); - } - - public static Dependency create( - String groupId, String artifactId, String version, String classifier) { - return new Dependency(groupId, artifactId, version, classifier, null, null); - } - - public String getGroupId() { - return groupId; - } - - public String getArtifactId() { - return artifactId; - } - - public String getVersion() { - return version; - } - - public Optional getClassifier() { - return Optional.ofNullable(classifier); - } - - public Optional getScope() { - return Optional.ofNullable(scope); - } - - public Optional isOptional() { - return Optional.ofNullable(isOptional); - } - - @Override - public String toString() { - return groupId - + ":" - + artifactId - + ":" - + version - + (classifier != null ? ":" + classifier : "") - + (scope != null ? ":" + scope : "") - + (isOptional != null && isOptional ? " (optional)" : ""); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - Dependency that = (Dependency) o; - return Objects.equals(groupId, that.groupId) - && Objects.equals(artifactId, that.artifactId) - && Objects.equals(version, that.version) - && Objects.equals(classifier, that.classifier) - && Objects.equals(scope, that.scope) - && Objects.equals(isOptional, that.isOptional); - } - - @Override - public int hashCode() { - return Objects.hash(groupId, artifactId, version, classifier, scope, isOptional); - } -} diff --git a/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/utils/shared/DependencyTree.java b/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/utils/shared/DependencyTree.java deleted file mode 100644 index b7afe11..0000000 --- a/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/utils/shared/DependencyTree.java +++ /dev/null @@ -1,123 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.paimon.tools.ci.utils.shared; - -import org.apache.paimon.shade.guava30.com.google.common.graph.Traverser; - -import javax.annotation.Nullable; - -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.stream.Stream; -import java.util.stream.StreamSupport; - -/** - * Represents a dependency tree. - * - *

Every dependency can only occur exactly once. - */ -public class DependencyTree { - - private final Map lookup = new LinkedHashMap<>(); - private final List directDependencies = new ArrayList<>(); - - public void addDirectDependency(Dependency dependency) { - final String key = getKey(dependency); - if (lookup.containsKey(key)) { - return; - } - final Node node = new Node(dependency, null); - - lookup.put(key, node); - directDependencies.add(node); - } - - public void addTransitiveDependencyTo(Dependency transitiveDependency, Dependency parent) { - final String key = getKey(transitiveDependency); - if (lookup.containsKey(key)) { - return; - } - final Node node = lookup.get(getKey(parent)).addTransitiveDependency(transitiveDependency); - - lookup.put(key, node); - } - - private static final class Node { - private final Dependency dependency; - @Nullable private final Node parent; - private final List children = new ArrayList<>(); - - private Node(Dependency dependency, @Nullable Node parent) { - this.dependency = dependency; - this.parent = parent; - } - - public Node addTransitiveDependency(Dependency dependency) { - final Node node = new Node(dependency, this); - this.children.add(node); - return node; - } - - private boolean isRoot() { - return parent == null; - } - } - - public List getPathTo(Dependency dependency) { - final LinkedList path = new LinkedList<>(); - - Node node = lookup.get(getKey(dependency)); - path.addFirst(node.dependency); - while (!node.isRoot()) { - node = node.parent; - path.addFirst(node.dependency); - } - - return path; - } - - public Stream flatten() { - return StreamSupport.stream( - Traverser.forTree(node -> node.children) - .depthFirstPreOrder(directDependencies) - .spliterator(), - false) - .map(node -> node.dependency); - } - - /** - * We don't use the {@link Dependency} as a key because we don't want lookups to be dependent on - * scope or the optional flag. - * - * @param dependency - * @return - */ - static String getKey(Dependency dependency) { - return dependency.getGroupId() - + ":" - + dependency.getArtifactId() - + ":" - + dependency.getVersion() - + ":" - + dependency.getClassifier().orElse("(no-classifier)"); - } -} diff --git a/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/utils/shared/ParserUtils.java b/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/utils/shared/ParserUtils.java deleted file mode 100644 index 3354668..0000000 --- a/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/utils/shared/ParserUtils.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.paimon.tools.ci.utils.shared; - -import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.function.Function; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import java.util.stream.Stream; - -/** Parsing utils. */ -public class ParserUtils { - - /** - * Iterates over the given lines, identifying plugin execution blocks with the given pattern and - * parses the plugin output with the given parser. - * - *

This method assumes that the given pattern matches at most once for each module. - * - *

The given pattern must include a {@code module} group that captures the module that the - * plugin runs on (without the scala suffix!). - * - * @param lines maven output lines - * @param executionLinePattern pattern that matches plugin executions - * @param blockParser parser for the plugin block - * @return map containing the parser result for each module - * @param block parser output - */ - public static Map parsePluginOutput( - Stream lines, - Pattern executionLinePattern, - Function, D> blockParser) { - final Map result = new LinkedHashMap<>(); - - final Iterator iterator = lines.iterator(); - - while (iterator.hasNext()) { - Matcher moduleMatcher = executionLinePattern.matcher(iterator.next()); - while (!moduleMatcher.find()) { - if (iterator.hasNext()) { - moduleMatcher = executionLinePattern.matcher(iterator.next()); - } else { - return result; - } - } - final String currentModule = moduleMatcher.group("module"); - - if (!iterator.hasNext()) { - throw new IllegalStateException("Expected more output from the plugin."); - } - - result.put(currentModule, blockParser.apply(iterator)); - } - return result; - } -} diff --git a/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/resources/log4j2.properties b/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/resources/log4j2.properties deleted file mode 100644 index b0b222b..0000000 --- a/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/resources/log4j2.properties +++ /dev/null @@ -1,25 +0,0 @@ -################################################################################ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -################################################################################ - -rootLogger.level = DEBUG -rootLogger.appenderRef.console.ref = ConsoleAppender - -appender.console.name = ConsoleAppender -appender.console.type = CONSOLE -appender.console.layout.type = PatternLayout -appender.console.layout.pattern = %d{HH:mm:ss,SSS} %-5p %-60c %x - %m%n diff --git a/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/resources/modules-defining-excess-dependencies.modulelist b/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/resources/modules-defining-excess-dependencies.modulelist deleted file mode 100644 index c496e6d..0000000 --- a/paimon_python_java/paimon-python-java-bridge/tools/ci/paimon-ci-tools/src/main/resources/modules-defining-excess-dependencies.modulelist +++ /dev/null @@ -1,20 +0,0 @@ -################################################################################ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -################################################################################ - -# This file lists modules which define additional dependencies, not shown by the maven shade plugin output. -# These are usually undeclared shaded (or otherwise included) dependencies from transitive dependencies. From 489792d23c7b755e53897528f2a4405679cdd77d Mon Sep 17 00:00:00 2001 From: yuzelin Date: Fri, 22 Nov 2024 14:24:39 +0800 Subject: [PATCH 5/5] fix --- paimon_python_java/paimon-python-java-bridge/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paimon_python_java/paimon-python-java-bridge/pom.xml b/paimon_python_java/paimon-python-java-bridge/pom.xml index e825956..dd7f7f2 100644 --- a/paimon_python_java/paimon-python-java-bridge/pom.xml +++ b/paimon_python_java/paimon-python-java-bridge/pom.xml @@ -23,7 +23,7 @@ org.apache.paimon paimon-python-java-bridge - 0.1-SNAPSHOT + 0.9-SNAPSHOT Paimon : Python-Java Bridge jar