Merge latest skeleton

Signed-off-by: Philippe Ombredanne <pombredanne@nexb.com>

Author: pombredanne

.github/workflows/docs-ci.yml

@@ -0,0 +1,37 @@
+name: CI Documentation
+
+on: [push, pull_request]
+
+jobs:
+  build:
+    runs-on: ubuntu-20.04
+
+    strategy:
+      max-parallel: 4
+      matrix:
+        python-version: [3.9]
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Give permission to run scripts
+        run: chmod +x ./docs/scripts/doc8_style_check.sh
+
+      - name: Install Dependencies
+        run:  pip install -e .[docs]
+
+      - name: Check Sphinx Documentation build minimally
+        working-directory: ./docs
+        run: sphinx-build -E -W source build
+
+      - name: Check for documentation style errors
+        working-directory: ./docs
+        run: ./scripts/doc8_style_check.sh
+
+

.github/workflows/pypi-release.yml

@@ -0,0 +1,27 @@
+name: Release library as a PyPI wheel and sdist on GH release creation
+
+on:
+  release:
+    types: [created]
+
+jobs:
+  build-and-publish-to-pypi:
+    name: Build and publish library to PyPI
+    runs-on: ubuntu-20.04
+    steps:
+     - uses: actions/checkout@master
+     - name: Set up Python
+       uses: actions/setup-python@v1
+       with:
+         python-version: 3.9
+     - name: Install pypa/build
+       run: python -m pip install build --user
+     - name: Build a binary wheel and a source tarball
+       run: python -m build --sdist --wheel --outdir dist/
+        .
+     - name: Publish distribution  to PyPI
+       if: startsWith(github.ref, 'refs/tags')
+       uses: pypa/gh-action-pypi-publish@master
+       with:
+         password: ${{ secrets.PYPI_API_TOKEN }}
+        

.gitignore

@@ -12,6 +12,7 @@
 /Lib
 /pip-selfcheck.json
 /tmp
+/venv
 .Python
 /include
 /Include

.travis.yml

@@ -7,58 +7,50 @@
 
 jobs:
 
-    - template: etc/ci/azure-posix.yml
-      parameters:
-          job_name: ubuntu16_cpython
-          image_name: ubuntu-16.04
-          python_versions: ['3.6', '3.7', '3.8', '3.9']
-          test_suites:
-              all: tmp/bin/pytest -vvs
-
     - template: etc/ci/azure-posix.yml
       parameters:
           job_name: ubuntu18_cpython
           image_name: ubuntu-18.04
-          python_versions: ['3.6', '3.7', '3.8', '3.9']
+          python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10']
           test_suites:
-              all: tmp/bin/pytest -n 2 -vvs
+              all: venv/bin/pytest -n 2 -vvs
 
     - template: etc/ci/azure-posix.yml
       parameters:
           job_name: ubuntu20_cpython
           image_name: ubuntu-20.04
-          python_versions: ['3.6', '3.7', '3.8', '3.9']
+          python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10']
           test_suites:
-              all: tmp/bin/pytest -n 2 -vvs
+              all: venv/bin/pytest -n 2 -vvs
 
     - template: etc/ci/azure-posix.yml
       parameters:
-          job_name: macos1014_cpython
-          image_name: macos-10.14
-          python_versions: ['3.6', '3.7', '3.8', '3.9']
+          job_name: macos1015_cpython
+          image_name: macos-10.15
+          python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10']
           test_suites:
-              all: tmp/bin/pytest -n 2 -vvs
+              all: venv/bin/pytest -n 2 -vvs
 
     - template: etc/ci/azure-posix.yml
       parameters:
-          job_name: macos1015_cpython
-          image_name: macos-10.15
-          python_versions: ['3.6', '3.7', '3.8', '3.9']
+          job_name: macos11_cpython
+          image_name: macos-11
+          python_versions: ['3.7', '3.8', '3.9', '3.10']
           test_suites:
-              all: tmp/bin/pytest -n 2 -vvs
+              all: venv/bin/pytest -n 2 -vvs
 
     - template: etc/ci/azure-win.yml
       parameters:
-          job_name: win2016_cpython
-          image_name: vs2017-win2016
-          python_versions: ['3.6', '3.7', '3.8', '3.9']
+          job_name: win2019_cpython
+          image_name: windows-2019
+          python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10']
           test_suites:
-              all: tmp\Scripts\pytest -n 2 -vvs
+              all: venv\Scripts\pytest -n 2 -vvs
 
     - template: etc/ci/azure-win.yml
       parameters:
-          job_name: win2019_cpython
-          image_name: windows-2019
-          python_versions: ['3.6', '3.7', '3.8', '3.9']
+          job_name: win2022_cpython
+          image_name: windows-2022
+          python_versions: ['3.7', '3.8', '3.9', '3.10']
           test_suites:
-              all: tmp\Scripts\pytest -n 2 -vvs
+              all: venv\Scripts\pytest -n 2 -vvs

azure-pipelines.yml

@@ -11,11 +11,13 @@ set -e
 #set -x
 
 ################################
-# A configuration script to set things up: 
+# A configuration script to set things up:
 # create a virtualenv and install or update thirdparty packages.
 # Source this script for initial configuration
 # Use configure --help for details
 #
+# NOTE: please keep in sync with Windows script configure.bat
+#
 # This script will search for a virtualenv.pyz app in etc/thirdparty/virtualenv.pyz
 # Otherwise it will download the latest from the VIRTUALENV_PYZ_URL default
 ################################
@@ -26,16 +28,15 @@ CLI_ARGS=$1
 ################################
 
 # Requirement arguments passed to pip and used by default or with --dev.
-REQUIREMENTS="--editable .[full]"
-DEV_REQUIREMENTS="--editable .[full,testing]"
+REQUIREMENTS="--editable .[full] --constraint requirements.txt"
+DEV_REQUIREMENTS="--editable .[full,testing] --constraint requirements.txt --constraint requirements-dev.txt"
+DOCS_REQUIREMENTS="--editable .[docs] --constraint requirements.txt"
 
 # where we create a virtualenv
-VIRTUALENV_DIR=tmp
+VIRTUALENV_DIR=venv
 
-# Cleanable files and directories with the --clean option
-CLEANABLE="
-    build
-	tmp"
+# Cleanable files and directories to delete with the --clean option
+CLEANABLE="build venv"
 
 # extra  arguments passed to pip
 PIP_EXTRA_ARGS=" "
@@ -52,59 +53,38 @@ CFG_BIN_DIR=$CFG_ROOT_DIR/$VIRTUALENV_DIR/bin
 
 
 ################################
-# Set the quiet flag to empty if not defined 
+# Thirdparty package locations and index handling
+# Find packages from the local thirdparty directory or from thirdparty.aboutcode.org
+if [ -d "$CFG_ROOT_DIR/thirdparty" ]; then
+    PIP_EXTRA_ARGS="--find-links $CFG_ROOT_DIR/thirdparty"
+fi
+PIP_EXTRA_ARGS="$PIP_EXTRA_ARGS --find-links https://thirdparty.aboutcode.org/pypi/simple/links.html"
+
+
+################################
+# Set the quiet flag to empty if not defined
 if [[ "$CFG_QUIET" == "" ]]; then
     CFG_QUIET=" "
 fi
 
 
 ################################
-# find a proper Python to run
+# Find a proper Python to run
 # Use environment variables or a file if available.
 # Otherwise the latest Python by default.
-if [[ "$PYTHON_EXECUTABLE" == "" ]]; then
-    # check for a file named PYTHON_EXECUTABLE 
-    if [ -f "$CFG_ROOT_DIR/PYTHON_EXECUTABLE" ]; then
-        PYTHON_EXECUTABLE=$(cat "$CFG_ROOT_DIR/PYTHON_EXECUTABLE")
-    else
-        PYTHON_EXECUTABLE=python3
+find_python() {
+    if [[ "$PYTHON_EXECUTABLE" == "" ]]; then
+        # check for a file named PYTHON_EXECUTABLE
+        if [ -f "$CFG_ROOT_DIR/PYTHON_EXECUTABLE" ]; then
+            PYTHON_EXECUTABLE=$(cat "$CFG_ROOT_DIR/PYTHON_EXECUTABLE")
+        else
+            PYTHON_EXECUTABLE=python3
+        fi
     fi
-fi
-
-
-################################
-cli_help() {
-    echo An initial configuration script
-    echo "  usage: ./configure [options]"
-    echo
-    echo The default is to configure for regular use. Use --dev for development.
-    echo
-    echo The options are:
-    echo " --clean: clean built and installed files and exit."
-    echo " --dev:   configure the environment for development."
-    echo " --help:  display this help message and exit."
-    echo
-    echo By default, the python interpreter version found in the path is used.
-    echo Alternatively, the PYTHON_EXECUTABLE environment variable can be set to
-    echo configure another Python executable interpreter to use. If this is not
-    echo set, a file named PYTHON_EXECUTABLE containing a single line with the
-    echo path of the Python executable to use will be checked last.
-    set +e
-    exit
-}
-
-
-clean() {
-    # Remove cleanable file and directories and files from the root dir.
-    echo "* Cleaning ..."
-    for cln in $CLEANABLE;
-        do rm -rf "${CFG_ROOT_DIR:?}/${cln:?}";
-    done
-    set +e
-    exit
 }
 
 
+################################
 create_virtualenv() {
     # create a virtualenv for Python
     # Note: we do not use the bundled Python 3 "venv" because its behavior and
@@ -120,7 +100,7 @@ create_virtualenv() {
             VIRTUALENV_PYZ="$CFG_ROOT_DIR/etc/thirdparty/virtualenv.pyz"
         else
             VIRTUALENV_PYZ="$CFG_ROOT_DIR/$VENV_DIR/virtualenv.pyz"
-            wget -O "$VIRTUALENV_PYZ" "$VIRTUALENV_PYZ_URL"
+            wget -O "$VIRTUALENV_PYZ" "$VIRTUALENV_PYZ_URL" 2>/dev/null || curl -o  "$VIRTUALENV_PYZ" "$VIRTUALENV_PYZ_URL"
         fi
 
         $PYTHON_EXECUTABLE "$VIRTUALENV_PYZ" \
@@ -135,6 +115,7 @@ create_virtualenv() {
 }
 
 
+################################
 install_packages() {
     # install requirements in virtualenv
     # note: --no-build-isolation means that pip/wheel/setuptools will not
@@ -151,19 +132,63 @@ install_packages() {
 }
 
 
+################################
+cli_help() {
+    echo An initial configuration script
+    echo "  usage: ./configure [options]"
+    echo
+    echo The default is to configure for regular use. Use --dev for development.
+    echo
+    echo The options are:
+    echo " --clean: clean built and installed files and exit."
+    echo " --dev:   configure the environment for development."
+    echo " --help:  display this help message and exit."
+    echo
+    echo By default, the python interpreter version found in the path is used.
+    echo Alternatively, the PYTHON_EXECUTABLE environment variable can be set to
+    echo configure another Python executable interpreter to use. If this is not
+    echo set, a file named PYTHON_EXECUTABLE containing a single line with the
+    echo path of the Python executable to use will be checked last.
+    set +e
+    exit
+}
+
+
+################################
+clean() {
+    # Remove cleanable file and directories and files from the root dir.
+    echo "* Cleaning ..."
+    for cln in $CLEANABLE;
+        do rm -rf "${CFG_ROOT_DIR:?}/${cln:?}";
+    done
+    set +e
+    exit
+}
+
+
 ################################
 # Main command line entry point
-CFG_DEV_MODE=0
 CFG_REQUIREMENTS=$REQUIREMENTS
 
-case "$CLI_ARGS" in
-    --help)    cli_help;;
-    --clean)   clean;;
-    --dev)     CFG_REQUIREMENTS="$DEV_REQUIREMENTS" && CFG_DEV_MODE=1;;
-esac
-
+# We are using getopts to parse option arguments that start with "-"
+while getopts :-: optchar; do
+    case "${optchar}" in
+        -)
+            case "${OPTARG}" in
+                help  ) cli_help;;
+                clean ) find_python && clean;;
+                dev   ) CFG_REQUIREMENTS="$DEV_REQUIREMENTS";;
+                docs   ) CFG_REQUIREMENTS="$DOCS_REQUIREMENTS";;
+            esac;;
+    esac
+done
+
+PIP_EXTRA_ARGS="$PIP_EXTRA_ARGS"
+
+find_python
 create_virtualenv "$VIRTUALENV_DIR"
 install_packages "$CFG_REQUIREMENTS"
 . "$CFG_BIN_DIR/activate"
 
+
 set +e