Merge pull request #293 from victormlg/jakubPR

ENT-12974 Implemented absolute path Git repository module

Author: larsewi

JSON.md

@@ -174,9 +174,10 @@ The modules inside `build`, `provides`, and `index` use these fields:
   For `provides` and `index` dictionaries, this name must be the key of each entry (not a field inside).
   For the `build` array, it must be inside each module object (with `name` as the key).
   Local modules (files and folders in same directory as `cfbs.json`), must start with `./`, and end with `/` if it's a directory.
+  Absolute modules (a directory given by absolute path containing a Git repository) must start with `/` and end with `/`.
   Module names should not be longer than 64 characters.
-  Module names (not including adfixes `./`, `/`, `.cf`, `.json` for local modules) should only contain lowercase ASCII alphanumeric characters possibly separated by dashes, and should start with a letter.
-  Local module names can contain underscores instead of dashes.
+  Module names (not including adfixes `./`, `/`, `.cf`, `.json` for local and absolute modules) should only contain lowercase ASCII alphanumeric characters possibly separated by dashes, and should start with a letter.
+  Local and absolute module names can contain underscores instead of dashes.
 - `description` (string): Human readable description of what this module does.
 - `tags` (array of strings): Mostly used for information / finding modules on [build.cfengine.com](https://build.cfengine.com).
   Some common examples include `supported`, `experimental`, `security`, `library`, `promise-type`.

cfbs/cfbs_config.py

@@ -39,7 +39,12 @@
 )
 from cfbs.pretty import pretty, CFBS_DEFAULT_SORTING_RULES
 from cfbs.cfbs_json import CFBSJson
-from cfbs.module import Module, is_module_added_manually, is_module_local
+from cfbs.module import (
+    Module,
+    is_module_added_manually,
+    is_module_local,
+    is_module_absolute,
+)
 from cfbs.prompts import prompt_user, prompt_user_yesno
 from cfbs.validate import validate_single_module
 
@@ -337,8 +342,12 @@ def _handle_local_module(self, module, use_default_build_steps=True):
             name.startswith("./")
             and name.endswith((".cf", "/"))
             and "local" in module["tags"]
+        ) and not (
+            name.startswith("/") and name.endswith("/") and "absolute" in module["tags"]
         ):
-            log.debug("Module '%s' does not appear to be a local module" % name)
+            log.debug(
+                "Module '%s' do not appear to be a local or absolute module" % name
+            )
             return
 
         if name.endswith(".cf"):
@@ -486,6 +495,12 @@ def add_command(
                     "URI scheme not supported. The supported URI schemes are: "
                     + ", ".join(SUPPORTED_URI_SCHEMES)
                 )
+            for m in to_add:
+                if is_module_absolute(m):
+                    if not os.path.exists(m):
+                        raise CFBSUserError("Absolute path module doesn't exist")
+                    if not os.path.isdir(m):
+                        raise CFBSUserError("Absolute path module is not a dir")
             self._add_modules(to_add, added_by, checksum, explicit_build_steps)
 
         added = {m["name"] for m in self["build"]}.difference(before)

cfbs/commands.py

@@ -104,6 +104,7 @@ def search_command(terms: List[str]):
     validate_single_module,
 )
 from cfbs.internal_file_management import (
+    absolute_module_copy,
     clone_url_repo,
     SUPPORTED_URI_SCHEMES,
     fetch_archive,
@@ -116,11 +117,12 @@ def search_command(terms: List[str]):
     git_configure_and_initialize,
     is_git_repo,
     CFBSGitError,
+    head_commit_hash,
 )
 
 from cfbs.git_magic import commit_after_command, git_commit_maybe_prompt
 from cfbs.prompts import prompt_user, prompt_user_yesno
-from cfbs.module import Module, is_module_added_manually
+from cfbs.module import Module, is_module_absolute, is_module_added_manually
 from cfbs.masterfiles.generate_release_information import generate_release_information
 
 _MODULES_URL = "https://archive.build.cfengine.com/modules"
@@ -634,6 +636,9 @@ def update_command(to_update):
                 continue
 
             new_module = provides[module_name]
+        elif is_module_absolute(old_module["name"]):
+            new_module = index.get_module_object(update.name)
+            new_module["commit"] = head_commit_hash(old_module["name"])
         else:
 
             if "version" not in old_module:
@@ -806,6 +811,10 @@ def _download_dependencies(config: CFBSConfig, redownload=False, ignore_versions
             local_module_copy(module, counter, max_length)
             counter += 1
             continue
+        if name.startswith("/"):
+            absolute_module_copy(module, counter, max_length)
+            counter += 1
+            continue
         if "commit" not in module:
             raise CFBSExitError("module %s must have a commit property" % name)
         commit = module["commit"]

cfbs/git.py

@@ -11,6 +11,7 @@
 import os
 import itertools
 import tempfile
+import shutil
 from subprocess import check_call, check_output, run, PIPE, DEVNULL, CalledProcessError
 from typing import Iterable, Union
 
@@ -258,3 +259,26 @@ def treeish_exists(treeish, repo_path):
     result = run(command, cwd=repo_path, stdout=DEVNULL, stderr=DEVNULL, check=False)
 
     return result.returncode == 0
+
+
+def head_commit_hash(repo_path):
+    result = run(
+        ["git", "rev-parse", "HEAD"],
+        cwd=repo_path,
+        stdout=PIPE,
+        stderr=DEVNULL,
+        check=True,
+    )
+
+    return result.stdout.decode("utf-8").strip()
+
+
+# Ensure reproducibility when copying git repositories
+# 1. hard reset to specific commit
+# 2. remove untracked files
+# 3. remove .git directory
+def git_clean_reset(repo_path, commit):
+    run(["git", "reset", "--hard", commit], cwd=repo_path, check=True, stdout=DEVNULL)
+    run(["git", "clean", "-fxd"], cwd=repo_path, check=True, stdout=DEVNULL)
+    git_dir = os.path.join(repo_path, ".git")
+    shutil.rmtree(git_dir, ignore_errors=True)

cfbs/index.py

@@ -3,9 +3,10 @@
 from collections import OrderedDict
 from typing import List, Optional, Union
 
-from cfbs.module import Module
+from cfbs.git import head_commit_hash, is_git_repo
+from cfbs.module import Module, is_module_absolute
 from cfbs.utils import CFBSNetworkError, get_or_read_json, CFBSExitError, get_json
-from cfbs.internal_file_management import local_module_name
+from cfbs.internal_file_management import absolute_module_name, local_module_name
 
 _DEFAULT_INDEX = (
     "https://raw.githubusercontent.com/cfengine/build-index/master/cfbs.json"
@@ -48,6 +49,30 @@ def _local_module_data_subdir(
         "description": "Local subdirectory added using cfbs command line",
         "tags": ["local"],
         "steps": build_steps,
+        # TODO: turn this into an argument, for when it's not "cfbs add" adding the module
+        "added_by": "cfbs add",
+    }
+
+
+def _absolute_module_data(module_name: str, version: Optional[str]):
+    assert module_name.startswith("/")
+    assert module_name.endswith("/")
+
+    if version is not None:
+        commit_hash = version
+    elif is_git_repo(module_name):
+        commit_hash = head_commit_hash(module_name)
+    else:
+        commit_hash = ""
+
+    dst = os.path.join("services", "cfbs", module_name[1:])
+    build_steps = ["directory ./ {}".format(dst)]
+    return {
+        "description": "Module added via absolute path to a Git repository directory",
+        "tags": ["absolute"],
+        "steps": build_steps,
+        "commit": commit_hash,
+        # TODO: turn this into an argument, for when it's not "cfbs add" adding the module
         "added_by": "cfbs add",
     }
 
@@ -67,6 +92,14 @@ def _generate_local_module_object(
         return _local_module_data_json_file(module_name)
 
 
+def _generate_absolute_module_object(module_name: str, version: Optional[str]):
+    assert module_name.startswith("/")
+    assert module_name.endswith("/")
+    assert os.path.isdir(module_name)
+
+    return _absolute_module_data(module_name, version)
+
+
 class Index:
     """Class representing the cfbs.json containing the index of available modules"""
 
@@ -171,7 +204,10 @@ def translate_alias(self, module: Module):
                 module.name = data["alias"]
         else:
             if os.path.exists(module.name):
-                module.name = local_module_name(module.name)
+                if is_module_absolute(module.name):
+                    module.name = absolute_module_name(module.name)
+                else:
+                    module.name = local_module_name(module.name)
 
     def get_module_object(
         self,
@@ -187,6 +223,13 @@ def get_module_object(
 
         if name.startswith("./"):
             object = _generate_local_module_object(name, explicit_build_steps)
+        elif is_module_absolute(name):
+            if not os.path.isdir(name):
+                pass
+            object = _generate_absolute_module_object(name, version)
+            # currently, the argument of cfbs-add is split by `@` in the `Module` constructor
+            # due to that, this hack is used to prevent creating the "version" field
+            module = Module(name).to_dict()
         else:
             object = self[name]
             if version:

cfbs/internal_file_management.py

@@ -28,12 +28,14 @@
     CFBSExitError,
 )
 
+from cfbs.git import git_clean_reset
+
 _SUPPORTED_TAR_TYPES = (".tar.gz", ".tgz")
 SUPPORTED_ARCHIVES = (".zip",) + _SUPPORTED_TAR_TYPES
 SUPPORTED_URI_SCHEMES = ("https://", "ssh://", "git://")
 
 
-def local_module_name(module_path):
+def local_module_name(module_path: str):
     assert os.path.exists(module_path)
     module = module_path
 
@@ -64,6 +66,27 @@ def local_module_name(module_path):
     return module
 
 
+def absolute_module_name(module_path: str):
+    assert os.path.exists(module_path)
+    module = module_path
+    assert module.startswith("/")
+
+    for illegal in ["//", "..", " ", "\n", "\t", " "]:
+        if illegal in module:
+            raise CFBSExitError("Module path cannot contain %s" % repr(illegal))
+
+    if not module.endswith("/"):
+        module = module + "/"
+    while "/./" in module:
+        module = module.replace("/./", "/")
+
+    assert os.path.exists(module)
+    if not os.path.isdir(module):
+        raise CFBSExitError("'%s' must be a directory" % module)
+
+    return module
+
+
 def get_download_path(module) -> str:
     downloads = os.path.join(cfbs_dir(), "downloads")
 
@@ -117,6 +140,23 @@ def local_module_copy(module, counter, max_length):
     )
 
 
+def absolute_module_copy(module, counter, max_length):
+    assert "commit" in module
+    name = module["name"]
+    pretty_name = _prettify_name(name)
+    target = "out/steps/%03d_%s_local/" % (counter, pretty_name)
+    module["_directory"] = target
+    module["_counter"] = counter
+
+    cp(name, target)
+    git_clean_reset(target, module["commit"])
+
+    print(
+        "%03d %s @ %s                                  (Copied)"
+        % (counter, pad_right(name, max_length), module["commit"][:7])
+    )
+
+
 def _get_path_from_url(url):
     if not url.startswith(SUPPORTED_URI_SCHEMES):
         if "://" in url:

cfbs/module.py

@@ -15,6 +15,13 @@ def is_module_local(name: str):
     return name.startswith("./")
 
 
+def is_module_absolute(name: str):
+    """A module might contain `"absolute"` in its `"tags"` but this is not required.
+    The source of truth for whether the module is absolute is whether it starts with `/`.
+    """
+    return name.startswith("/")
+
+
 class Module:
     """Class representing a module in cfbs.json"""