diff --git a/app.cfg.example b/app.cfg.example index be8e2198..374f209f 100644 --- a/app.cfg.example +++ b/app.cfg.example @@ -17,6 +17,10 @@ # Also see documentation at https://github.com/EESSI/eessi-bot-software-layer/blob/main/README.md#step5.5 +[git] +# Name of the Git hosting platform (supported values are 'github' and 'gitlab') +hosting_platform = github + [github] # API timeout, time limit for requests to GitHub's REST API api_timeout = 10 @@ -43,6 +47,20 @@ installation_id = 12345678 # path to the private key that was generated when the GitHub App was registered private_key = PATH_TO_PRIVATE_KEY +[gitlab] +# NOTE: Access token required for GitLab: +# https://docs.gitlab.com/user/project/settings/project_access_tokens/#bot-users-for-projects +# Must be stored in environment variable 'GITLAB_PROJECT_ACCESS_TOKEN'. + +# API timeout, time limit for requests to GitLab's REST API +api_timeout = 10 + +# Name used to refer to your bot instance - see comment for github.app_name config +bot_name = MY-bot + +# The base URL of your GitLab instance +instance_url = https://gitlab.com + [bot_control] # which GH accounts have the permission to send commands to the bot diff --git a/connections/gitlab.py b/connections/gitlab.py new file mode 100644 index 00000000..30da0858 --- /dev/null +++ b/connections/gitlab.py @@ -0,0 +1,87 @@ +# This file is part of the EESSI build-and-deploy bot, +# see https://github.com/EESSI/eessi-bot-software-layer +# +# The bot helps with requests to add software installations to the +# EESSI software layer, see https://github.com/EESSI/software-layer +# +# author: Sondre Bergsvaag Risanger (@sondrebr) +# +# license: GPLv2 +# + +# Standard library imports +import os + +# Third party imports (anything installed into the local Python environment) +import gitlab + +# Local application imports (anything from EESSI/eessi-bot-software-layer) +from tools import config, logging + + +_gl = None + + +def verify_connection(gl): + """ + Verifies connection to GitLab. Exits if verification fails. + + Args: + Instance of gitlab.Gitlab (from python-gitlab) + + Returns: + None (implicit) + """ + try: + # auth tests the instance's credentials by retrieving the access token user + gl.auth() + if type(gl.user) is not gl._objects.CurrentUser: + raise Exception("'user' attribute of Gitlab class instance is not of type 'CurrentUser'.") + except Exception as err: + logging.error(f"Failed to verify GitLab connection: {err}") + + +def connect(): + """ + Creates a gitlab.Gitlab instance (from python-gitlab), then verifies the connection to GitLab. + + Args: + No arguments + + Returns: + None (implicit) + """ + global _gl + cfg = config.read_config() + gitlab_cfg = cfg[config.SECTION_GITLAB] + timeout = int(gitlab_cfg.get(config.GITLAB_SETTING_API_TIMEOUT, 10)) + url = gitlab_cfg.get(config.GITLAB_SETTING_INSTANCE_URL) + + access_token = os.getenv('GITLAB_PROJECT_ACCESS_TOKEN') + if access_token is None: + logging.error("GitLab token is not available via $GITLAB_PROJECT_ACCESS_TOKEN!") + else: + del os.environ['GITLAB_PROJECT_ACCESS_TOKEN'] + + _gl = gitlab.Gitlab(url, access_token) + _gl.timeout = timeout + _gl.retry_transient_errors = True + verify_connection(_gl) + + +def get_instance(): + """ + Returns a gitlab.Gitlab instance. Creates an instance if one does not exist, + otherwise verifies the existing instance. + + Args: + No arguments + + Returns: + Instance of gitlab.Gitlab (from python-gitlab) + """ + if not _gl: + connect() + else: + verify_connection(_gl) + return _gl diff --git a/eessi_bot_event_handler.py b/eessi_bot_event_handler.py index 41787e1b..2967ad5d 100644 --- a/eessi_bot_event_handler.py +++ b/eessi_bot_event_handler.py @@ -37,6 +37,8 @@ from tools.args import event_handler_parse from tools.commands import EESSIBotCommand, EESSIBotCommandError, \ contains_any_bot_command, get_bot_command +from tools.event_info import create_event_info_instance +from tools.git import connect_to_git_hosting_platform, get_git_hosting_platform from tools.permissions import check_command_permission from tools.pr_comments import ChatLevels, create_comment @@ -95,12 +97,18 @@ config.DOWNLOAD_PR_COMMENTS_SETTING_PR_DIFF_TIP], # required config.SECTION_EVENT_HANDLER: [ config.EVENT_HANDLER_SETTING_LOG_PATH], # required + config.SECTION_GIT: [ + config.GIT_SETTING_HOSTING_PLATFORM], # required config.SECTION_GITHUB: [ - config.GITHUB_SETTING_API_TIMEOUT, # required - config.GITHUB_SETTING_APP_ID, # required - config.GITHUB_SETTING_APP_NAME, # required - config.GITHUB_SETTING_INSTALLATION_ID, # required - config.GITHUB_SETTING_PRIVATE_KEY], # required + config.GITHUB_SETTING_API_TIMEOUT, # required for github + config.GITHUB_SETTING_APP_ID, # required for github + config.GITHUB_SETTING_APP_NAME, # required for github + config.GITHUB_SETTING_INSTALLATION_ID, # required for github + config.GITHUB_SETTING_PRIVATE_KEY], # required for github + config.SECTION_GITLAB: [ + config.GITLAB_SETTING_API_TIMEOUT, # required for gitlab + config.GITLAB_SETTING_BOT_NAME, # required for gitlab + config.GITLAB_SETTING_INSTANCE_URL], # required for gitlab # the poll interval setting is required for the alternative job handover # protocol (delayed_begin) config.SECTION_JOB_MANAGER: [ @@ -133,7 +141,8 @@ def __init__(self, *args, **kwargs): EESSIBotSoftwareLayer constructor. Calls constructor of PyGHee and initializes some configuration settings. """ - super(EESSIBotSoftwareLayer, self).__init__(*args, **kwargs) + event_source = get_git_hosting_platform() + super(EESSIBotSoftwareLayer, self).__init__(event_source, *args, **kwargs) self.cfg = config.read_config() event_handler_cfg = self.cfg[config.SECTION_EVENT_HANDLER] @@ -157,6 +166,22 @@ def log(self, msg, *args): msg = "[%s]: %s" % (funcname, msg) log(msg, log_file=self.logfile) + def handle_event(self, event_info, log_file=None): + """ + Override of PyGHee's handle_event method. + Create EventInfo instance using event_info, + then pass that to PyGHee's handle_event method. + + Args: + event_info (dict): event received by event_handler + log_file (string): path to log messages to + + Returns: + None (implicit) + """ + event_info_object = create_event_info_instance(event_info) + super().handle_event(event_info_object, log_file) + def handle_issue_comment_event(self, event_info, log_file=None): """ Handle events of type issue_comment. Main action is to parse new issue @@ -833,7 +858,9 @@ def main(): else: print("Configuration check: FAILED") sys.exit(1) - github.connect() + + # Verify that the event handler is able to connect to the Git hosting platform + connect_to_git_hosting_platform() if opts.file: app = create_app(klass=EESSIBotSoftwareLayer) diff --git a/requirements.txt b/requirements.txt index faecf277..35c6ecf4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,11 +7,14 @@ # author: Bob Droege (@bedroge) # author: Kenneth Hoste (@boegel) # author: Thomas Roeblitz (@trz42) +# author: Sondre Bergsvaag Risanger (@sondrebr) # # license: GPLv2 # PyGithub +python-gitlab==6.5.0;python_version=="3.9" # Last version with Python 3.9 support +python-gitlab==8.3.0;python_version>="3.10" # Most recent version on 2026-05-04 Waitress>=3.0.1 # required to fix vulnerabilities detected by scorecards cryptography>=44.0.1 # required to fix vulnerabilities detected by scorecards -PyGHee>=0.0.3 +PyGHee @ git+https://github.com/boegel/PyGHee.git@c5e10632a45db5ca94f5cbf87ac7a90a2064e8fd # Pin commit with GL support retry diff --git a/tools/config.py b/tools/config.py index 10a7590d..fbe7e820 100644 --- a/tools/config.py +++ b/tools/config.py @@ -23,6 +23,7 @@ # (none yet) # Local application imports (anything from EESSI/eessi-bot-software-layer) +from .git import get_git_hosting_platform, SUPPORTED_GIT_HOSTS from .logging import error # define configuration constants @@ -95,6 +96,9 @@ FINISHED_JOB_COMMENTS_SETTING_JOB_RESULT_UNKNOWN_FMT = 'job_result_unknown_fmt' FINISHED_JOB_COMMENTS_SETTING_JOB_TEST_UNKNOWN_FMT = 'job_test_unknown_fmt' +SECTION_GIT = 'git' +GIT_SETTING_HOSTING_PLATFORM = 'hosting_platform' + SECTION_GITHUB = 'github' GITHUB_SETTING_API_TIMEOUT = 'api_timeout' GITHUB_SETTING_APP_ID = 'app_id' @@ -102,6 +106,11 @@ GITHUB_SETTING_INSTALLATION_ID = 'installation_id' GITHUB_SETTING_PRIVATE_KEY = 'private_key' +SECTION_GITLAB = 'gitlab' +GITLAB_SETTING_API_TIMEOUT = 'api_timeout' +GITLAB_SETTING_BOT_NAME = 'bot_name' +GITLAB_SETTING_INSTANCE_URL = 'instance_url' + SECTION_JOB_MANAGER = 'job_manager' JOB_MANAGER_SETTING_LOG_PATH = 'log_path' JOB_MANAGER_SETTING_JOB_IDS_DIR = 'job_ids_dir' @@ -207,9 +216,13 @@ def check_cfg_settings(req_settings, path="app.cfg"): """ # TODO argument path is not being used cfg = read_config() + git_host = get_git_hosting_platform(cfg) # iterate over keys in req_settings which correspond to sections ([name]) # in the configuration file (.ini format) for section in req_settings.keys(): + # Skip checking the GitLab section if the bot is configured for GitHub and vice versa + if git_host and (section in SUPPORTED_GIT_HOSTS) and (section != git_host): + continue if section not in cfg: error(f'Missing section "{section}" in configuration file {path}.') # iterate over list elements required for the current section diff --git a/tools/event_info.py b/tools/event_info.py new file mode 100644 index 00000000..2d1a902c --- /dev/null +++ b/tools/event_info.py @@ -0,0 +1,336 @@ +# This file is part of the EESSI build-and-deploy bot, +# see https://github.com/EESSI/eessi-bot-software-layer +# +# The bot helps with requests to add software installations to the +# EESSI software layer, see https://github.com/EESSI/software-layer +# +# author: Sondre Bergsvaag Risanger (@sondrebr) +# +# license: GPLv2 +# + +# Standard library imports +from functools import cached_property + +# Third party imports (anything installed into the local Python environment) +# (none) + +# Local application imports (anything from EESSI/eessi-bot-software-layer) +from connections import gitlab +from tools.git import get_git_hosting_platform, GITHUB, GITLAB + + +class BaseEventInfo(): + """ + Base class to use for handling event info, which works differently + for GitHub vs. GitLab. Subscripting is implemented for compatibility. + If a new field needs to be accessed, add a new property to + retrieve it instead of subscripting/using the event_info dict. + """ + def __init__(self, event_info): + if self.__class__ is BaseEventInfo: + err_msg = "Do not use this base class directly. " + err_msg += "Please use one of its subclasses instead." + raise NotImplementedError(err_msg) + self.event_info = event_info + + # Prevents subclasses from overriding __getitem__ + def __init_subclass__(cls, **kwargs): + super().__init_subclass__(**kwargs) + if "__getitem__" in cls.__dict__: + raise Exception(f"{cls.__name__} must not override __getitem__") + + # Do not override - implements subscripting for compatibility + def __getitem__(self, key): + return self.event_info[key] + + @cached_property + def action(self): + raise NotImplementedError() + + @cached_property + def comment_id(self): + raise NotImplementedError() + + @cached_property + def comment_body(self): + raise NotImplementedError() + + @cached_property + def comment_created_by(self): + raise NotImplementedError() + + @cached_property + def event_id(self): + raise NotImplementedError() + + @cached_property + def event_triggered_by(self): + raise NotImplementedError() + + @cached_property + def event_type(self): + raise NotImplementedError() + + @cached_property + def issue_number(self): + raise NotImplementedError() + + @cached_property + def issue_url(self): + raise NotImplementedError() + + @cached_property + def label_name(self): + raise NotImplementedError() + + @cached_property + def pr_number(self): + raise NotImplementedError() + + @cached_property + def pr_merged_status(self): + raise NotImplementedError() + + @cached_property + def pr_url(self): + raise NotImplementedError() + + @cached_property + def repo_name(self): + raise NotImplementedError() + + +class GitHubEventInfo(BaseEventInfo): + """ + EventInfo class for use with GitHub webhooks. + """ + def __init__(self, event_info): + super().__init__(event_info) + self._request_body = event_info["raw_request_body"] + + @cached_property + def action(self): + return self.event_info["action"] + + @cached_property + def comment_id(self): + return self._request_body["comment"]["id"] + + @cached_property + def comment_body(self): + return self._request_body["comment"]["body"] + + @cached_property + def comment_created_by(self): + return self._request_body["comment"]["user"]["login"] + + @cached_property + def event_id(self): + return self.event_info["id"] + + @cached_property + def event_triggered_by(self): + return self._request_body["sender"]["login"] + + @cached_property + def event_type(self): + return self.event_info["type"] + + @cached_property + def issue_number(self): + return self._request_body["issue"]["number"] + + @cached_property + def issue_url(self): + return self._request_body["issue"]["html_url"] + + @cached_property + def label_name(self): + return self._request_body["label"]["name"] + + @cached_property + def pr_number(self): + return self._request_body["pull_request"]["number"] + + @cached_property + def pr_merged_status(self): + return self._request_body["pull_request"]["merged"] + + @cached_property + def pr_url(self): + return self._request_body["pull_request"]["html_url"] + + @cached_property + def repo_name(self): + return self._request_body["repository"]["full_name"] + + +class GitLabEventInfo(BaseEventInfo): + """ + EventInfo class for use with GitLab webhooks. Converts GL terminology to + GH equivalents where needed, e.g. event type 'note' becomes 'issue_comment'. + """ + def __init__(self, event_info): + super().__init__(event_info) + self._request_body = event_info["raw_request_body"] + self._object_attributes = self._request_body.get("object_attributes", {}) + + # Map GitLab actions to GitHub actions + _ACTION_MAP = { + # Note -> comment actions + "create": "created", + "update": "edited", + "delete": "deleted", + + # MR -> PR actions + # MR 'update' handled separately + "open": "opened", + "merge": "closed", + "close": "closed", + } + _UNKNOWN = "UNKNOWN" + + @cached_property + def action(self): + gl_action = self._object_attributes["action"] + # GL uses a single 'update' action for MRs + # Need to check changes to find exact action, e.g. 'labeled' + if self.event_type == "pull_request" and gl_action == "update": + changes = self._request_body["changes"] + if "labels" in changes: + action = "labeled" + else: + action = self._UNKNOWN + else: + action = self._ACTION_MAP.get(gl_action, self._UNKNOWN) + return action + + @cached_property + def comment_id(self): + return self._object_attributes["id"] + + @cached_property + def comment_body(self): + return self._object_attributes["note"] + + @cached_property + def comment_created_by(self): + created_by_id = self._object_attributes["author_id"] + triggered_by_id = self._request_body["user"]["id"] + # GL events only include the username of the user who triggered the event. + # E.g., if a comment was updated by someone other than the original author, + # we need to retrieve the name of the author from the server. + if triggered_by_id == created_by_id: + created_by = self._request_body["user"]["username"] + else: + gl = gitlab.get_instance() + user = gl.users.get(created_by_id) + created_by = user.username + return created_by + + @cached_property + def event_id(self): + return self.event_info["id"] + + @cached_property + def event_triggered_by(self): + return self._request_body["user"]["username"] + + # Map (relevant) GitLab events to GitHub events + _EVENT_TYPE_MAP = { + "note": "issue_comment", + "merge_request": "pull_request", + } + + @cached_property + def event_type(self): + gl_event_type = self.event_info["type"] + return self._EVENT_TYPE_MAP.get(gl_event_type, self._UNKNOWN) + + # The bot does not handle issue events, but comment events can come from both issue and MR comments. + # We therefore need to check what type of comment it is to get the issue numbers and URLs. + @cached_property + def issue_number(self): + notable_type = self._object_attributes["notable_type"] + if notable_type == "MergeRequest": + issue_iid = self._request_body["merge_request"]["iid"] + elif notable_type == "Issue": + issue_iid = self._request_body["issue"]["iid"] + else: + # Comments may also come from commits etc. - default to -1 + issue_iid = -1 + return issue_iid + + @cached_property + def issue_url(self): + notable_type = self._object_attributes["notable_type"] + if notable_type == "MergeRequest": + issue_url = self._request_body["merge_request"]["url"] + elif notable_type == "Issue": + issue_url = self._request_body["issue"]["url"] + else: + # Comments may also come from commits etc. - default to empty string + issue_url = "" + return issue_url + + @cached_property + def label_name(self): + # GL sends a single event containing all previous and current labels. + # Since we currently only use one label, 'bot:deploy', we can check just for that. + label_changes = self._request_body["changes"]["labels"] + # The difference between the sets will yield all newly added labels + added_labels = set(label_changes["current"]) - set(label_changes["previous"]) + if "bot:deploy" in added_labels: + return "bot:deploy" + else: + return None + + # GL uses the 'object_attributes' field to store data about the event object. + # For example, MR events store information about the MR in 'object_attributes', while + # events from comments on MRs store information about the MR in the 'merge_request' field. + @cached_property + def pr_number(self): + if self.event_type == "pull_request": + pr_iid = self._object_attributes["iid"] + else: + pr_iid = self._request_body["merge_request"]["iid"] + return pr_iid + + @cached_property + def pr_merged_status(self): + if self.event_type == "pull_request": + state = self._object_attributes["state"] + else: + state = self._request_body["merge_request"]["state"] + return state == "merged" + + @cached_property + def pr_url(self): + if self.event_type == "pull_request": + url = self._object_attributes["url"] + else: + url = self._request_body["merge_request"]["url"] + return url + + @cached_property + def repo_name(self): + return self._request_body["project"]["path_with_namespace"] + + +def create_event_info_instance(event_info): + """ + Creates an EventInfo instance for the configured Git hosting platform. + + Args: + event_info (dict): The event info dictionary created by PyGHee + + Returns: + Instance of BaseEventInfo subclass + """ + git_host = get_git_hosting_platform() + if git_host == GITHUB: + return GitHubEventInfo(event_info) + elif git_host == GITLAB: + return GitLabEventInfo(event_info) + return None diff --git a/tools/git.py b/tools/git.py new file mode 100644 index 00000000..25a61a57 --- /dev/null +++ b/tools/git.py @@ -0,0 +1,73 @@ +# This file is part of the EESSI build-and-deploy bot, +# see https://github.com/EESSI/eessi-bot-software-layer +# +# The bot helps with requests to add software installations to the +# EESSI software layer, see https://github.com/EESSI/software-layer +# +# author: Sondre Bergsvaag Risanger (@sondrebr) +# +# license: GPLv2 +# + +# Standard library imports +# (none) + +# Third party imports (anything installed into the local Python environment) +# (none) + +# Local application imports (anything from EESSI/eessi-bot-software-layer) +from connections import github, gitlab +from tools import config, logging + + +GITHUB = "github" +GITLAB = "gitlab" + +SUPPORTED_GIT_HOSTS = { + GITHUB, + GITLAB, +} + +_git_host = None + + +def get_git_hosting_platform(cfg=None): + """ + Read the config and get the Git hosting platform the bot is configured for. + Exit if the setting is invalid or not set. + + Args: + cfg (ConfigParser): Instance of ConfigParser containing the configuration. + May be passed by caller to avoid re-reading the configuration file. + + Returns: + (str): The configured Git hosting platform + """ + global _git_host + if not _git_host: + if not cfg: + cfg = config.read_config() + _git_host = cfg.get(config.SECTION_GIT, config.GIT_SETTING_HOSTING_PLATFORM, fallback=None) + if _git_host not in SUPPORTED_GIT_HOSTS: + logging.error(f"Invalid Git host configured: '{_git_host}'") + return _git_host + + +def connect_to_git_hosting_platform(): + """ + Establish connection to Git hosting platform. Exit if the configured hosting + platform is not supported by the bot. + + Args: + No arguments + + Returns: + None (implicit) + """ + git_host = get_git_hosting_platform() + if git_host == GITHUB: + github.connect() + elif git_host == GITLAB: + gitlab.connect() + else: + logging.error(f"Git host not supported: '{git_host}'")