diff --git a/Makefile b/Makefile index d8d765a3..01fd9237 100644 --- a/Makefile +++ b/Makefile @@ -1,17 +1,17 @@ .PHONY: install build lint pyinstaller clean -venv: - python3 -m venv venv +.venv: + python3 -m venv .venv clean: - rm -rf venv + rm -rf .venv -install: venv - . venv/bin/activate && python -m pip install --editable .[dev] +install: .venv + . .venv/bin/activate && python -m pip install --editable .[dev] -build: venv +build: .venv rm -f README.rst - . venv/bin/activate && python -m build + . .venv/bin/activate && python -m build # Note: "make install" needs to be ran once before this target will work, but we # don't want to set it as a dependency otherwise it unnecessarily slows down @@ -19,12 +19,12 @@ build: venv # package which is linked to the files you are editing so there is no need to # re-install after each change. unit-test: - . venv/bin/activate && pytest test/src/mock + . .venv/bin/activate && pytest test/src/mock -lint: venv +lint: .venv rm -f README.rst - . venv/bin/activate && flake8 src --count --select=E9,F63,F7,F82 --show-source --statistics && flake8 src --count --exit-zero --max-complexity=10 --max-line-length=200 --statistics + . .venv/bin/activate && flake8 src --count --select=E9,F63,F7,F82 --show-source --statistics && flake8 src --count --exit-zero --max-complexity=10 --max-line-length=200 --statistics pyinstaller: venv rm -f README.rst - . venv/bin/activate && pyinstaller src/mas-upgrade --onefile --noconfirm --add-data="src/mas/devops/templates/ibm-mas-tekton.yaml:mas/devops/templates" --add-data="src/mas/devops/templates/subscription.yml.j2:mas/devops/templates/" --add-data="src/mas/devops/templates/pipelinerun-upgrade.yml.j2:mas/devops/templates/" + . .venv/bin/activate && pyinstaller src/mas-upgrade --onefile --noconfirm --add-data="src/mas/devops/templates/ibm-mas-tekton.yaml:mas/devops/templates" --add-data="src/mas/devops/templates/subscription.yml.j2:mas/devops/templates/" --add-data="src/mas/devops/templates/pipelinerun-upgrade.yml.j2:mas/devops/templates/" diff --git a/src/mas/devops/aiservice.py b/src/mas/devops/aiservice.py new file mode 100644 index 00000000..3cd0dc40 --- /dev/null +++ b/src/mas/devops/aiservice.py @@ -0,0 +1,56 @@ +# ***************************************************************************** +# Copyright (c) 2025 IBM Corporation and other Contributors. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +# +# ***************************************************************************** + +import logging +from openshift.dynamic import DynamicClient +from openshift.dynamic.exceptions import NotFoundError, ResourceNotFoundError, UnauthorizedError + +from .ocp import listInstances +from .olm import getSubscription + +logger = logging.getLogger(__name__) + + +def listAiServiceInstances(dynClient: DynamicClient) -> list: + """ + Get a list of AI Service instances on the cluster + """ + return listInstances(dynClient, "aiservice.ibm.com/v1", "AIServiceApp") + + +def verifyAiServiceInstance(dynClient: DynamicClient, instanceId: str) -> bool: + """ + Validate that the chosen AI Service instance exists + """ + try: + aiserviceAPI = dynClient.resources.get(api_version="aiservice.ibm.com/v1", kind="AIServiceApp") + aiserviceAPI.get(name=instanceId, namespace=f"aiservice-{instanceId}") + return True + except NotFoundError: + print("NOT FOUND") + return False + except ResourceNotFoundError: + # The AIServiceApp CRD has not even been installed in the cluster + print("RESOURCE NOT FOUND") + return False + except UnauthorizedError as e: + logger.error(f"Error: Unable to verify AI Service instance due to failed authorization: {e}") + return False + + +def getAiserviceChannel(dynClient: DynamicClient, instanceId: str) -> str: + """ + Get the AI Service channel from the subscription + """ + aiserviceSubscription = getSubscription(dynClient, f"aiservice-{instanceId}", "ibm-aiservice") + if aiserviceSubscription is None: + return None + else: + return aiserviceSubscription.spec.channel diff --git a/src/mas/devops/mas.py b/src/mas/devops/mas.py deleted file mode 100644 index 61b4bc23..00000000 --- a/src/mas/devops/mas.py +++ /dev/null @@ -1,432 +0,0 @@ -# ***************************************************************************** -# Copyright (c) 2024 IBM Corporation and other Contributors. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Eclipse Public License v1.0 -# which accompanies this distribution, and is available at -# http://www.eclipse.org/legal/epl-v10.html -# -# ***************************************************************************** - -import logging -import re -import yaml -from os import path -from time import sleep -from types import SimpleNamespace -from kubernetes.dynamic.resource import ResourceInstance -from openshift.dynamic import DynamicClient -from openshift.dynamic.exceptions import NotFoundError, ResourceNotFoundError, UnauthorizedError -from jinja2 import Environment, FileSystemLoader -import semver - -from .ocp import getStorageClasses -from .olm import getSubscription - -logger = logging.getLogger(__name__) - - -def isAirgapInstall(dynClient: DynamicClient, checkICSP: bool = False) -> bool: - if checkICSP: - try: - ICSPApi = dynClient.resources.get(api_version="operator.openshift.io/v1alpha1", kind="ImageContentSourcePolicy") - ICSPApi.get(name="ibm-mas-and-dependencies") - return True - except NotFoundError: - return False - else: - IDMSApi = dynClient.resources.get(api_version="config.openshift.io/v1", kind="ImageDigestMirrorSet") - masIDMS = IDMSApi.get(label_selector="mas.ibm.com/idmsContent=ibm") - aiserviceIDMS = IDMSApi.get(label_selector="aiservice.ibm.com/idmsContent=ibm") - return len(masIDMS.items) + len(aiserviceIDMS.items) > 0 - - -def getDefaultStorageClasses(dynClient: DynamicClient) -> dict: - result = SimpleNamespace( - provider=None, - providerName=None, - rwo=None, - rwx=None - ) - - # Iterate through storage classes until we find one that we recognize - # We make an assumption that if one of the paired classes if available, both will be - storageClasses = getStorageClasses(dynClient) - for storageClass in storageClasses: - if storageClass.metadata.name in ["ibmc-block-gold", "ibmc-file-gold-gid"]: - result.provider = "ibmc" - result.providerName = "IBMCloud ROKS" - result.rwo = "ibmc-block-gold" - result.rwx = "ibmc-file-gold-gid" - break - elif storageClass.metadata.name in ["ocs-storagecluster-ceph-rbd", "ocs-storagecluster-cephfs"]: - result.provider = "ocs" - result.providerName = "OpenShift Container Storage" - result.rwo = "ocs-storagecluster-ceph-rbd" - result.rwx = "ocs-storagecluster-cephfs" - break - elif storageClass.metadata.name in ["ocs-external-storagecluster-ceph-rbd", "ocs-external-storagecluster-cephfs"]: - result.provider = "ocs-external" - result.providerName = "OpenShift Container Storage (External)" - result.rwo = "ocs-external-storagecluster-ceph-rbd" - result.rwx = "ocs-external-storagecluster-cephfs" - break - elif storageClass.metadata.name == "longhorn": - result.provider = "longhorn" - result.providerName = "Longhorn" - result.rwo = "longhorn" - result.rwx = "longhorn" - break - elif storageClass.metadata.name == "nfs-client": - result.provider = "nfs" - result.providerName = "NFS Client" - result.rwo = "nfs-client" - result.rwx = "nfs-client" - break - elif storageClass.metadata.name in ["managed-premium", "azurefiles-premium"]: - result.provider = "azure" - result.providerName = "Azure Managed" - result.rwo = "managed-premium" - result.rwx = "azurefiles-premium" - break - elif storageClass.metadata.name in ["gp3-csi", "efs"]: - result.provider = "aws" - result.providerName = "AWS GP3" - result.rwo = "gp3-csi" - result.rwx = "efs" - break - logger.debug(f"Default storage class: {result}") - return result - - -def getCurrentCatalog(dynClient: DynamicClient) -> dict: - catalogsAPI = dynClient.resources.get(api_version="operators.coreos.com/v1alpha1", kind="CatalogSource") - try: - catalog = catalogsAPI.get(name="ibm-operator-catalog", namespace="openshift-marketplace") - catalogDisplayName = catalog.spec.displayName - catalogImage = catalog.spec.image - - m = re.match(r".+(?Pv[89]-(?P[0-9]+)-(amd64|s390x|ppc64le))", catalogDisplayName) - if m: - # catalogId = v9-yymmdd-amd64 - # catalogVersion = yymmdd - installedCatalogId = m.group("catalogId") - elif re.match(r".+v8-amd64", catalogDisplayName): - installedCatalogId = "v8-amd64" - else: - installedCatalogId = None - - return { - "displayName": catalogDisplayName, - "image": catalogImage, - "catalogId": installedCatalogId, - } - except NotFoundError: - return None - - -def listMasInstances(dynClient: DynamicClient) -> list: - """ - Get a list of MAS instances on the cluster - """ - return listInstances(dynClient, "core.mas.ibm.com/v1", "Suite") - - -def listAiServiceInstances(dynClient: DynamicClient) -> list: - """ - Get a list of AI Service instances on the cluster - """ - return listInstances(dynClient, "aiservice.ibm.com/v1", "AIServiceApp") - - -def listInstances(dynClient: DynamicClient, apiVersion: str, kind: str) -> list: - """ - Get a list of instances of a particular CR on the cluster - """ - api = dynClient.resources.get(api_version=apiVersion, kind=kind) - instances = api.get().to_dict()['items'] - if len(instances) > 0: - logger.info(f"There are {len(instances)} {kind} instances installed on this cluster:") - for instance in instances: - logger.info(f" * {instance['metadata']['name']} v{instance['status']['versions']['reconciled']}") - else: - logger.info(f"There are no {kind} instances installed on this cluster") - return instances - - -def getWorkspaceId(dynClient: DynamicClient, instanceId: str) -> str: - """ - Get the MAS workspace ID for namespace "mas-{instanceId}-core" - """ - workspaceId = None - workspacesAPI = dynClient.resources.get(api_version="core.mas.ibm.com/v1", kind="Workspace") - workspaces = workspacesAPI.get(namespace=f"mas-{instanceId}-core") - if len(workspaces["items"]) > 0: - workspaceId = workspaces["items"][0]["metadata"]["labels"]["mas.ibm.com/workspaceId"] - else: - logger.info("There are no MAS workspaces for the provided instanceId on this cluster") - return workspaceId - - -def verifyMasInstance(dynClient: DynamicClient, instanceId: str) -> bool: - """ - Validate that the chosen MAS instance exists - """ - try: - suitesAPI = dynClient.resources.get(api_version="core.mas.ibm.com/v1", kind="Suite") - suitesAPI.get(name=instanceId, namespace=f"mas-{instanceId}-core") - return True - except NotFoundError: - return False - except ResourceNotFoundError: - # The MAS Suite CRD has not even been installed in the cluster - return False - except UnauthorizedError as e: - logger.error(f"Error: Unable to verify MAS instance due to failed authorization: {e}") - return False - - -def verifyAiServiceInstance(dynClient: DynamicClient, instanceId: str) -> bool: - """ - Validate that the chosen AI Service instance exists - """ - try: - aiserviceAPI = dynClient.resources.get(api_version="aiservice.ibm.com/v1", kind="AIServiceApp") - aiserviceAPI.get(name=instanceId, namespace=f"aiservice-{instanceId}") - return True - except NotFoundError: - print("NOT FOUND") - return False - except ResourceNotFoundError: - # The AIServiceApp CRD has not even been installed in the cluster - print("RESOURCE NOT FOUND") - return False - except UnauthorizedError as e: - logger.error(f"Error: Unable to verify AI Service instance due to failed authorization: {e}") - return False - - -def verifyAppInstance(dynClient: DynamicClient, instanceId: str, applicationId: str) -> bool: - """ - Validate that the chosen app instance exists - """ - try: - # IoT has a different api version - operatorApiVersions = dict(iot="iot.ibm.com/v1") - apiVersion = operatorApiVersions[applicationId] if applicationId in operatorApiVersions else "apps.mas.ibm.com/v1" - operatorKinds = dict( - health="HealthApp", - predict="PredictApp", - monitor="MonitorApp", - iot="IoT", - visualinspection="VisualInspectionApp", - assist="AssistApp", - safety="SafetyApp", - manage="ManageApp", - hputilities="HPUtilitiesApp", - mso="MSOApp", - optimizer="OptimizerApp", - facilities="FacilitiesApp", - ) - appAPI = dynClient.resources.get(api_version=apiVersion, kind=operatorKinds[applicationId]) - appAPI.get(name=instanceId, namespace=f"mas-{instanceId}-{applicationId}") - return True - except NotFoundError: - return False - except ResourceNotFoundError: - # The MAS App CRD has not even been installed in the cluster - return False - except UnauthorizedError: - logger.error("Error: Unable to verify MAS app instance due to failed authorization: {e}") - return False - - -def getMasChannel(dynClient: DynamicClient, instanceId: str) -> str: - """ - Get the MAS channel from the subscription - """ - masSubscription = getSubscription(dynClient, f"mas-{instanceId}-core", "ibm-mas") - if masSubscription is None: - return None - else: - return masSubscription.spec.channel - - -def getAppsSubscriptionChannel(dynClient: DynamicClient, instanceId: str) -> list: - """ - Return list of installed apps with their subscribed channel - """ - try: - installedApps = [] - appKinds = [ - "assist", - "facilities", - "health", - "hputilities", - "iot", - "manage", - "monitor", - "mso", - "optimizer", - "safety", - "predict", - "visualinspection", - "aibroker" - ] - for appKind in appKinds: - appSubscription = getSubscription(dynClient, f"mas-{instanceId}-{appKind}", f"ibm-mas-{appKind}") - if appSubscription is not None: - installedApps.append({"appId": appKind, "channel": appSubscription.spec.channel}) - return installedApps - except NotFoundError: - return [] - except ResourceNotFoundError: - return [] - except UnauthorizedError: - logger.error("Error: Unable to get MAS app subscriptions due to failed authorization: {e}") - return [] - - -def getAiserviceChannel(dynClient: DynamicClient, instanceId: str) -> str: - """ - Get the AI Service channel from the subscription - """ - aiserviceSubscription = getSubscription(dynClient, f"aiservice-{instanceId}", "ibm-aiservice") - if aiserviceSubscription is None: - return None - else: - return aiserviceSubscription.spec.channel - - -def updateIBMEntitlementKey(dynClient: DynamicClient, namespace: str, icrUsername: str, icrPassword: str, artifactoryUsername: str = None, artifactoryPassword: str = None, secretName: str = "ibm-entitlement") -> ResourceInstance: - if secretName is None: - secretName = "ibm-entitlement" - if artifactoryUsername is not None: - logger.info(f"Updating IBM Entitlement ({secretName}) in namespace '{namespace}' (with Artifactory access)") - else: - logger.info(f"Updating IBM Entitlement ({secretName}) in namespace '{namespace}'") - - templateDir = path.join(path.abspath(path.dirname(__file__)), "templates") - env = Environment( - loader=FileSystemLoader(searchpath=templateDir), - extensions=["jinja2_base64_filters.Base64Filters"] - ) - - contentTemplate = env.get_template("ibm-entitlement-dockerconfig.json.j2") - dockerConfig = contentTemplate.render( - artifactory_username=artifactoryUsername, - artifactory_token=artifactoryPassword, - icr_username=icrUsername, - icr_password=icrPassword - ) - - template = env.get_template("ibm-entitlement-secret.yml.j2") - renderedTemplate = template.render( - name=secretName, - namespace=namespace, - docker_config=dockerConfig - ) - secret = yaml.safe_load(renderedTemplate) - secretsAPI = dynClient.resources.get(api_version="v1", kind="Secret") - - secret = secretsAPI.apply(body=secret, namespace=namespace) - return secret - - -def waitForPVC(dynClient: DynamicClient, namespace: str, pvcName: str) -> bool: - pvcAPI = dynClient.resources.get(api_version="v1", kind="PersistentVolumeClaim") - maxRetries = 60 - foundReadyPVC = False - retries = 0 - while not foundReadyPVC and retries < maxRetries: - retries += 1 - try: - pvc = pvcAPI.get(name=pvcName, namespace=namespace) - if pvc.status.phase == "Bound": - foundReadyPVC = True - else: - logger.debug(f"Waiting 5s for PVC {pvcName} to be ready before checking again ...") - sleep(5) - except NotFoundError: - logger.debug(f"Waiting 5s for PVC {pvcName} to be created before checking again ...") - sleep(5) - - return foundReadyPVC - - -def patchPendingPVC(dynClient: DynamicClient, namespace: str, pvcName: str, storageClassName: str = None) -> bool: - pvcAPI = dynClient.resources.get(api_version="v1", kind="PersistentVolumeClaim") - try: - pvc = pvcAPI.get(name=pvcName, namespace=namespace) - if pvc.status.phase == "Pending" and pvc.spec.storageClassName is None: - if storageClassName is not None and storageClassName(dynClient, name=storageClassName) is not None: - pvc.spec.storageClassName = storageClassName - else: - defaultStorageClasses = getDefaultStorageClasses(dynClient) - if defaultStorageClasses.provider is not None: - pvc.spec.storageClassName = defaultStorageClasses.rwo - else: - logger.error(f"Unable to set storageClassName in PVC {pvcName}.") - return False - - pvcAPI.patch(body=pvc, namespace=namespace) - - maxRetries = 60 - foundReadyPVC = False - retries = 0 - while not foundReadyPVC and retries < maxRetries: - retries += 1 - try: - patchedPVC = pvcAPI.get(name=pvcName, namespace=namespace) - if patchedPVC.status.phase == "Bound": - foundReadyPVC = True - else: - logger.debug(f"Waiting 5s for PVC {pvcName} to be bound before checking again ...") - sleep(5) - except NotFoundError: - logger.error(f"The patched PVC {pvcName} does not exist.") - return False - - return foundReadyPVC - - except NotFoundError: - logger.error(f"PVC {pvcName} does not exist") - return False - - -def isVersionBefore(_compare_to_version, _current_version): - """ - The method does a modified semantic version comparison, - as we want to treat any pre-release as == to the real release - but in strict semantic versioning it is < - ie. '8.6.0-pre.m1dev86' is converted to '8.6.0' - """ - if _current_version is None: - print("Version is not informed. Returning False") - return False - - strippedVersion = _current_version.split("-")[0] - if '.x' in strippedVersion: - strippedVersion = strippedVersion.replace('.x', '.0') - current_version = semver.VersionInfo.parse(strippedVersion) - compareToVersion = semver.VersionInfo.parse(_compare_to_version) - return current_version.compare(compareToVersion) < 0 - - -def isVersionEqualOrAfter(_compare_to_version, _current_version): - """ - The method does a modified semantic version comparison, - as we want to treat any pre-release as == to the real release - but in strict semantic versioning it is < - ie. '8.6.0-pre.m1dev86' is converted to '8.6.0' - """ - if _current_version is None: - print("Version is not informed. Returning False") - return False - - strippedVersion = _current_version.split("-")[0] - if '.x' in strippedVersion: - strippedVersion = strippedVersion.replace('.x', '.0') - current_version = semver.VersionInfo.parse(strippedVersion) - compareToVersion = semver.VersionInfo.parse(_compare_to_version) - return current_version.compare(compareToVersion) >= 0 diff --git a/src/mas/devops/mas/__init__.py b/src/mas/devops/mas/__init__.py new file mode 100644 index 00000000..dfbecf04 --- /dev/null +++ b/src/mas/devops/mas/__init__.py @@ -0,0 +1,16 @@ +from .apps import ( # noqa: F401 + verifyAppInstance, + getAppsSubscriptionChannel, + waitForAppReady +) + +from .suite import ( # noqa: F401 + isAirgapInstall, + getDefaultStorageClasses, + getCurrentCatalog, + listMasInstances, + getWorkspaceId, + verifyMasInstance, + getMasChannel, + updateIBMEntitlementKey, +) diff --git a/src/mas/devops/mas/apps.py b/src/mas/devops/mas/apps.py new file mode 100644 index 00000000..28ca355c --- /dev/null +++ b/src/mas/devops/mas/apps.py @@ -0,0 +1,194 @@ +# ***************************************************************************** +# Copyright (c) 2024 IBM Corporation and other Contributors. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +# +# ***************************************************************************** + +import logging +import json +from time import sleep +from openshift.dynamic import DynamicClient +from openshift.dynamic.exceptions import NotFoundError, ResourceNotFoundError, UnauthorizedError + +from ..olm import getSubscription + +logger = logging.getLogger(__name__) + +# IoT has a different api version +APP_API_VERSIONS = dict(iot="iot.ibm.com/v1") + +APP_IDS = [ + "assist", + "facilities", + "iot", + "manage", + "monitor", + "optimizer", + "predict", + "visualinspection" +] +APP_KINDS = dict( + predict="PredictApp", + monitor="MonitorApp", + iot="IoT", + visualinspection="VisualInspectionApp", + assist="AssistApp", + manage="ManageApp", + optimizer="OptimizerApp", + facilities="FacilitiesApp", +) +APPWS_KINDS = dict( + predict="PredictWorkspace", + monitor="MonitorWorkspace", + iot="IoTWorkspace", + visualinspection="VisualInspectionAppWorkspace", + assist="AssistWorkspace", + manage="ManageWorkspace", + optimizer="OptimizerWorkspace", + facilities="FacilitiesWorkspace", +) + + +def getAppResource(dynClient: DynamicClient, instanceId: str, applicationId: str, workspaceId: str = None) -> bool: + """ + Get the application or workspace Custom Resource + + :param dynClient: Description + :type dynClient: DynamicClient + :param instanceId: Description + :type instanceId: str + :param applicationId: Description + :type applicationId: str + :return: Description + :rtype: bool + :type workspaceId: str + :return: Description + :rtype: bool + """ + + apiVersion = APP_API_VERSIONS[applicationId] if applicationId in APP_API_VERSIONS else "apps.mas.ibm.com/v1" + kind = APP_KINDS[applicationId] if workspaceId is None else APPWS_KINDS[applicationId] + name = instanceId if workspaceId is None else f"{instanceId}-{workspaceId}" + namespace = f"mas-{instanceId}-{applicationId}" + + # logger.debug(f"Getting {kind}.{apiVersion} {name} from {namespace}") + + try: + appAPI = dynClient.resources.get(api_version=apiVersion, kind=kind) + resource = appAPI.get(name=name, namespace=namespace) + return resource + except NotFoundError: + return None + except ResourceNotFoundError: + # The CRD has not even been installed in the cluster + return None + except UnauthorizedError as e: + logger.error(f"Error: Unable to lookup {kind}.{apiVersion} due to authorization failure: {e}") + return None + + +def verifyAppInstance(dynClient: DynamicClient, instanceId: str, applicationId: str) -> bool: + """ + Validate that the chosen app instance exists + """ + return getAppResource(dynClient, instanceId, applicationId) is not None + + +def waitForAppReady( + dynClient: DynamicClient, + instanceId: str, + applicationId: str, + workspaceId: str = None, + retries: int = 100, + delay: int = 600, + debugLogFunction=logger.debug, + infoLogFunction=logger.info) -> bool: + """ + Docstring for waitForAppReady + + :param dynClient: Description + :type dynClient: DynamicClient + :param instanceId: Description + :type instanceId: str + :param applicationId: Description + :type applicationId: str + :param workspaceId: Description + :type workspaceId: str + :param retries: Description + :type retries: int + :param delay: Description + :type delay: int + :return: Description + :rtype: bool + """ + + resourceName = f"{APP_KINDS[applicationId]}/{instanceId}" + if workspaceId is not None: + resourceName = f"{APPWS_KINDS[applicationId]}/{instanceId}-{workspaceId}" + + appCR = None + appStatus = None + + attempt = 0 + infoLogFunction(f"Polling for {resourceName} to report ready state with {delay}s delay and {retries} retry limit") + + while attempt < retries: + attempt += 1 + appCR = getAppResource(dynClient, instanceId, applicationId, workspaceId) + + if appCR is None: + infoLogFunction(f"[{attempt}/{retries}] {resourceName} does not exist") + else: + appStatus = appCR.status + if appStatus is None: + infoLogFunction(f"[{attempt}/{retries}] {resourceName} has no status") + else: + if appStatus.conditions is None: + infoLogFunction(f"[{attempt}/{retries}] {resourceName} has no status conditions") + else: + foundReadyCondition: bool = False + for condition in appStatus.conditions: + if condition.type == "Ready": + foundReadyCondition = True + if condition.status == "True": + infoLogFunction(f"[{attempt}/{retries}] {resourceName} is in ready state: {condition.message}") + debugLogFunction(f"{resourceName} status={json.dumps(appStatus.to_dict())}") + return True + else: + infoLogFunction(f"[{attempt}/{retries}] {resourceName} is not in ready state: {condition.message}") + continue + if not foundReadyCondition: + infoLogFunction(f"[{attempt}/{retries}] {resourceName} has no ready status condition") + sleep(delay) + + # If we made it this far it means that the application was not ready in time + logger.warning(f"Retry limit reached polling for {resourceName} to report ready state") + if appStatus is None: + infoLogFunction(f"No {resourceName} status available") + else: + debugLogFunction(f"{resourceName} status={json.dumps(appStatus.to_dict())}") + return False + + +def getAppsSubscriptionChannel(dynClient: DynamicClient, instanceId: str) -> list: + """ + Return list of installed apps with their subscribed channel + """ + try: + installedApps = [] + for appId in APP_IDS: + appSubscription = getSubscription(dynClient, f"mas-{instanceId}-{appId}", f"ibm-mas-{appId}") + if appSubscription is not None: + installedApps.append({"appId": appId, "channel": appSubscription.spec.channel}) + return installedApps + except NotFoundError: + return [] + except ResourceNotFoundError: + return [] + except UnauthorizedError: + logger.error("Error: Unable to get MAS app subscriptions due to failed authorization: {e}") + return [] diff --git a/src/mas/devops/mas/suite.py b/src/mas/devops/mas/suite.py new file mode 100644 index 00000000..ab3fb727 --- /dev/null +++ b/src/mas/devops/mas/suite.py @@ -0,0 +1,208 @@ +# ***************************************************************************** +# Copyright (c) 2024 IBM Corporation and other Contributors. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +# +# ***************************************************************************** + +import logging +import re +import yaml +from os import path +from types import SimpleNamespace +from kubernetes.dynamic.resource import ResourceInstance +from openshift.dynamic import DynamicClient +from openshift.dynamic.exceptions import NotFoundError, ResourceNotFoundError, UnauthorizedError +from jinja2 import Environment, FileSystemLoader + +from ..ocp import getStorageClasses, listInstances +from ..olm import getSubscription + +logger = logging.getLogger(__name__) + + +def isAirgapInstall(dynClient: DynamicClient, checkICSP: bool = False) -> bool: + if checkICSP: + try: + ICSPApi = dynClient.resources.get(api_version="operator.openshift.io/v1alpha1", kind="ImageContentSourcePolicy") + ICSPApi.get(name="ibm-mas-and-dependencies") + return True + except NotFoundError: + return False + else: + IDMSApi = dynClient.resources.get(api_version="config.openshift.io/v1", kind="ImageDigestMirrorSet") + masIDMS = IDMSApi.get(label_selector="mas.ibm.com/idmsContent=ibm") + aiserviceIDMS = IDMSApi.get(label_selector="aiservice.ibm.com/idmsContent=ibm") + return len(masIDMS.items) + len(aiserviceIDMS.items) > 0 + + +def getDefaultStorageClasses(dynClient: DynamicClient) -> dict: + result = SimpleNamespace( + provider=None, + providerName=None, + rwo=None, + rwx=None + ) + + # Iterate through storage classes until we find one that we recognize + # We make an assumption that if one of the paired classes if available, both will be + storageClasses = getStorageClasses(dynClient) + for storageClass in storageClasses: + if storageClass.metadata.name in ["ibmc-block-gold", "ibmc-file-gold-gid"]: + result.provider = "ibmc" + result.providerName = "IBMCloud ROKS" + result.rwo = "ibmc-block-gold" + result.rwx = "ibmc-file-gold-gid" + break + elif storageClass.metadata.name in ["ocs-storagecluster-ceph-rbd", "ocs-storagecluster-cephfs"]: + result.provider = "ocs" + result.providerName = "OpenShift Container Storage" + result.rwo = "ocs-storagecluster-ceph-rbd" + result.rwx = "ocs-storagecluster-cephfs" + break + elif storageClass.metadata.name in ["ocs-external-storagecluster-ceph-rbd", "ocs-external-storagecluster-cephfs"]: + result.provider = "ocs-external" + result.providerName = "OpenShift Container Storage (External)" + result.rwo = "ocs-external-storagecluster-ceph-rbd" + result.rwx = "ocs-external-storagecluster-cephfs" + break + elif storageClass.metadata.name == "longhorn": + result.provider = "longhorn" + result.providerName = "Longhorn" + result.rwo = "longhorn" + result.rwx = "longhorn" + break + elif storageClass.metadata.name == "nfs-client": + result.provider = "nfs" + result.providerName = "NFS Client" + result.rwo = "nfs-client" + result.rwx = "nfs-client" + break + elif storageClass.metadata.name in ["managed-premium", "azurefiles-premium"]: + result.provider = "azure" + result.providerName = "Azure Managed" + result.rwo = "managed-premium" + result.rwx = "azurefiles-premium" + break + elif storageClass.metadata.name in ["gp3-csi", "efs"]: + result.provider = "aws" + result.providerName = "AWS GP3" + result.rwo = "gp3-csi" + result.rwx = "efs" + break + logger.debug(f"Default storage class: {result}") + return result + + +def getCurrentCatalog(dynClient: DynamicClient) -> dict: + catalogsAPI = dynClient.resources.get(api_version="operators.coreos.com/v1alpha1", kind="CatalogSource") + try: + catalog = catalogsAPI.get(name="ibm-operator-catalog", namespace="openshift-marketplace") + catalogDisplayName = catalog.spec.displayName + catalogImage = catalog.spec.image + + m = re.match(r".+(?Pv[89]-(?P[0-9]+)-(amd64|s390x|ppc64le))", catalogDisplayName) + if m: + # catalogId = v9-yymmdd-amd64 + # catalogVersion = yymmdd + installedCatalogId = m.group("catalogId") + elif re.match(r".+v8-amd64", catalogDisplayName): + installedCatalogId = "v8-amd64" + else: + installedCatalogId = None + + return { + "displayName": catalogDisplayName, + "image": catalogImage, + "catalogId": installedCatalogId, + } + except NotFoundError: + return None + + +def listMasInstances(dynClient: DynamicClient) -> list: + """ + Get a list of MAS instances on the cluster + """ + return listInstances(dynClient, "core.mas.ibm.com/v1", "Suite") + + +def getWorkspaceId(dynClient: DynamicClient, instanceId: str) -> str: + """ + Get the MAS workspace ID for namespace "mas-{instanceId}-core" + """ + workspaceId = None + workspacesAPI = dynClient.resources.get(api_version="core.mas.ibm.com/v1", kind="Workspace") + workspaces = workspacesAPI.get(namespace=f"mas-{instanceId}-core") + if len(workspaces["items"]) > 0: + workspaceId = workspaces["items"][0]["metadata"]["labels"]["mas.ibm.com/workspaceId"] + else: + logger.info("There are no MAS workspaces for the provided instanceId on this cluster") + return workspaceId + + +def verifyMasInstance(dynClient: DynamicClient, instanceId: str) -> bool: + """ + Validate that the chosen MAS instance exists + """ + try: + suitesAPI = dynClient.resources.get(api_version="core.mas.ibm.com/v1", kind="Suite") + suitesAPI.get(name=instanceId, namespace=f"mas-{instanceId}-core") + return True + except NotFoundError: + return False + except ResourceNotFoundError: + # The MAS Suite CRD has not even been installed in the cluster + return False + except UnauthorizedError as e: + logger.error(f"Error: Unable to verify MAS instance due to failed authorization: {e}") + return False + + +def getMasChannel(dynClient: DynamicClient, instanceId: str) -> str: + """ + Get the MAS channel from the subscription + """ + masSubscription = getSubscription(dynClient, f"mas-{instanceId}-core", "ibm-mas") + if masSubscription is None: + return None + else: + return masSubscription.spec.channel + + +def updateIBMEntitlementKey(dynClient: DynamicClient, namespace: str, icrUsername: str, icrPassword: str, artifactoryUsername: str = None, artifactoryPassword: str = None, secretName: str = "ibm-entitlement") -> ResourceInstance: + if secretName is None: + secretName = "ibm-entitlement" + if artifactoryUsername is not None: + logger.info(f"Updating IBM Entitlement ({secretName}) in namespace '{namespace}' (with Artifactory access)") + else: + logger.info(f"Updating IBM Entitlement ({secretName}) in namespace '{namespace}'") + + templateDir = path.join(path.abspath(path.dirname(__file__)), "..", "templates") + env = Environment( + loader=FileSystemLoader(searchpath=templateDir), + extensions=["jinja2_base64_filters.Base64Filters"] + ) + + contentTemplate = env.get_template("ibm-entitlement-dockerconfig.json.j2") + dockerConfig = contentTemplate.render( + artifactory_username=artifactoryUsername, + artifactory_token=artifactoryPassword, + icr_username=icrUsername, + icr_password=icrPassword + ) + + template = env.get_template("ibm-entitlement-secret.yml.j2") + renderedTemplate = template.render( + name=secretName, + namespace=namespace, + docker_config=dockerConfig + ) + secret = yaml.safe_load(renderedTemplate) + secretsAPI = dynClient.resources.get(api_version="v1", kind="Secret") + + secret = secretsAPI.apply(body=secret, namespace=namespace) + return secret diff --git a/src/mas/devops/ocp.py b/src/mas/devops/ocp.py index b5988010..4219c5cc 100644 --- a/src/mas/devops/ocp.py +++ b/src/mas/devops/ocp.py @@ -243,6 +243,42 @@ def crdExists(dynClient: DynamicClient, crdName: str) -> bool: return False +def listInstances(dynClient: DynamicClient, apiVersion: str, kind: str) -> list: + """ + Get a list of instances of a particular CR on the cluster + """ + api = dynClient.resources.get(api_version=apiVersion, kind=kind) + instances = api.get().to_dict()['items'] + if len(instances) > 0: + logger.info(f"There are {len(instances)} {kind} instances installed on this cluster:") + for instance in instances: + logger.info(f" * {instance['metadata']['name']} v{instance['status']['versions']['reconciled']}") + else: + logger.info(f"There are no {kind} instances installed on this cluster") + return instances + + +def waitForPVC(dynClient: DynamicClient, namespace: str, pvcName: str) -> bool: + pvcAPI = dynClient.resources.get(api_version="v1", kind="PersistentVolumeClaim") + maxRetries = 60 + foundReadyPVC = False + retries = 0 + while not foundReadyPVC and retries < maxRetries: + retries += 1 + try: + pvc = pvcAPI.get(name=pvcName, namespace=namespace) + if pvc.status.phase == "Bound": + foundReadyPVC = True + else: + logger.debug(f"Waiting 5s for PVC {pvcName} to be ready before checking again ...") + sleep(5) + except NotFoundError: + logger.debug(f"Waiting 5s for PVC {pvcName} to be created before checking again ...") + sleep(5) + + return foundReadyPVC + + # Assisted by WCA@IBM # Latest GenAI contribution: ibm/granite-8b-code-instruct def execInPod(core_v1_api: client.CoreV1Api, pod_name: str, namespace, command: list, timeout: int = 60) -> str: diff --git a/src/mas/devops/tekton.py b/src/mas/devops/tekton.py index e408e141..6167bb34 100644 --- a/src/mas/devops/tekton.py +++ b/src/mas/devops/tekton.py @@ -22,14 +22,11 @@ from jinja2 import Environment, FileSystemLoader -from .ocp import getConsoleURL, waitForCRD, waitForDeployment, crdExists -from .mas import waitForPVC, patchPendingPVC +from .ocp import getConsoleURL, waitForCRD, waitForDeployment, crdExists, waitForPVC logger = logging.getLogger(__name__) -# customStorageClassName is used when no default Storageclass is available on cluster, -# openshift-pipelines creates PVC which looks for default. customStorageClassName is patched into PVC when default is unavailable. def installOpenShiftPipelines(dynClient: DynamicClient, customStorageClassName: str = None) -> bool: """ Install the OpenShift Pipelines Operator and wait for it to be ready to use @@ -86,6 +83,8 @@ def installOpenShiftPipelines(dynClient: DynamicClient, customStorageClassName: logger.error("OpenShift Pipelines Webhook is NOT installed and ready") return False + # Workaround for bug in OpenShift Pipelines/Tekton + # ------------------------------------------------------------------------- # Wait for the postgredb-tekton-results-postgres-0 PVC to be ready # this PVC doesn't come up when there's no default storage class is in the cluster, # this is causing the pvc to be in pending state and causing the tekton-results-postgres statefulSet in pending, @@ -98,8 +97,13 @@ def installOpenShiftPipelines(dynClient: DynamicClient, customStorageClassName: logger.info("OpenShift Pipelines postgres is installed and ready") return True else: - patchedPVC = patchPendingPVC(dynClient, namespace="openshift-pipelines", pvcName="postgredb-tekton-results-postgres-0", storageClassName=customStorageClassName) - if patchedPVC: + tektonPVCisReady = addMissingStorageClassToTektonPVC( + dynClient=dynClient, + namespace="openshift-pipelines", + pvcName="postgredb-tekton-results-postgres-0", + storageClassName=customStorageClassName + ) + if tektonPVCisReady: logger.info("OpenShift Pipelines postgres is installed and ready") return True else: @@ -107,6 +111,53 @@ def installOpenShiftPipelines(dynClient: DynamicClient, customStorageClassName: return False +def addMissingStorageClassToTektonPVC(dynClient: DynamicClient, namespace: str, pvcName: str, storageClassName: str) -> bool: + """ + OpenShift Pipelines has a problem when there is no default storage class defined in a cluster, this function + patches the PVC used to store pipeline results to add a specific storage class into the PVC spec and waits for the + PVC to be bound. + + :param dynClient: Kubernetes client, required to work with PVC + :type dynClient: DynamicClient + :param namespace: Namespace where OpenShift Pipelines is installed + :type namespace: str + :param pvcName: Name of the PVC that we want to fix + :type pvcName: str + :param storageClassName: Name of the storage class that we want to update the PVC to reference + :type storageClassName: str + :return: Description + :rtype: bool + """ + pvcAPI = dynClient.resources.get(api_version="v1", kind="PersistentVolumeClaim") + try: + pvc = pvcAPI.get(name=pvcName, namespace=namespace) + if pvc.status.phase == "Pending" and pvc.spec.storageClassName is None: + pvc.spec.storageClassName = storageClassName + pvcAPI.patch(body=pvc, namespace=namespace) + + maxRetries = 60 + foundReadyPVC = False + retries = 0 + while not foundReadyPVC and retries < maxRetries: + retries += 1 + try: + patchedPVC = pvcAPI.get(name=pvcName, namespace=namespace) + if patchedPVC.status.phase == "Bound": + foundReadyPVC = True + else: + logger.debug(f"Waiting 5s for PVC {pvcName} to be bound before checking again ...") + sleep(5) + except NotFoundError: + logger.error(f"The patched PVC {pvcName} does not exist.") + return False + + return foundReadyPVC + + except NotFoundError: + logger.error(f"PVC {pvcName} does not exist") + return False + + def updateTektonDefinitions(namespace: str, yamlFile: str) -> None: """ Install/update the MAS tekton pipeline and task definitions diff --git a/src/mas/devops/utils.py b/src/mas/devops/utils.py new file mode 100644 index 00000000..6b75f112 --- /dev/null +++ b/src/mas/devops/utils.py @@ -0,0 +1,39 @@ +import semver + + +def isVersionBefore(_compare_to_version, _current_version): + """ + The method does a modified semantic version comparison, + as we want to treat any pre-release as == to the real release + but in strict semantic versioning it is < + ie. '8.6.0-pre.m1dev86' is converted to '8.6.0' + """ + if _current_version is None: + print("Version is not informed. Returning False") + return False + + strippedVersion = _current_version.split("-")[0] + if '.x' in strippedVersion: + strippedVersion = strippedVersion.replace('.x', '.0') + current_version = semver.VersionInfo.parse(strippedVersion) + compareToVersion = semver.VersionInfo.parse(_compare_to_version) + return current_version.compare(compareToVersion) < 0 + + +def isVersionEqualOrAfter(_compare_to_version, _current_version): + """ + The method does a modified semantic version comparison, + as we want to treat any pre-release as == to the real release + but in strict semantic versioning it is < + ie. '8.6.0-pre.m1dev86' is converted to '8.6.0' + """ + if _current_version is None: + print("Version is not informed. Returning False") + return False + + strippedVersion = _current_version.split("-")[0] + if '.x' in strippedVersion: + strippedVersion = strippedVersion.replace('.x', '.0') + current_version = semver.VersionInfo.parse(strippedVersion) + compareToVersion = semver.VersionInfo.parse(_compare_to_version) + return current_version.compare(compareToVersion) >= 0 diff --git a/test/src/test_mas.py b/test/src/test_mas.py index 79db0ac6..10583926 100644 --- a/test/src/test_mas.py +++ b/test/src/test_mas.py @@ -64,15 +64,6 @@ def test_is_airgap_install(): assert mas.isAirgapInstall(dynClient, checkICSP=False) is False -def test_version_before(): - assert mas.isVersionBefore('9.1.0', '9.1.x-feature') is False - assert mas.isVersionBefore('9.1.0', '9.0.0') is True - assert mas.isVersionBefore('8.11.1', '9.1.0') is False - assert mas.isVersionBefore('9.1.0', '9.1.x-stable') is False - - -def test_version_equal_of_after(): - assert mas.isVersionEqualOrAfter('9.1.0', '9.2.x-feature') is True - assert mas.isVersionEqualOrAfter('9.1.0', '9.0.0') is False - assert mas.isVersionEqualOrAfter('8.11.1', '9.1.0') is True - assert mas.isVersionEqualOrAfter('9.2.0', '9.1.x-stable') is False +# def test_is_app_ready(): +# mas.waitForAppReady(dynClient, "fvtcpd", "iot") +# mas.waitForAppReady(dynClient, "fvtcpd", "iot", "masdev") diff --git a/test/src/test_utils.py b/test/src/test_utils.py new file mode 100644 index 00000000..d4ca5bdc --- /dev/null +++ b/test/src/test_utils.py @@ -0,0 +1,25 @@ +# ***************************************************************************** +# Copyright (c) 2024 IBM Corporation and other Contributors. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +# +# ***************************************************************************** + +from mas.devops import utils + + +def test_version_before(): + assert utils.isVersionBefore('9.1.0', '9.1.x-feature') is False + assert utils.isVersionBefore('9.1.0', '9.0.0') is True + assert utils.isVersionBefore('8.11.1', '9.1.0') is False + assert utils.isVersionBefore('9.1.0', '9.1.x-stable') is False + + +def test_version_equal_of_after(): + assert utils.isVersionEqualOrAfter('9.1.0', '9.2.x-feature') is True + assert utils.isVersionEqualOrAfter('9.1.0', '9.0.0') is False + assert utils.isVersionEqualOrAfter('8.11.1', '9.1.0') is True + assert utils.isVersionEqualOrAfter('9.2.0', '9.1.x-stable') is False