From 344eeb6b5f5edd8b404969313279650919573b5c Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Thu, 30 May 2024 14:41:07 -0600 Subject: [PATCH 1/9] Add GitHub Actions workflow Signed-off-by: Zack Cerza --- .github/workflows/ci.yml | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..8615d0af --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,38 @@ +name: CI +on: + pull_request: + branches: + - main + workflow_dispatch: + +jobs: + test: + name: CI on python${{ matrix.python }} via ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + include: + - os: ubuntu-24.04 + python: "3.12" + steps: + - uses: actions/checkout@v2 + - name: Install packages + run: sudo apt install podman golang-github-containernetworking-plugin-dnsname + - name: Create virtualenv + run: python3 -m venv venv + - name: Install + run: ./venv/bin/pip3 install -e . + - name: Doctor + run: ./venv/bin/ceph-devstack -v doctor + - name: Build + run: ./venv/bin/ceph-devstack -v build + - name: Create + run: ./venv/bin/ceph-devstack -v create + - name: Start + run: ./venv/bin/ceph-devstack -v start + - name: Wait + run: podman logs -f teuthology + - name: Stop + run: ./venv/bin/ceph-devstack -v stop + - name: Remove + run: ./venv/bin/ceph-devstack -v remove From f03d09d52d9325047cd1b75d911633dcf8d1706e Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Thu, 20 Jun 2024 16:22:52 -0600 Subject: [PATCH 2/9] Add wait command This is much like `podman wait`, and uses it under the hood, but will also exit with the same code as the container did. Signed-off-by: Zack Cerza --- ceph_devstack/__init__.py | 8 ++++++++ ceph_devstack/cli.py | 8 ++++++-- ceph_devstack/resources/ceph/__init__.py | 9 +++++++++ ceph_devstack/resources/container.py | 9 +++++++++ 4 files changed, 32 insertions(+), 2 deletions(-) diff --git a/ceph_devstack/__init__.py b/ceph_devstack/__init__.py index e5d822b8..c2c88d66 100644 --- a/ceph_devstack/__init__.py +++ b/ceph_devstack/__init__.py @@ -96,6 +96,14 @@ def parse_args(args: List[str]) -> argparse.Namespace: subparsers.add_parser( "watch", help="Monitor the cluster, recreating containers as necessary" ) + parser_wait = subparsers.add_parser( + "wait", + help="Wait for the specified container to exit. Exit with its exit code.", + ) + parser_wait.add_argument( + "container", + help="The container to wait for", + ) subparsers.add_parser("show-conf", help="show the configuration") return parser.parse_args(args) diff --git a/ceph_devstack/cli.py b/ceph_devstack/cli.py index d73b7617..b3116f94 100644 --- a/ceph_devstack/cli.py +++ b/ceph_devstack/cli.py @@ -34,9 +34,13 @@ async def run(): sys.exit(1) if args.command == "doctor": return - await obj.apply(args.command) + elif args.command == "wait": + return await obj.wait(container_name=args.container) + else: + await obj.apply(args.command) + return 0 try: - asyncio.run(run()) + sys.exit(asyncio.run(run())) except KeyboardInterrupt: logger.debug("Exiting!") diff --git a/ceph_devstack/resources/ceph/__init__.py b/ceph_devstack/resources/ceph/__init__.py index dbc59b70..95a8754e 100644 --- a/ceph_devstack/resources/ceph/__init__.py +++ b/ceph_devstack/resources/ceph/__init__.py @@ -220,3 +220,12 @@ async def watch(self): await container.start() except KeyboardInterrupt: break + + async def wait(self, container_name: str): + for kind in (await self.get_containers()).keys(): + for name in await self.get_container_names(kind): + container = kind(name=name) + if container.name == container_name: + return await container.wait() + logger.error(f"Could not find container {container_name}") + return 1 diff --git a/ceph_devstack/resources/container.py b/ceph_devstack/resources/container.py index 1ce90222..c0c6d9ef 100644 --- a/ceph_devstack/resources/container.py +++ b/ceph_devstack/resources/container.py @@ -25,6 +25,7 @@ class Container(PodmanResource): stop_cmd: List[str] = ["podman", "container", "stop", "{name}"] exists_cmd: List[str] = ["podman", "container", "inspect", "{name}"] pull_cmd: List[str] = ["podman", "pull", "{image}"] + wait_cmd: List[str] = ["podman", "wait", "{name}"] env_vars: Dict[str, Optional[str]] = {} def __init__(self, name: str = ""): @@ -153,3 +154,11 @@ async def is_running(self): if not result: return False return result[0]["State"]["Status"].lower() == "running" + + async def wait(self) -> Optional[int]: + proc = await self.cmd(self.format_cmd(self.wait_cmd)) + out, err = await proc.communicate() + if proc.returncode: + logger.error(f"Could not wait for {self.name}: {err.decode().strip()}") + return proc.returncode + return int(out.decode().strip()) From 3a53089798e440ae47acd0ae7976acac4b7f0d3c Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Fri, 21 Jun 2024 16:09:26 -0600 Subject: [PATCH 3/9] ci: Use new wait command Signed-off-by: Zack Cerza --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8615d0af..df1171c7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,6 +31,8 @@ jobs: - name: Start run: ./venv/bin/ceph-devstack -v start - name: Wait + run: ./venv/bin/ceph-devstack wait teuthology + - name: Dump logs run: podman logs -f teuthology - name: Stop run: ./venv/bin/ceph-devstack -v stop From 673431e82a5304c3bf0176e7d5aca72e1af1b892 Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Fri, 21 Jun 2024 16:19:25 -0600 Subject: [PATCH 4/9] ci: Use doctor --fix Signed-off-by: Zack Cerza --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index df1171c7..95902659 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,7 +23,7 @@ jobs: - name: Install run: ./venv/bin/pip3 install -e . - name: Doctor - run: ./venv/bin/ceph-devstack -v doctor + run: ./venv/bin/ceph-devstack -v doctor --fix - name: Build run: ./venv/bin/ceph-devstack -v build - name: Create From b51a70f86fea80406e2bc3d7c160ac05b6600a88 Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Fri, 21 Jun 2024 16:31:21 -0600 Subject: [PATCH 5/9] ci: Always dump logs Signed-off-by: Zack Cerza --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 95902659..5914f127 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,6 +33,7 @@ jobs: - name: Wait run: ./venv/bin/ceph-devstack wait teuthology - name: Dump logs + if: success() || failure() run: podman logs -f teuthology - name: Stop run: ./venv/bin/ceph-devstack -v stop From d62268d87fc3310598b36cc76427e37a4c1c8236 Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Mon, 6 Jan 2025 11:01:04 -0700 Subject: [PATCH 6/9] ci: Set owner on /dev/loop-control Signed-off-by: Zack Cerza --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5914f127..26d2af05 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,6 +22,8 @@ jobs: run: python3 -m venv venv - name: Install run: ./venv/bin/pip3 install -e . + - name: Set owner for /dev/loop-control + run: sudo chown $(whoami) /dev/loop-control - name: Doctor run: ./venv/bin/ceph-devstack -v doctor --fix - name: Build From e708654d414c732124237778358977ff5fb8f25f Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Mon, 6 Jan 2025 11:10:41 -0700 Subject: [PATCH 7/9] requirements: Pass -y to dnf install commands doctor --fix should not be interactive. Signed-off-by: Zack Cerza --- ceph_devstack/requirements.py | 4 ++-- ceph_devstack/resources/ceph/requirements.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ceph_devstack/requirements.py b/ceph_devstack/requirements.py index 9332515f..bad5dd32 100644 --- a/ceph_devstack/requirements.py +++ b/ceph_devstack/requirements.py @@ -194,13 +194,13 @@ class PodmanDNSPlugin(FixableRequirement): dns_plugin_path = "/usr/libexec/cni/dnsname" check_cmd = ["test", "-x", dns_plugin_path] suggest_msg = "Could not find the podman DNS plugin" - fix_cmd = ["sudo", "dnf", "install", dns_plugin_path] + fix_cmd = ["sudo", "dnf", "install", "-y", dns_plugin_path] class FuseOverlayfsPresence(FixableRequirement): check_cmd = ["command", "-v", "fuse-overlayfs"] suggest_msg = "Could not find fuse-overlayfs" - fix_cmd = ["sudo", "dnf", "install", "fuse-overlayfs"] + fix_cmd = ["sudo", "dnf", "install", "-y", "fuse-overlayfs"] async def check_requirements(): diff --git a/ceph_devstack/resources/ceph/requirements.py b/ceph_devstack/resources/ceph/requirements.py index 2ae3c373..00d79678 100644 --- a/ceph_devstack/resources/ceph/requirements.py +++ b/ceph_devstack/resources/ceph/requirements.py @@ -53,6 +53,7 @@ def __init__(self): "(sudo", "dnf", "install", + "-y", "policycoreutils-devel", "selinux-policy-devel", "&&", From dfac66a65319553609574701795588e454754484 Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Mon, 6 Jan 2025 11:44:48 -0700 Subject: [PATCH 8/9] Host: Add os_type() This returns the ID field from /etc/os-release. Signed-off-by: Zack Cerza --- ceph_devstack/host.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ceph_devstack/host.py b/ceph_devstack/host.py index 58ad4b2d..cf1cc641 100644 --- a/ceph_devstack/host.py +++ b/ceph_devstack/host.py @@ -71,6 +71,14 @@ def kernel_version(self) -> Version: self._kernel_version = parse_version(raw_version.split("-")[0]) return self._kernel_version + def os_type(self) -> str: + if not hasattr(self, "_os_type"): + proc = self.run(["bash", "-c", ". /etc/os-release && echo $ID"]) + assert proc.stdout is not None + assert proc.wait() == 0, "is /etc/os-release missing?" + self._os_type = proc.stdout.read().decode().strip().lower() + return self._os_type + async def podman_info(self, force: bool = False) -> Dict: if force or not hasattr(self, "_podman_info"): proc = await self.arun(["podman", "info"]) From 626b28ca24613706bcbd2845dadb0fddacc455f8 Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Mon, 6 Jan 2025 11:45:54 -0700 Subject: [PATCH 9/9] PodmanDNSPlugin: Support Ubuntu, Debian Signed-off-by: Zack Cerza --- ceph_devstack/requirements.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/ceph_devstack/requirements.py b/ceph_devstack/requirements.py index bad5dd32..0fb01193 100644 --- a/ceph_devstack/requirements.py +++ b/ceph_devstack/requirements.py @@ -191,10 +191,24 @@ async def check(self): class PodmanDNSPlugin(FixableRequirement): - dns_plugin_path = "/usr/libexec/cni/dnsname" - check_cmd = ["test", "-x", dns_plugin_path] suggest_msg = "Could not find the podman DNS plugin" - fix_cmd = ["sudo", "dnf", "install", "-y", dns_plugin_path] + + def __init__(self): + os_type = self.host.os_type() + if os_type == "centos": + dns_plugin_path = "/usr/libexec/cni/dnsname" + self.check_cmd = ["test", "-x", dns_plugin_path] + self.fix_cmd = ["sudo", "dnf", "install", "-y", dns_plugin_path] + elif os_type in ["ubuntu", "debian"]: + dns_plugin_path = "/usr/lib/cni/dnsname" + self.check_cmd = ["test", "-x", dns_plugin_path] + self.fix_cmd = [ + "sudo", + "apt", + "install", + "-y", + "golang-github-containernetworking-plugin-dnsname", + ] class FuseOverlayfsPresence(FixableRequirement):