diff --git a/.github/workflows/test-examples.yml b/.github/workflows/test-examples.yml index d372ee3..e17b231 100644 --- a/.github/workflows/test-examples.yml +++ b/.github/workflows/test-examples.yml @@ -223,6 +223,9 @@ jobs: - example: networking-py args: "--net -- /urllib_get.py" expect: "SUCCESS: urllib GET worked!" + - example: networking-py + args: "--net -- /urllib_get_no_timeout.py" + expect: "SUCCESS: urllib GET \\(no timeout\\) worked!" - example: networking-py args: "--port 8080 -- /echo_server_test.py" expect: "SUCCESS: bind\\+listen on port 8080 allowed" @@ -613,6 +616,9 @@ jobs: - example: networking-py args: "--net -- /urllib_get.py" expect: "SUCCESS: urllib GET worked!" + - example: networking-py + args: "--net -- /urllib_get_no_timeout.py" + expect: "SUCCESS: urllib GET \\(no timeout\\) worked!" - example: networking-py args: "--port 8080 -- /echo_server_test.py" expect: "SUCCESS: bind\\+listen on port 8080 allowed" @@ -753,16 +759,33 @@ jobs: } } + # Helper: run a command with a per-test timeout (seconds). + function Invoke-WithTimeout { + param([int]$Seconds, [string]$Exe, [string[]]$ArgList) + $outFile = Join-Path $env:RUNNER_TEMP 'hl-test-out.log' + $proc = Start-Process -FilePath $Exe -ArgumentList $ArgList ` + -NoNewWindow -RedirectStandardOutput $outFile -RedirectStandardError "$outFile.err" -PassThru + if (-not $proc.WaitForExit($Seconds * 1000)) { + Stop-Process -Id $proc.Id -Force -ErrorAction SilentlyContinue + Write-Host "FAIL: timed out after ${Seconds}s" + Get-Content $outFile -ErrorAction SilentlyContinue + Get-Content "$outFile.err" -ErrorAction SilentlyContinue + exit 1 + } + $stdoutContent = Get-Content $outFile -Raw -ErrorAction SilentlyContinue + $stderrContent = Get-Content "$outFile.err" -Raw -ErrorAction SilentlyContinue + $script:out = "$stdoutContent$stderrContent".Trim() + $script:rc = $proc.ExitCode + } + switch ($driver) { 'multifn-test' { - $out = & multifn-test $kernel $cpio 2>&1 - $rc = $LASTEXITCODE + Invoke-WithTimeout -Seconds 60 -Exe 'multifn-test' -ArgList @($kernel, $cpio) } 'pydriver-run' { "print('hello from driver')" | Out-File -Encoding ascii tiny.py $tiny = (Resolve-Path "tiny.py").Path - $out = & pydriver-run $kernel $cpio $tiny 2>&1 - $rc = $LASTEXITCODE + Invoke-WithTimeout -Seconds 120 -Exe 'pydriver-run' -ArgList @($kernel, $cpio, $tiny) } default { $argList = @() @@ -773,16 +796,15 @@ jobs: if ($memory -ne '') { $memArgs = @('-m', $memory) } - $out = & hyperlight-unikraft -q @memArgs ` - $kernel --initrd $cpio @mountArgs @toolArgs @argList 2>&1 - $rc = $LASTEXITCODE + Invoke-WithTimeout -Seconds 120 -Exe 'hyperlight-unikraft' ` + -ArgList (@('-q') + $memArgs + @($kernel, '--initrd', $cpio) + $mountArgs + $toolArgs + $argList) } } - $outStr = ($out | Out-String) + if ($null -eq $out) { $out = '' } Write-Host "=== output (exit=$rc) ===" - Write-Host $outStr - if (-not ($outStr -match $expect)) { + Write-Host $out + if (-not ($out -match $expect)) { Write-Host "FAIL: did not match /$expect/" exit 1 } @@ -946,6 +968,32 @@ jobs: exit 1 } + - name: pyhl run (busybox subprocess demo) + if: runner.os == 'Windows' || steps.kvm_check.outputs.available == 'true' + shell: pwsh + run: | + $out = pyhl run examples/python-agent-driver/demo_busybox.py 2>&1 + Write-Host $out + if ($out -match 'hello from hyperlight guest') { + Write-Host "PASS: busybox subprocess demo" + } else { + Write-Error "FAIL: did not match /hello from hyperlight guest/" + exit 1 + } + + - name: pyhl run (pip install subprocess demo) + if: runner.os == 'Windows' || steps.kvm_check.outputs.available == 'true' + shell: pwsh + run: | + $out = pyhl run --net examples/python-agent-driver/demo_pip_install.py 2>&1 + Write-Host $out + if ($out -match 'Installed and imported six') { + Write-Host "PASS: pip install subprocess demo" + } else { + Write-Error "FAIL: did not match /Installed and imported six/" + exit 1 + } + test-examples-passed: if: always() needs: diff --git a/examples/networking-py/Dockerfile b/examples/networking-py/Dockerfile index c16ba5b..b550171 100644 --- a/examples/networking-py/Dockerfile +++ b/examples/networking-py/Dockerfile @@ -8,6 +8,7 @@ FROM ${BASE} AS rootfs COPY http_get.py /http_get.py COPY echo_server.py /echo_server.py COPY urllib_get.py /urllib_get.py +COPY urllib_get_no_timeout.py /urllib_get_no_timeout.py COPY https_test.py /https_test.py COPY echo_server_test.py /echo_server_test.py diff --git a/examples/networking-py/urllib_get_no_timeout.py b/examples/networking-py/urllib_get_no_timeout.py new file mode 100644 index 0000000..fe937a4 --- /dev/null +++ b/examples/networking-py/urllib_get_no_timeout.py @@ -0,0 +1,25 @@ +"""HTTP GET using urllib WITHOUT an explicit timeout. + +Same as urllib_get.py but omits the timeout= argument to urlopen(). +This exercises the code path used by mxc, where the Unikraft guest +kernel relies on the idle thread's halt_irq callback to poll sockets +via __hl_sleep rather than an explicit timeout-driven poll cycle. +""" +import urllib.request +import sys + +URL = "http://example.com/" + +print(f"Fetching {URL} (no timeout) ...") +try: + with urllib.request.urlopen(URL) as resp: + body = resp.read().decode("utf-8", errors="replace") + print(f"Status: {resp.status}") + print(f"Body length: {len(body)} bytes") + if "Example Domain" in body: + print("SUCCESS: urllib GET (no timeout) worked!") + else: + print("WARNING: unexpected body content") +except Exception as e: + print(f"FAILED: {e}") + sys.exit(1) diff --git a/examples/python-agent-driver/Dockerfile b/examples/python-agent-driver/Dockerfile index 82f8076..e97580e 100644 --- a/examples/python-agent-driver/Dockerfile +++ b/examples/python-agent-driver/Dockerfile @@ -71,6 +71,17 @@ COPY --from=deps /deps /usr/local/lib/python3.12/site-package COPY --from=driver-build /src/hl_pydriver /bin/hl_pydriver COPY pydoc_stub.py /usr/local/lib/python3.12/pydoc.py +# pip: the python-base image strips pip for size; bring it back from the +# deps stage (python:3.12-slim) so `python3 -m pip install` works as a +# subprocess. +COPY --from=deps /usr/local/lib/python3.12/site-packages/pip \ + /usr/local/lib/python3.12/site-packages/pip + +# stdlib modules that python-base strips but pip needs at runtime +COPY --from=deps /usr/local/lib/python3.12/xmlrpc \ + /usr/local/lib/python3.12/xmlrpc + + # Stage 4: pack CPIO. FROM alpine:3.20 AS cpio RUN apk add --no-cache cpio findutils ca-certificates @@ -88,4 +99,22 @@ RUN mkdir -p /rootfs/etc/ssl/certs /rootfs/usr/lib/ssl && \ cp /etc/ssl/certs/ca-certificates.crt /rootfs/etc/ssl/certs/ && \ ln -sf /etc/ssl/certs/ca-certificates.crt /rootfs/usr/lib/ssl/cert.pem +# pip configuration: disable version-check (no internet during warm-up) +RUN mkdir -p /rootfs/etc && \ + printf '[global]\ndisable-pip-version-check = true\n' > /rootfs/etc/pip.conf + +# /tmp for pip's temp files during install +RUN mkdir -p /rootfs/tmp + +# Busybox: use the static-PIE build from the shell-base image. +# Alpine's busybox-static is ET_EXEC (non-PIE) which the elfloader +# rejects; the shell-base image has a static-PIE (ET_DYN) build. +COPY --from=ghcr.io/hyperlight-dev/hyperlight-unikraft/shell-base:latest \ + /bin/busybox /rootfs/bin/busybox +RUN for cmd in sh echo ls cat grep find wc head tail sort cut sed awk \ + cp mv rm mkdir rmdir ln basename dirname env sleep date \ + hostname id uname whoami which test true tr uniq tee xargs; do \ + ln -sf busybox /rootfs/bin/$cmd; \ + done + RUN cd /rootfs && find . | cpio -o -H newc > /output.cpio 2>/dev/null diff --git a/examples/python-agent-driver/Justfile b/examples/python-agent-driver/Justfile index 2414c98..43e185b 100644 --- a/examples/python-agent-driver/Justfile +++ b/examples/python-agent-driver/Justfile @@ -30,6 +30,10 @@ run-5: [unix] rootfs: + docker build --target base -t local-python-base-dev:latest \ + -f ../../runtimes/python.Dockerfile ../../runtimes/ + docker build -t local-python-base:latest \ + -f ../../runtimes/python.Dockerfile ../../runtimes/ docker build --platform linux/amd64 --build-arg BASE=local-python-base:latest \ --target cpio -t {{image}}-cpio . - docker rm -f {{image}}-tmp diff --git a/examples/python-agent-driver/demo_busybox.py b/examples/python-agent-driver/demo_busybox.py new file mode 100644 index 0000000..7bdebbf --- /dev/null +++ b/examples/python-agent-driver/demo_busybox.py @@ -0,0 +1,22 @@ +import subprocess + +cmds = [ + (["echo", "hello from hyperlight guest"], None), + (["uname", "-a"], None), + (["ls", "/bin"], None), + (["grep", "nameserver", "/etc/resolv.conf"], None), + (["find", "/etc", "-name", "*.conf"], None), + (["wc", "-l", "/etc/resolv.conf"], None), + (["sh", "-c", "echo hello from sh"], None), +] + +for cmd, stdin in cmds: + label = " ".join(cmd) + print(f"\n$ {label}") + r = subprocess.run(cmd, capture_output=True, text=True, input=stdin) + if r.stdout: + print(r.stdout.rstrip()) + if r.stderr: + print(f"stderr: {r.stderr.rstrip()}") + if r.returncode != 0: + print(f"exit code: {r.returncode}") diff --git a/examples/python-agent-driver/demo_pip_install.py b/examples/python-agent-driver/demo_pip_install.py new file mode 100644 index 0000000..8e5ef31 --- /dev/null +++ b/examples/python-agent-driver/demo_pip_install.py @@ -0,0 +1,17 @@ +import subprocess, sys, os + +target = "/tmp/pypackages" +os.makedirs(target, exist_ok=True) + +result = subprocess.run( + [sys.executable, "-m", "pip", "install", "--target", target, "six"], + capture_output=True, text=True, +) +print(result.stdout) +if result.returncode != 0: + print(result.stderr) + sys.exit(result.returncode) + +sys.path.insert(0, target) +import six +print(f"Installed and imported six {six.__version__}") diff --git a/examples/python-agent-driver/hl_pydriver.c b/examples/python-agent-driver/hl_pydriver.c index 16c8266..e563765 100644 --- a/examples/python-agent-driver/hl_pydriver.c +++ b/examples/python-agent-driver/hl_pydriver.c @@ -110,18 +110,28 @@ static const char *fc_arg0_string(const uint8_t *fc, size_t fc_len, * FC-aware callback the kernel dispatches to * on every call after it's set */ +static uintptr_t parse_hex(const char *s) +{ + uintptr_t v = 0; + if (s[0] == '0' && (s[1] == 'x' || s[1] == 'X')) + s += 2; + for (; *s; s++) { + unsigned d; + if (*s >= '0' && *s <= '9') d = *s - '0'; + else if (*s >= 'a' && *s <= 'f') d = *s - 'a' + 10; + else if (*s >= 'A' && *s <= 'F') d = *s - 'A' + 10; + else break; + v = (v << 4) | d; + } + return v; +} + typedef void (*hl_dispatch_fn_t)(const uint8_t *fc, size_t fc_len); static const uint8_t **g_fc_bytes_slot; static size_t *g_fc_len_slot; static hl_dispatch_fn_t *g_v2_callback_slot; -/* Saved FS_BASE value captured right after Py_Initialize / warm-up - * finishes. Restored at the head of every v2-callback invocation so - * Python's TLS pointer stays valid even if something in the dispatch - * preamble (dispatch_prepare's MSR restore, Hyperlight's own - * save/restore of segment state) leaves FS_BASE pointing elsewhere. - */ static uint64_t g_py_fsbase; static inline uint64_t rdmsr_fsbase(void) @@ -158,7 +168,6 @@ static int run_code_with_exceptions(const char *code) if (!m) return 1; PyObject *d = PyModule_GetDict(m); if (!d) return 1; - PyObject *result = PyRun_String(code, Py_file_input, d, d); if (result) { Py_DECREF(result); @@ -307,14 +316,14 @@ int main(int argc, char **argv, char **envp) if (!g_fc_bytes_slot) { for (char **p = envp; p && *p; p++) { if (!strncmp(*p, "HL_FC_BYTES_PTR=", 16)) - g_fc_bytes_slot = (const uint8_t **)(uintptr_t) - strtoul(*p + 16, NULL, 16); + g_fc_bytes_slot = (const uint8_t **) + parse_hex(*p + 16); else if (!strncmp(*p, "HL_FC_LEN_PTR=", 14)) - g_fc_len_slot = (size_t *)(uintptr_t) - strtoul(*p + 14, NULL, 16); + g_fc_len_slot = (size_t *) + parse_hex(*p + 14); else if (!strncmp(*p, "HL_V2_CALLBACK_PTR=", 19)) - g_v2_callback_slot = (hl_dispatch_fn_t *)(uintptr_t) - strtoul(*p + 19, NULL, 16); + g_v2_callback_slot = (hl_dispatch_fn_t *) + parse_hex(*p + 19); } if (!g_fc_bytes_slot || !g_fc_len_slot || !g_v2_callback_slot) { diff --git a/examples/python-agent-driver/kraft.yaml b/examples/python-agent-driver/kraft.yaml index 425b327..cd1dd19 100644 --- a/examples/python-agent-driver/kraft.yaml +++ b/examples/python-agent-driver/kraft.yaml @@ -79,7 +79,9 @@ unikraft: CONFIG_LIBCONTEXT: 'y' CONFIG_LIBCONTEXT_CLEAR_TBSS: 'y' - CONFIG_LIBPOSIX_FDTAB_MULTITAB: 'n' + CONFIG_LIBPOSIX_FDTAB_MULTITAB: 'y' + + CONFIG_LIBPOSIX_PROCESS_BRK: 'y' libraries: app-elfloader: diff --git a/host/src/bin/pydriver_run.rs b/host/src/bin/pydriver_run.rs index 448f83c..88a3827 100644 --- a/host/src/bin/pydriver_run.rs +++ b/host/src/bin/pydriver_run.rs @@ -44,7 +44,7 @@ fn main() -> Result<()> { let t_evolve = Instant::now(); let mut sandbox = Sandbox::builder(&kernel) .initrd_file(&initrd) - .heap_size(1280 * 1024 * 1024) + .heap_size(2560 * 1024 * 1024) .build()?; eprintln!( "[timing] evolve={:.1}ms", diff --git a/host/src/bin/pyhl.rs b/host/src/bin/pyhl.rs index ac66a48..6f5bf2a 100644 --- a/host/src/bin/pyhl.rs +++ b/host/src/bin/pyhl.rs @@ -463,7 +463,7 @@ fn cmd_setup(args: SetupArgs) -> Result<()> { { let mut builder = Sandbox::builder(&dst_kernel) .initrd_file(&dst_initrd) - .heap_size(1280 * 1024 * 1024); + .heap_size(2560 * 1024 * 1024); for p in &setup_preopens { builder = builder.preopen(p.clone()); }