diff --git a/.github/actions/archive/action.yml b/.github/actions/archive/action.yml new file mode 100644 index 00000000000000..40a84b864e1ae6 --- /dev/null +++ b/.github/actions/archive/action.yml @@ -0,0 +1,24 @@ +# SPDX-License-Identifier: GPL-2.0 +--- +name: Archive results +description: Archive kdevops results in https://github.com/linux-kdevops/kdevops-results-archive.git +inputs: + dir: + description: 'Directory' + required: true + default: 'workdir' +runs: + using: "composite" + steps: + - name: Get systemd journal files + working-directory: ${{ inputs.dir }}/kdevops + shell: bash + run: | + make journal-dump + + - name: Build our kdevops archive results + working-directory: ${{ inputs.dir }}/kdevops + shell: bash + run: | + make ci-archive CI_WORKFLOW="${{ inputs.ci_workflow }}" + diff --git a/.github/actions/cleanup/action.yml b/.github/actions/cleanup/action.yml new file mode 100644 index 00000000000000..18a8656aa165b6 --- /dev/null +++ b/.github/actions/cleanup/action.yml @@ -0,0 +1,24 @@ +# SPDX-License-Identifier: GPL-2.0 +--- +name: Cleanup kdevops VMs +description: Destroy VMs and cleanup workspace + +inputs: + dir: + description: 'Directory' + required: true + default: 'workdir' + +runs: + using: "composite" + steps: + - name: Run kdevops make destroy + working-directory: ${{ inputs.dir }}/kdevops + shell: bash + run: | + make destroy + + - name: Remove working-directory + shell: bash + run: | + rm --recursive --force --verbose ${{ inputs.dir }} diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml new file mode 100644 index 00000000000000..37d06d9e0c8d78 --- /dev/null +++ b/.github/actions/setup/action.yml @@ -0,0 +1,181 @@ +# SPDX-License-Identifier: GPL-2.0 +--- +name: Setup kdevops +description: Setup kdevops workspace + +inputs: + dir: + description: 'Directory' + required: true + default: 'workdir' + kernel_tree: + required: false + type: string + default: 'linux' + kernel_ref: + required: false + type: string + default: 'master' + ci_workflow: + required: false + type: string + default: 'demo' + +runs: + using: "composite" + steps: + - name: Create workspace directory + shell: bash + run: | + pwd + rm --recursive --force --verbose ${{ inputs.dir }} + mkdir --parent --verbose ${{ inputs.dir }} + find . + + - name: Configure git + shell: bash + run: | + git config --global --add safe.directory '*' + git config --global user.name "kdevops" + git config --global user.email "kdevops@lists.linux.dev" + + - name: Checkout kdevops + working-directory: ${{ inputs.dir }} + shell: bash + run: | + rm --recursive --force --verbose kdevops/ + git clone https://github.com/dkruces/kdevops.git --branch ci-workflow kdevops + + - name: Checkout custom branch with delta on kdevops/linux + working-directory: ${{ inputs.dir }}/kdevops + shell: bash + run: | + set -euxo pipefail + LINUX_TREE="/mirror/${{ inputs.kernel_tree }}.git" + LINUX_TREE_REF="${{ inputs.kernel_ref }}" + git clone $LINUX_TREE linux + cd linux + git checkout $LINUX_TREE_REF + git log -1 + + - name: Make sure our repo kdevops defconfig exists + id: defconfig + working-directory: ${{ inputs.dir }}/kdevops + shell: bash + run: | + set -euxo pipefail + if [[ -z "${{ inputs.kdevops_defconfig }}" ]]; then + KDEVOPS_DEFCONFIG=${{ inputs.ci_workflow }} + else + KDEVOPS_DEFCONFIG="${{ inputs.kdevops_defconfig }}" + fi + + if [[ ! -f defconfigs/$KDEVOPS_DEFCONFIG ]]; then + echo "kdevops lacks a defconfig for this repository, expected to find: defconfigs/$KDEVOPS_DEFCONFIG" + exit 1 + fi + + "${{ github.workspace }}/scripts/github_output.sh" KDEVOPS_DEFCONFIG "$KDEVOPS_DEFCONFIG" + + - name: Initialize CI metadata for kdevops-results-archive for linux + id: metadata + working-directory: ${{ inputs.dir }}/kdevops/linux + shell: bash + run: | + set -euxo pipefail + echo "${{ inputs.kernel_tree }}" > ../ci.trigger + echo "testing" > ../ci.subject + echo "${{ inputs.kernel_ref }}" > ../ci.ref + + RELEVANT_GIT_TAG=$(cat ../ci.ref) + RELEVANT_GIT_REF=$(git rev-parse --short=12 $RELEVANT_GIT_TAG) + + "${{ github.workspace }}/scripts/github_output.sh" LINUX_GIT_REF "$RELEVANT_GIT_REF" + "${{ github.workspace }}/scripts/github_output.sh" LINUX_GIT_TAG "$RELEVANT_GIT_TAG" + + # Start out pessimistic + echo "unknown" > ../ci.result + echo "Nothing to write home about." > ../ci.commit_extra + + - name: Run a quick Linux kernel defconfig build test + working-directory: ${{ inputs.dir }}/kdevops/linux + env: + LINUX_GIT_TAG: ${{ steps.metadata.outputs.LINUX_GIT_TAG }} + shell: bash + run: | + set -euxo pipefail + git reset --hard "$LINUX_GIT_TAG" + make defconfig + make -j$(nproc) + + - name: Run kdevops make defconfig-repo + working-directory: ${{ inputs.dir }}/kdevops + env: + LINUX_GIT_TAG: ${{ steps.metadata.outputs.LINUX_GIT_TAG }} + LINUX_GIT_REF: ${{ steps.metadata.outputs.LINUX_GIT_REF }} + KDEVOPS_DEFCONFIG: ${{ steps.defconfig.outputs.KDEVOPS_DEFCONFIG }} + shell: bash + run: | + LINUX_TREE="/mirror/${{ inputs.kernel_tree }}.git" + LINUX_TREE_REF="$LINUX_GIT_TAG" + + # We make the compromise here to use a relevant git tag for the + # host prefix so that folks can easily tell what exact kernel tree + # is being tested by using the relevant git ref. That is, if you + # pushed a tree with the .github/ directory as the top of the tree, + # that commit will not be used, we'll use the last one as that is + # the relevant git ref we want to annotate a test for. + # + # The compromise here is that we expect no two same identical tests + # on the same self-hosted server. We could extend this with something + # like github.run_id but its not yet clear if we can have kdevops + # hosts with a bundled prefix ID like that ref-runid-testname. Tests + # have been done with the full lenght sha1sum though and we know that + # does work. + KDEVOPS_HOSTS_PREFIX="$LINUX_GIT_REF" + + echo "Going to use defconfig-$KDEVOPS_DEFCONFIG" + + echo "Linux tree: $LINUX_TREE" + echo "Linux trigger ref: $LINUX_TREE_REF" + echo "Linux tag: $LINUX_GIT_TAG" + echo "Runner ID: ${{ github.run_id }}" + echo "kdevops host prefix: $KDEVOPS_HOSTS_PREFIX" + echo "kdevops defconfig: defconfig-$KDEVOPS_DEFCONFIG" + + KDEVOPS_ARGS="\ + KDEVOPS_HOSTS_PREFIX=$KDEVOPS_HOSTS_PREFIX \ + LINUX_TREE=$LINUX_TREE \ + LINUX_TREE_REF=$LINUX_TREE_REF \ + ANSIBLE_CFG_CALLBACK_PLUGIN="debug" \ + defconfig-$KDEVOPS_DEFCONFIG" + echo "Going to run:" + echo "make $KDEVOPS_ARGS" + + make $KDEVOPS_ARGS + + - name: Run kdevops make + working-directory: ${{ inputs.dir }}/kdevops + shell: bash + run: | + make -j$(nproc) + + - name: Run kdevops make bringup + working-directory: ${{ inputs.dir }}/kdevops + shell: bash + run: | + ls -ld linux + make destroy + make bringup + + - name: Build linux and boot test nodes on test kernel + working-directory: ${{ inputs.dir }}/kdevops + shell: bash + run: | + make linux + + - name: Build required ci tests + working-directory: ${{ inputs.dir }}/kdevops + shell: bash + run: | + make ci-build-test CI_WORKFLOW=${{ inputs.ci_workflow }} diff --git a/.github/actions/test/action.yml b/.github/actions/test/action.yml new file mode 100644 index 00000000000000..75b95c3721c2a5 --- /dev/null +++ b/.github/actions/test/action.yml @@ -0,0 +1,44 @@ +# SPDX-License-Identifier: GPL-2.0 +--- +name: Setup kdevops +description: Setup kdevops workspace + +inputs: + dir: + description: 'Directory' + required: true + default: 'workdir' + ci_workflow: + required: false + type: string + default: 'demo' + +runs: + using: "composite" + steps: + - name: Run CI tests + working-directory: ${{ inputs.dir }}/kdevops + shell: bash + run: | + make ci-test CI_WORKFLOW="${{ inputs.ci_workflow }}" + + echo -e "Kernel tests results:\n" > ci.commit_extra + + - name: Generate CI commit info + working-directory: ${{ inputs.dir }}/kdevops + shell: bash + run: | + find workflows/blktests/results/last-run/ -name '*.dmesg.log' \ + -exec tail -n 1 {} + >> ci.commit_extra + + echo -e "\n\n" >> ci.commit_extra + echo -e "Userspace test results:\n" >> ci.commit_extra + find workflows/blktests/results/last-run/ -name '*.userspace.log' \ + -exec tail -n 1 {} + >> ci.commit_extra + echo -e "\n\n" >> ci.commit_extra + + if grep -i -q "fail" ci.commit_extra ; then + echo "fail" > ci.result + else + echo "ok" > ci.result + fi diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 00000000000000..13dba4eb512f0f --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,107 @@ +# SPDX-License-Identifier: GPL-2.0 +--- +name: Run kdevops CI Workflow - Reusable + +on: + workflow_call: + inputs: + ci_workflow: + description: "CI Workflow" + required: true + default: 'blktests_nvme' + type: string + kernel_tree: + description: "Linux kernel tree to use" + required: true + default: 'linux' + type: string + kernel_ref: + description: "Linux tree git reference (branch/tag/commit-id)" + required: true + default: 'master' + type: string + +jobs: + check_ref: + name: Check Linux kernel Git Reference + runs-on: [self-hosted] + steps: + - name: Checkout repo + uses: actions/checkout@v4 + + - name: Check kernel_ref exists + id: check_kernel_ref + run: | + set -euxo pipefail + + ref="${{ github.event.inputs.kernel_ref }}" + tree="${{ github.event.inputs.kernel_tree }}" + mirror="/mirror/${tree}.git" + ls_remote="$(git ls-remote "$mirror" "refs/*/${ref}" || true)" + contains_branch="$(git -C "$mirror" branch --contains "${ref}" || true)" + contains_tag="$(git -C "$mirror" branch --contains "${ref}" || true)" + + if [ -z "$ls_remote" ] && [ -z "$contains_branch" ] && [ -z "$contains_tag" ]; then + echo "Linux kernel ${ref} does not exist." + exit 1 + fi + + setup: + name: Setup kdevops workspace + runs-on: [self-hosted] + needs: [check_ref] + steps: + - name: Checkout kdevops-ci + uses: actions/checkout@v4 + + - name: kdevops setup + uses: ./.github/actions/setup + with: + dir: ${{ inputs.ci_workflow }} + kernel_tree: ${{ inputs.kernel_tree }} + kernel_ref: ${{ inputs.kernel_ref }} + ci_workflow: ${{ inputs.ci_workflow }} + + test: + name: Run kdevops ci-test + runs-on: [self-hosted] + needs: [setup] + steps: + - name: kdevops ci-test + uses: ./.github/actions/test + with: + dir: ${{ inputs.ci_workflow }} + ci_workflow: ${{ inputs.ci_workflow }} + + archive: + name: Archive kdevops + runs-on: [self-hosted] + needs: [setup, test] + steps: + - name: Start SSH Agent + uses: webfactory/ssh-agent@v0.9.0 + with: + ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }} + + - name: Archive ci-test results + uses: ./.github/actions/archive + with: + dir: ${{ inputs.ci_workflow }} + + - name: Upload our kdevops results archive + uses: actions/upload-artifact@v4 + with: + name: kdevops-ci-results + path: ${{ inputs.ci_workflow }}/kdevops/archive/*.zip + + cleanup: + name: Cleanup kdevops workspace + runs-on: [self-hosted] + needs: [setup, test, archive] + if: always() + steps: + - name: kdevops cleanup + uses: ./.github/actions/cleanup + with: + dir: ${{ inputs.ci_workflow }} + ci_workflow: ${{ inputs.ci_workflow }} diff --git a/.github/workflows/manual.yml b/.github/workflows/manual.yml new file mode 100644 index 00000000000000..7b570265c45685 --- /dev/null +++ b/.github/workflows/manual.yml @@ -0,0 +1,58 @@ +# SPDX-License-Identifier: GPL-2.0 +--- +name: Run kdevops CI Workflow - Manual + +on: + workflow_dispatch: + inputs: + ci_workflow: + description: "CI Workflow" + required: true + default: 'blktests_nvme' + type: choice + options: + - blktests + - blktests_block + - blktests_loop + - blktests_meta + - blktests_nbd + - blktests_nvme + - blktests_nvmemp + - blktests_scsi + - blktests_srp + - blktests_zbd + - tmpfs + - tmpfs_default + - tmpfs_huge + - tmpfs_noswap + - linux-btrfs-kpd + - linux-ext4-kpd + - linux-firmware-kpd + - linux-mm-kpd + - linux-modules-kpd + - linux-xfs-kpd + - selftests + kernel_tree: + description: "Linux kernel tree to use" + required: true + default: 'linux' + type: choice + options: + - linux + - linux-next + - linux-stable + kernel_ref: + description: "Linux tree git reference (branch/tag/commit-id)" + required: true + default: 'master' + type: string + +jobs: + manual: + name: Manual kdevops CI + uses: ./.github/workflows/main.yml + secrets: inherit + with: + ci_workflow: ${{ inputs.ci_workflow }} + kernel_ref: ${{ inputs.kernel_ref }} + kernel_tree: ${{ inputs.kernel_tree }} diff --git a/.github/workflows/schedule.yml b/.github/workflows/schedule.yml new file mode 100644 index 00000000000000..d0221550845fa6 --- /dev/null +++ b/.github/workflows/schedule.yml @@ -0,0 +1,50 @@ +# SPDX-License-Identifier: GPL-2.0 +--- +name: Run kdevops CI Workflow - Schedule + +on: + schedule: + - cron: '0 14 * * *' + + workflow_dispatch: + +jobs: + check_ref: + name: Check Linux kernel Git Reference + outputs: + kernel_ref: ${{ steps.check_kernel_ref.outputs.kernel_ref }} + kernel_tree: ${{ steps.check_kernel_ref.outputs.kernel_tree }} + runs-on: [self-hosted] + steps: + - name: Checkout repo + uses: actions/checkout@v4 + + - name: Generate kernel_ref + id: check_kernel_ref + run: | + set -euxo pipefail + day_of_week=$(date +%u) + + kernel_ref=$(./scripts/korg-releases.py --moniker linux-next) + kernel_tree=linux-next + if [ "$day_of_week" -eq 1 ]; then + kernel_ref=$(./scripts/korg-releases.py --moniker mainline) + kernel_tree=linux + fi + + "${{ github.workspace }}/scripts/github_output.sh" kernel_ref "$kernel_ref" + "${{ github.workspace }}/scripts/github_output.sh" kernel_tree "$kernel_tree" + + schedule: + name: Scheduled kdevops CI + needs: [check_ref] + runs-on: [self-hosted] + secrets: inherit + uses: ./.github/workflows/main.yml + strategy: + matrix: + ci_workflow: [blktests-block, blktests-loop, blktests-meta] + with: + ci_workflow: ${{ matrix.ci_workflow }} + kernel_ref: ${{ needs.check_ref.outputs.kernel_ref }} + kernel_tree: ${{ needs.check_ref.outputs.kernel_tree }} diff --git a/rust/Makefile b/rust/Makefile index fa0eea8a9eca22..404a77d2237f96 100644 --- a/rust/Makefile +++ b/rust/Makefile @@ -192,6 +192,7 @@ quiet_cmd_rustdoc_test = RUSTDOC T $< RUST_MODFILE=test.rs \ OBJTREE=$(abspath $(objtree)) \ $(RUSTDOC) --test $(rust_common_flags) \ + -Zcrate-attr='feature(used_with_arg)' \ @$(objtree)/include/generated/rustc_cfg \ $(rustc_target_flags) $(rustdoc_test_target_flags) \ $(rustdoc_test_quiet) \ diff --git a/rust/kernel/firmware.rs b/rust/kernel/firmware.rs index f04b058b09b2d2..b28bcf60b98c5c 100644 --- a/rust/kernel/firmware.rs +++ b/rust/kernel/firmware.rs @@ -198,7 +198,7 @@ macro_rules! module_firmware { }; #[link_section = ".modinfo"] - #[used] + #[used(compiler)] static __MODULE_FIRMWARE: [u8; $($builder)*::create(__MODULE_FIRMWARE_PREFIX) .build_length()] = $($builder)*::create(__MODULE_FIRMWARE_PREFIX).build(); }; diff --git a/rust/kernel/kunit.rs b/rust/kernel/kunit.rs index 1604fb6a5b1b00..cf95700c8612b4 100644 --- a/rust/kernel/kunit.rs +++ b/rust/kernel/kunit.rs @@ -278,7 +278,7 @@ macro_rules! kunit_unsafe_test_suite { is_init: false, }; - #[used] + #[used(compiler)] #[allow(unused_unsafe)] #[cfg_attr(not(target_os = "macos"), link_section = ".kunit_test_suites")] static mut KUNIT_TEST_SUITE_ENTRY: *const ::kernel::bindings::kunit_suite = diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index de07aadd1ff5fe..2bdf9d14ec43ac 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -26,6 +26,8 @@ #![feature(const_mut_refs)] #![feature(const_ptr_write)] #![feature(const_refs_to_cell)] +// To be determined. +#![feature(used_with_arg)] // Ensure conditional compilation based on the kernel configuration works; // otherwise we may silently break things like initcall handling. diff --git a/rust/macros/module.rs b/rust/macros/module.rs index a9418fbc9b4453..e9b55930989457 100644 --- a/rust/macros/module.rs +++ b/rust/macros/module.rs @@ -57,7 +57,7 @@ impl<'a> ModInfoBuilder<'a> { {cfg} #[doc(hidden)] #[cfg_attr(not(target_os = \"macos\"), link_section = \".modinfo\")] - #[used] + #[used(compiler)] pub static __{module}_{counter}: [u8; {length}] = *{string}; ", cfg = if builtin { @@ -256,7 +256,7 @@ pub(crate) fn module(ts: TokenStream) -> TokenStream { // key or a new section. For the moment, keep it simple. #[cfg(MODULE)] #[doc(hidden)] - #[used] + #[used(compiler)] static __IS_RUST_MODULE: () = (); static mut __MOD: core::mem::MaybeUninit<{type_}> = @@ -280,7 +280,7 @@ pub(crate) fn module(ts: TokenStream) -> TokenStream { #[cfg(MODULE)] #[doc(hidden)] - #[used] + #[used(compiler)] #[link_section = \".init.data\"] static __UNIQUE_ID___addressable_init_module: unsafe extern \"C\" fn() -> i32 = init_module; @@ -299,7 +299,7 @@ pub(crate) fn module(ts: TokenStream) -> TokenStream { #[cfg(MODULE)] #[doc(hidden)] - #[used] + #[used(compiler)] #[link_section = \".exit.data\"] static __UNIQUE_ID___addressable_cleanup_module: extern \"C\" fn() = cleanup_module; @@ -309,7 +309,7 @@ pub(crate) fn module(ts: TokenStream) -> TokenStream { #[cfg(not(CONFIG_HAVE_ARCH_PREL32_RELOCATIONS))] #[doc(hidden)] #[link_section = \"{initcall_section}\"] - #[used] + #[used(compiler)] pub static __{name}_initcall: extern \"C\" fn() -> kernel::ffi::c_int = __{name}_init; #[cfg(not(MODULE))] diff --git a/scripts/Makefile.build b/scripts/Makefile.build index 13dcd86e74ca83..eb0d27812aec29 100644 --- a/scripts/Makefile.build +++ b/scripts/Makefile.build @@ -222,7 +222,7 @@ $(obj)/%.lst: $(obj)/%.c FORCE # Compile Rust sources (.rs) # --------------------------------------------------------------------------- -rust_allowed_features := asm_const,asm_goto,arbitrary_self_types,lint_reasons,raw_ref_op +rust_allowed_features := asm_const,asm_goto,arbitrary_self_types,lint_reasons,raw_ref_op,used_with_arg # `--out-dir` is required to avoid temporaries being created by `rustc` in the # current working directory, which may be not accessible in the out-of-tree diff --git a/scripts/github_output.sh b/scripts/github_output.sh new file mode 100755 index 00000000000000..711306c38cb5ff --- /dev/null +++ b/scripts/github_output.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# +# Usage: ./github_output.sh key value +set -euxo pipefail + +key="$1" +value="$2" + +echo "$key=$value" >> "$GITHUB_OUTPUT" diff --git a/scripts/korg-releases.py b/scripts/korg-releases.py new file mode 100755 index 00000000000000..bce182e28d7926 --- /dev/null +++ b/scripts/korg-releases.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: copyleft-next-0.3.1 +import sys +import logging +import argparse +import json +import urllib.request +import socket +import re + + +def parser(): + parser = argparse.ArgumentParser(description="kernel.org/releases.json checker") + parser.add_argument("--debug", action="store_true", help="debug") + parser.add_argument( + "--moniker", + help="moniker (mainline, stable, longterm or linux-next)", + required=True, + ) + parser.add_argument( + "--pname", + help="project name for User-Agent request", + default="kdevops", + ) + parser.add_argument( + "--pversion", + help="project version for User-Agent request", + default="5.0.2", + ) + return parser + + +def _check_connection(host, port, timeout=2): + try: + with socket.create_connection((host, port), timeout): + logging.debug(f"Connection to {host} on port {port} succeeded!") + return True + except (socket.timeout, socket.error) as e: + logging.debug(f"Connection to {host} on port {port} failed: {e}") + return False + + +def kreleases(args) -> None: + """Get the latest kernel releases from kernel.org/releases.json""" + + reflist = [] + if _check_connection("kernel.org", 80): + _url = "https://www.kernel.org/releases.json" + req = urllib.request.Request( + _url, + headers={ + "User-Agent": f"{args.pname}/{args.pversion} (kdevops@lists.linux.dev)" + }, + ) + with urllib.request.urlopen(req) as url: + data = json.load(url) + + for release in data["releases"]: + if release["moniker"] == args.moniker: + # Check if release.json is aa.bb.cc type + if re.compile(r'^\d+\.\d+(\.\d+|-rc\d+)?$').match(release["version"]): + reflist.append("v" + release["version"]) + else: + reflist.append(release["version"]) + + logging.debug(f"{reflist}") + for r in reflist: + print(r) + + +def main() -> None: + """Kconfig choice generator for git refereces""" + log = logging.getLogger() + log.setLevel(logging.INFO) + p = parser() + args, _ = p.parse_known_args() + if args.debug: + log.setLevel(logging.DEBUG) + + kreleases(args) + + +if __name__ == "__main__": + ret = 0 + try: + main() + except Exception: + ret = 1 + import traceback + + traceback.print_exc() + sys.exit(ret)