diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml index fa03ffb6eb..c4aacd6b88 100644 --- a/.github/actionlint.yaml +++ b/.github/actionlint.yaml @@ -3,5 +3,4 @@ self-hosted-runner: labels: - fireactions-turbo-16 - fireactions-turbo-8 - - fireactions-tryruntime - Benchmarking diff --git a/.github/actions/run-typescript-e2e/action.yml b/.github/actions/run-typescript-e2e/action.yml new file mode 100644 index 0000000000..b79a3d3854 --- /dev/null +++ b/.github/actions/run-typescript-e2e/action.yml @@ -0,0 +1,52 @@ +name: Run TypeScript E2E suite +description: Download a prebuilt node and run one Moonwall environment. + +inputs: + binary: + description: Node artifact variant to download. + required: true + test: + description: Moonwall environment to run. + required: true + +runs: + using: composite + steps: + - name: Download binary + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: node-subtensor-${{ inputs.binary }} + path: target/release + + - name: Make binary executable + shell: bash + run: chmod +x target/release/node-subtensor + + - name: Setup Node.js + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version-file: ts-tests/.nvmrc + + - name: Setup pnpm + uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4 + with: + version: 10 + + - name: Install e2e dependencies + shell: bash + working-directory: ts-tests + run: pnpm install --frozen-lockfile + + - name: Install lsof (dev foundation RPC port discovery) + if: inputs.test == 'dev' + shell: bash + run: | + sudo DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=a apt-get update + sudo DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=a apt-get install -y --no-install-recommends lsof + + - name: Run tests + shell: bash + working-directory: ts-tests + env: + MOONWALL_TEST: ${{ inputs.test }} + run: pnpm moonwall test "$MOONWALL_TEST" diff --git a/.github/scripts/classify-runtime-changes.sh b/.github/scripts/classify-runtime-changes.sh new file mode 100755 index 0000000000..521142e17f --- /dev/null +++ b/.github/scripts/classify-runtime-changes.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if [[ $# -ne 1 ]]; then + echo "usage: classify-runtime-changes.sh OUTPUT_FILE" >&2 + exit 2 +fi + +output_file="$1" + +runtime=false +docs=false +python_sdk=false +sdk_drift=false +snapshot_ci=false + +# Keep path ownership explicit. SDK-only changes are covered by sdk-checks and +# the Rust SDK e2e workflow; they should not force clone-upgrade or SDK drift. +while IFS= read -r path; do + case "$path" in + common/*|node/*|pallets/*|precompiles/*|primitives/*|runtime/*|support/*|chain-extensions/*|src/*|vendor/*|Cargo.toml|build.rs|rust-toolchain.toml) + runtime=true + sdk_drift=true + ;; + clones/*|website/apps/bittensor-website/scripts/*) + runtime=true + ;; + .github/workflows/runtime-checks.yml|.github/workflows/refresh-mainnet-snapshot.yml|.github/actions/rust-setup/*|.github/actions/sccache-setup/*|.github/scripts/sccache-configure.sh) + runtime=true + ;; + .github/scripts/classify-runtime-changes.sh|.github/scripts/test-runtime-change-filter.sh|.github/scripts/snapshot-artifact.sh|.github/scripts/test-snapshot-artifact.sh) + runtime=true + snapshot_ci=true + ;; + esac + + case "$path" in + website/*|sdk/python/*|.github/workflows/runtime-checks.yml) docs=true ;; + esac + + case "$path" in + sdk/python/*|sdk/bittensor-core/*|sdk/bittensor-core-py/*|sdk/bittensor-core-wasm/*|Cargo.lock|.github/workflows/runtime-checks.yml) + python_sdk=true + ;; + esac + + case "$path" in + .github/workflows/runtime-checks.yml|.github/workflows/refresh-mainnet-snapshot.yml) + snapshot_ci=true + ;; + esac +done + +echo "runtime=$runtime, docs=$docs, python_sdk=$python_sdk, sdk_drift=$sdk_drift, snapshot_ci=$snapshot_ci" +{ + echo "runtime=$runtime" + echo "docs=$docs" + echo "python_sdk=$python_sdk" + echo "sdk_drift=$sdk_drift" + echo "snapshot_ci=$snapshot_ci" +} >> "$output_file" diff --git a/.github/scripts/snapshot-artifact.sh b/.github/scripts/snapshot-artifact.sh index 0edaa9e0b2..d2c977ecf6 100755 --- a/.github/scripts/snapshot-artifact.sh +++ b/.github/scripts/snapshot-artifact.sh @@ -6,13 +6,11 @@ usage() { cat >&2 <<'EOF' Usage: snapshot-artifact.sh mode EVENT_NAME LABELS_JSON MANUAL_FRESH OUTPUT_FILE - snapshot-artifact.sh select ARTIFACT_NAME BRANCH REPOSITORY_ID MAX_AGE_HOURS OUTPUT_FILE [required|optional] - snapshot-artifact.sh download ARTIFACT_ID DIGEST DESTINATION - snapshot-artifact.sh validate MANIFEST_FILE SNAPSHOT_FILE NETWORK GENESIS_HASH CLI_VERSION + snapshot-artifact.sh select ARTIFACT_NAME BRANCH REPOSITORY_ID WORKFLOW_PATH MAX_AGE_HOURS OUTPUT_FILE [required|optional] + snapshot-artifact.sh validate MANIFEST_FILE SNAPSHOT_FILE NETWORK GENESIS_HASH CLI_VERSION PRODUCER_SHA -For tests, set ARTIFACTS_JSON_FILE instead of calling the GitHub API, -ARTIFACT_ZIP_FILE instead of downloading an artifact, and NOW_EPOCH to -override the current time. +For tests, set ARTIFACTS_JSON_FILE and WORKFLOW_RUNS_JSON_FILE instead of +calling the GitHub API and NOW_EPOCH to override the current time. EOF exit 2 } @@ -62,16 +60,21 @@ resolve_mode() { } select_artifact() { - [[ $# -eq 6 ]] || usage + [[ $# -eq 7 ]] || usage local artifact_name="$1" local branch="$2" local repository_id="$3" - local max_age_hours="$4" - local output_file="$5" - local requirement="$6" - local payload now candidate created_epoch age_seconds age_hours + local workflow_path="$4" + local max_age_hours="$5" + local output_file="$6" + local requirement="$7" + local payload workflow_runs workflow_file now candidate created_epoch age_seconds age_hours [[ "$repository_id" =~ ^[0-9]+$ ]] || { echo "invalid repository id: $repository_id" >&2; exit 2; } + [[ "$workflow_path" =~ ^\.github/workflows/[A-Za-z0-9._-]+\.ya?ml$ ]] || { + echo "invalid workflow path: $workflow_path" >&2 + exit 2 + } [[ "$max_age_hours" =~ ^[0-9]+$ ]] || { echo "invalid maximum age: $max_age_hours" >&2; exit 2; } [[ "$requirement" == required || "$requirement" == optional ]] || usage @@ -82,30 +85,58 @@ select_artifact() { payload=$(gh api \ "repos/$GITHUB_REPOSITORY/actions/artifacts?name=$artifact_name&per_page=100") fi + if [[ -n "${WORKFLOW_RUNS_JSON_FILE:-}" ]]; then + workflow_runs=$(<"$WORKFLOW_RUNS_JSON_FILE") + else + workflow_file="${workflow_path##*/}" + workflow_runs=$(gh api --method GET \ + "repos/$GITHUB_REPOSITORY/actions/workflows/$workflow_file/runs" \ + -f branch="$branch" -F per_page=100 -F exclude_pull_requests=true \ + --jq '{workflow_runs: [.workflow_runs[] | { + id, path, head_branch, head_sha, + repository: {id: .repository.id}, + head_repository: {id: .head_repository.id} + }]}') + fi now="${NOW_EPOCH:-$(date -u +%s)}" [[ "$now" =~ ^[0-9]+$ ]] || { echo "invalid NOW_EPOCH: $now" >&2; exit 2; } - candidate=$(jq -cer \ + candidate=$(jq -ncer \ --arg name "$artifact_name" \ --arg branch "$branch" \ + --arg workflow_path "$workflow_path" \ --argjson repository_id "$repository_id" \ --argjson now "$now" \ - --argjson max_age_seconds "$((max_age_hours * 3600))" ' + --argjson max_age_seconds "$((max_age_hours * 3600))" \ + --argjson artifacts "$payload" \ + --argjson workflow_runs "$workflow_runs" ' + ($workflow_runs.workflow_runs + | map(select( + .path == $workflow_path and + .head_branch == $branch and + .repository.id == $repository_id and + .head_repository.id == $repository_id and + (.head_sha | test("^[0-9a-f]{40}$")) + )) + | map({key: (.id | tostring), value: .head_sha}) + | from_entries) as $trusted_runs + | [ - .artifacts[] + $artifacts.artifacts[] | select(.name == $name) | select(.expired == false) | select(.workflow_run.head_branch == $branch) | select(.workflow_run.repository_id == $repository_id) | select(.workflow_run.head_repository_id == $repository_id) + | select($trusted_runs[(.workflow_run.id | tostring)] == .workflow_run.head_sha) | .created_epoch = (.created_at | fromdateiso8601) | select(.created_epoch <= $now) | select(($now - .created_epoch) <= $max_age_seconds) ] | sort_by(.created_epoch) | last // empty - ' <<<"$payload" 2>/dev/null || true) + ' 2>/dev/null || true) if [[ -z "$candidate" ]]; then set_output "$output_file" found false @@ -124,57 +155,23 @@ select_artifact() { set_output "$output_file" found true set_output "$output_file" artifact-id "$(jq -er '.id' <<<"$candidate")" set_output "$output_file" run-id "$(jq -er '.workflow_run.id' <<<"$candidate")" + set_output "$output_file" producer-sha "$(jq -er '.workflow_run.head_sha' <<<"$candidate")" set_output "$output_file" created-at "$(jq -er '.created_at' <<<"$candidate")" set_output "$output_file" age-hours "$age_hours" - set_output "$output_file" size-bytes "$(jq -er '.size_in_bytes' <<<"$candidate")" - set_output "$output_file" digest "$(jq -er '.digest // ""' <<<"$candidate")" - if ((age_seconds > 36 * 3600)); then echo "::warning::$artifact_name is ${age_hours}h old; refresh is expected daily and this artifact becomes unusable after ${max_age_hours}h." fi echo "Selected $artifact_name artifact $(jq -er '.id' <<<"$candidate") from run $(jq -er '.workflow_run.id' <<<"$candidate") (${age_hours}h old)." } -download_artifact() { - [[ $# -eq 3 ]] || usage - local artifact_id="$1" - local digest="$2" - local destination="$3" - local archive actual_digest - - [[ "$artifact_id" =~ ^[0-9]+$ ]] || { echo "invalid artifact id: $artifact_id" >&2; exit 2; } - [[ "$digest" =~ ^sha256:[0-9a-f]{64}$ ]] || { echo "invalid artifact digest: $digest" >&2; exit 2; } - - archive=$(mktemp) - trap 'rm -f "$archive"' RETURN - if [[ -n "${ARTIFACT_ZIP_FILE:-}" ]]; then - cp "$ARTIFACT_ZIP_FILE" "$archive" - else - : "${GITHUB_REPOSITORY:?GITHUB_REPOSITORY must be set}" - gh api \ - -H 'Accept: application/vnd.github+json' \ - "repos/$GITHUB_REPOSITORY/actions/artifacts/$artifact_id/zip" > "$archive" - fi - - actual_digest="sha256:$(sha256sum "$archive" | awk '{print $1}')" - [[ "$actual_digest" == "$digest" ]] || { - echo "artifact archive checksum mismatch: expected $digest, got $actual_digest" >&2 - exit 1 - } - mkdir -p "$destination" - unzip -q "$archive" -d "$destination" - rm -f "$archive" - trap - RETURN - echo "Downloaded and verified artifact $artifact_id." -} - validate_manifest() { - [[ $# -eq 5 ]] || usage + [[ $# -eq 6 ]] || usage local manifest_file="$1" local snapshot_file="$2" local network="$3" local genesis_hash="$4" local cli_version="$5" + local producer_sha="$6" local expected_file expected_size expected_sha actual_size actual_sha [[ -f "$manifest_file" ]] || { echo "missing snapshot manifest: $manifest_file" >&2; exit 1; } @@ -183,7 +180,8 @@ validate_manifest() { jq -e \ --arg network "$network" \ --arg genesis "$genesis_hash" \ - --arg cli "$cli_version" ' + --arg cli "$cli_version" \ + --arg producer_sha "$producer_sha" ' .schema_version == 1 and .kind == "try-runtime-state" and .network == $network and @@ -194,7 +192,7 @@ validate_manifest() { (.source_spec_name | type == "string" and length > 0) and (.source_spec_version | type == "number") and (.created_at | fromdateiso8601 | type == "number") and - (.producer_sha | test("^[0-9a-f]{40}$")) and + .producer_sha == $producer_sha and (.snapshot_file | type == "string" and length > 0) and (.snapshot_size_bytes | type == "number") and (.snapshot_sha256 | test("^[0-9a-f]{64}$")) @@ -231,7 +229,6 @@ shift || true case "$command" in mode) resolve_mode "$@" ;; select) select_artifact "$@" ;; - download) download_artifact "$@" ;; validate) validate_manifest "$@" ;; *) usage ;; esac diff --git a/.github/scripts/test-runtime-change-filter.sh b/.github/scripts/test-runtime-change-filter.sh new file mode 100755 index 0000000000..8ee778d3cb --- /dev/null +++ b/.github/scripts/test-runtime-change-filter.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash + +set -euo pipefail + +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +classifier="$script_dir/classify-runtime-changes.sh" +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT + +assert_classification() { + local path="$1" expected="$2" output="$tmp/output" + : > "$output" + printf '%s\n' "$path" | "$classifier" "$output" >/dev/null + diff -u <(printf '%s\n' "$expected") "$output" +} + +all_false=$'runtime=false\ndocs=false\npython_sdk=false\nsdk_drift=false\nsnapshot_ci=false' +runtime_only=$'runtime=true\ndocs=false\npython_sdk=false\nsdk_drift=false\nsnapshot_ci=false' +runtime_and_sdk=$'runtime=true\ndocs=false\npython_sdk=false\nsdk_drift=true\nsnapshot_ci=false' +runtime_and_docs=$'runtime=true\ndocs=true\npython_sdk=false\nsdk_drift=false\nsnapshot_ci=false' +runtime_and_snapshot_only=$'runtime=true\ndocs=false\npython_sdk=false\nsdk_drift=false\nsnapshot_ci=true' +runtime_and_snapshot=$'runtime=true\ndocs=true\npython_sdk=true\nsdk_drift=false\nsnapshot_ci=true' +docs_and_python=$'runtime=false\ndocs=true\npython_sdk=true\nsdk_drift=false\nsnapshot_ci=false' +python_only=$'runtime=false\ndocs=false\npython_sdk=true\nsdk_drift=false\nsnapshot_ci=false' + +assert_classification README.md "$all_false" +assert_classification .github/actions/rust-setup/action.yml "$runtime_only" +assert_classification .github/actions/sccache-setup/action.yml "$runtime_only" +assert_classification .github/scripts/classify-runtime-changes.sh "$runtime_and_snapshot_only" +assert_classification .github/scripts/test-runtime-change-filter.sh "$runtime_and_snapshot_only" +assert_classification clones/scripts/start-local-clone-and-wait.sh "$runtime_only" +assert_classification .github/workflows/refresh-mainnet-snapshot.yml "$runtime_and_snapshot_only" +assert_classification .github/workflows/runtime-checks.yml "$runtime_and_snapshot" +assert_classification website/apps/bittensor-website/scripts/generate-metadata.mjs "$runtime_and_docs" +assert_classification sdk/bittensor-core/src/lib.rs "$python_only" +assert_classification Cargo.lock "$python_only" +assert_classification rust-toolchain.toml "$runtime_and_sdk" +assert_classification $'README.md\nsdk/python/example.py' "$docs_and_python" +assert_classification $'README.md\nnode/src/renamed-service.rs' "$runtime_and_sdk" + +echo "runtime change filter tests passed" diff --git a/.github/scripts/test-snapshot-artifact.sh b/.github/scripts/test-snapshot-artifact.sh index 203270731b..a76d59f5d8 100755 --- a/.github/scripts/test-snapshot-artifact.sh +++ b/.github/scripts/test-snapshot-artifact.sh @@ -13,14 +13,34 @@ now=$(jq -nr '"2026-07-12T14:00:00Z" | fromdateiso8601') repo_id=608683796 write_artifacts() { - jq -n --argjson artifacts "$1" '{artifacts: $artifacts}' > "$tmp/artifacts.json" + local artifacts="$1" + jq -n --argjson artifacts "$artifacts" '{artifacts: $artifacts}' > "$tmp/artifacts.json" + jq -n --argjson artifacts "$artifacts" ' + { + workflow_runs: [ + $artifacts[] + | { + id: .workflow_run.id, + path: ".github/workflows/refresh-mainnet-snapshot.yml", + head_branch: .workflow_run.head_branch, + head_sha: .workflow_run.head_sha, + repository: {id: .workflow_run.repository_id}, + head_repository: {id: .workflow_run.head_repository_id}, + conclusion: (.producer_conclusion // "success") + } + ] | unique_by(.id) + } + ' > "$tmp/workflow-runs.json" } artifact() { local id="$1" name="$2" created="$3" branch="$4" repository_id="$5" expired="$6" run_id="$7" + local head_sha + printf -v head_sha '%040x' "$run_id" jq -nc \ --argjson id "$id" --arg name "$name" --arg created "$created" --arg branch "$branch" \ - --argjson repository_id "$repository_id" --argjson expired "$expired" --argjson run_id "$run_id" ' + --argjson repository_id "$repository_id" --argjson expired "$expired" --argjson run_id "$run_id" \ + --arg head_sha "$head_sha" ' { id: $id, name: $name, @@ -32,7 +52,8 @@ artifact() { id: $run_id, repository_id: $repository_id, head_repository_id: $repository_id, - head_branch: $branch + head_branch: $branch, + head_sha: $head_sha } } ' @@ -43,8 +64,11 @@ select_fixture() { local output="$tmp/output" local status=0 : > "$output" - ARTIFACTS_JSON_FILE="$tmp/artifacts.json" NOW_EPOCH="$now" \ - "$helper" select try-runtime-snap-v0.10.1-mainnet main "$repo_id" 72 "$output" "$requirement" || status=$? + ARTIFACTS_JSON_FILE="$tmp/artifacts.json" \ + WORKFLOW_RUNS_JSON_FILE="$tmp/workflow-runs.json" \ + NOW_EPOCH="$now" \ + "$helper" select try-runtime-snap-v0.10.1-mainnet main "$repo_id" \ + .github/workflows/refresh-mainnet-snapshot.yml 72 "$output" "$requirement" || status=$? cat "$output" return "$status" } @@ -76,26 +100,19 @@ valid_new=$(artifact 12 try-runtime-snap-v0.10.1-mainnet 2026-07-12T02:00:00Z ma wrong_branch=$(artifact 13 try-runtime-snap-v0.10.1-mainnet 2026-07-12T13:00:00Z feature "$repo_id" false 103) wrong_repo=$(artifact 14 try-runtime-snap-v0.10.1-mainnet 2026-07-12T13:30:00Z main 999 false 104) expired=$(artifact 15 try-runtime-snap-v0.10.1-mainnet 2026-07-12T13:45:00Z main "$repo_id" true 105) -write_artifacts "$(jq -nc --argjson a "$valid_old" --argjson b "$valid_new" --argjson c "$wrong_branch" --argjson d "$wrong_repo" --argjson e "$expired" '[ $a, $b, $c, $d, $e ]')" +wrong_workflow=$(artifact 16 try-runtime-snap-v0.10.1-mainnet 2026-07-12T13:50:00Z main "$repo_id" false 106) +wrong_sha=$(artifact 17 try-runtime-snap-v0.10.1-mainnet 2026-07-12T13:55:00Z main "$repo_id" false 107) +write_artifacts "$(jq -nc --argjson a "$valid_old" --argjson b "$valid_new" --argjson c "$wrong_branch" --argjson d "$wrong_repo" --argjson e "$expired" --argjson f "$wrong_workflow" --argjson g "$wrong_sha" '[ $a, $b, $c, $d, $e, $f, $g ]')" +jq '(.workflow_runs[] | select(.id == 106) | .path) = ".github/workflows/untrusted-producer.yml"' \ + "$tmp/workflow-runs.json" > "$tmp/workflow-runs-updated.json" +mv "$tmp/workflow-runs-updated.json" "$tmp/workflow-runs.json" +jq '(.workflow_runs[] | select(.id == 107) | .head_sha) = "ffffffffffffffffffffffffffffffffffffffff"' \ + "$tmp/workflow-runs.json" > "$tmp/workflow-runs-updated.json" +mv "$tmp/workflow-runs-updated.json" "$tmp/workflow-runs.json" selected=$(select_fixture) grep -qx 'artifact-id=12' <<<"$selected" grep -qx 'run-id=102' <<<"$selected" - -# The selected immutable artifact ID is downloaded as its archive, verified -# against the API digest, and only then extracted. -mkdir -p "$tmp/archive-input" -printf 'artifact payload' > "$tmp/archive-input/payload.txt" -(cd "$tmp/archive-input" && zip -q "$tmp/artifact.zip" payload.txt) -archive_digest="sha256:$(sha256sum "$tmp/artifact.zip" | awk '{print $1}')" -ARTIFACT_ZIP_FILE="$tmp/artifact.zip" \ - "$helper" download 12 "$archive_digest" "$tmp/downloaded" >/dev/null -grep -qx 'artifact payload' "$tmp/downloaded/payload.txt" -if ARTIFACT_ZIP_FILE="$tmp/artifact.zip" \ - "$helper" download 12 sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb \ - "$tmp/bad-download" >/dev/null 2>&1; then - echo "expected artifact archive checksum mismatch to fail" >&2 - exit 1 -fi +grep -qx 'producer-sha=0000000000000000000000000000000000000066' <<<"$selected" # Exactly 72 hours is accepted, one second older fails, and optional lookup # reports a miss. An artifact older than 36 hours remains usable with a warning. @@ -108,8 +125,10 @@ warning_age=$(artifact 18 try-runtime-snap-v0.10.1-mainnet 2026-07-11T01:59:59Z write_artifacts "[$warning_age]" warning_output="$tmp/warning-output" : > "$warning_output" -warning_log=$(ARTIFACTS_JSON_FILE="$tmp/artifacts.json" NOW_EPOCH="$now" \ - "$helper" select try-runtime-snap-v0.10.1-mainnet main "$repo_id" 72 "$warning_output" required) +warning_log=$(ARTIFACTS_JSON_FILE="$tmp/artifacts.json" \ + WORKFLOW_RUNS_JSON_FILE="$tmp/workflow-runs.json" NOW_EPOCH="$now" \ + "$helper" select try-runtime-snap-v0.10.1-mainnet main "$repo_id" \ + .github/workflows/refresh-mainnet-snapshot.yml 72 "$warning_output" required) grep -q '^::warning::' <<<"$warning_log" too_old=$(artifact 20 try-runtime-snap-v0.10.1-mainnet 2026-07-09T13:59:59Z main "$repo_id" false 200) @@ -146,17 +165,27 @@ jq -n \ ' > "$tmp/mainnet.manifest.json" "$helper" validate "$tmp/mainnet.manifest.json" "$tmp/mainnet.snap" mainnet \ - 0x2f0555cc76fc2840a25a6ea3b9637146806f1f44b090c175ffde2a7e5ab36c03 0.10.1 >/dev/null + 0x2f0555cc76fc2840a25a6ea3b9637146806f1f44b090c175ffde2a7e5ab36c03 \ + 0.10.1 647ca2b0493ed5c74399b73f2595643ba785c1b8 >/dev/null + +if "$helper" validate "$tmp/mainnet.manifest.json" "$tmp/mainnet.snap" mainnet \ + 0x2f0555cc76fc2840a25a6ea3b9637146806f1f44b090c175ffde2a7e5ab36c03 \ + 0.10.1 ffffffffffffffffffffffffffffffffffffffff >/dev/null 2>&1; then + echo "expected producer SHA mismatch to fail" >&2 + exit 1 +fi if "$helper" validate "$tmp/mainnet.manifest.json" "$tmp/mainnet.snap" testnet \ - 0x2f0555cc76fc2840a25a6ea3b9637146806f1f44b090c175ffde2a7e5ab36c03 0.10.1 >/dev/null 2>&1; then + 0x2f0555cc76fc2840a25a6ea3b9637146806f1f44b090c175ffde2a7e5ab36c03 \ + 0.10.1 647ca2b0493ed5c74399b73f2595643ba785c1b8 >/dev/null 2>&1; then echo "expected network mismatch to fail" >&2 exit 1 fi printf 'corrupt' >> "$tmp/mainnet.snap" if "$helper" validate "$tmp/mainnet.manifest.json" "$tmp/mainnet.snap" mainnet \ - 0x2f0555cc76fc2840a25a6ea3b9637146806f1f44b090c175ffde2a7e5ab36c03 0.10.1 >/dev/null 2>&1; then + 0x2f0555cc76fc2840a25a6ea3b9637146806f1f44b090c175ffde2a7e5ab36c03 \ + 0.10.1 647ca2b0493ed5c74399b73f2595643ba785c1b8 >/dev/null 2>&1; then echo "expected checksum/size mismatch to fail" >&2 exit 1 fi diff --git a/.github/workflows/refresh-mainnet-snapshot.yml b/.github/workflows/refresh-mainnet-snapshot.yml index 0e48e72a7a..cc98c77aa6 100644 --- a/.github/workflows/refresh-mainnet-snapshot.yml +++ b/.github/workflows/refresh-mainnet-snapshot.yml @@ -24,7 +24,7 @@ on: workflow_dispatch: inputs: try_runtime_only: - description: "Generate only try-runtime snapshots (measurement/debugging)" + description: "Generate only try-runtime snapshots (targeted recovery)" type: boolean default: false @@ -32,12 +32,16 @@ permissions: contents: read concurrency: + # Serialize every ref globally: diagnostic dispatches must not cancel a + # running production refresh or consume a second RPC/runner slot alongside it. group: refresh-mainnet-snapshot - cancel-in-progress: true + cancel-in-progress: false env: CARGO_TERM_COLOR: always TRY_RUNTIME_VERSION: "0.10.1" + # v0.10.1 exposes parallelism only through the number of URI values. + SNAPSHOT_RPC_CLIENTS: "4" jobs: snapshot: @@ -71,31 +75,11 @@ jobs: - name: Genesis-init the local clone database run: | - nohup ./clones/scripts/start-local-clone.sh > clone-node.log 2>&1 & # Genesis init from the ~2 GB chainspec takes several minutes before - # the RPC server comes up. - up=false - for _ in $(seq 1 450); do - if curl -sf -H "Content-Type: application/json" \ - -d '{"id":1,"jsonrpc":"2.0","method":"system_health","params":[]}' \ - http://127.0.0.1:9944 > /dev/null; then - up=true - break - fi - sleep 2 - done - if [ "$up" != true ]; then - echo "Clone node failed to start:" - cat clone-node.log - exit 1 - fi + # the RPC server comes up. Preserve the node's default consensus mode + # and only require RPC health before packaging. + ./clones/scripts/start-local-clone-and-wait.sh node-default ./clones/scripts/stop-local-clone.sh - # pkill returns before the node finishes flushing the db; wait for - # the process to actually exit before packaging. - for _ in $(seq 1 60); do - pgrep -f "node-subtensor.*--base-path clones/mainnet-clone" > /dev/null || break - sleep 2 - done - name: Package snapshot run: | @@ -120,13 +104,15 @@ jobs: # consumer pin in runtime-checks.yml. try-runtime-snapshot: name: try-runtime snapshot ${{ matrix.network.name }} - runs-on: [self-hosted, fireactions-tryruntime] + # Snapshot assembly exceeds the memory available on ubuntu-latest. This is + # scheduled, off-critical-path work, so use the shared 8-core pool instead + # of retaining a dedicated runner host. + runs-on: [self-hosted, fireactions-turbo-8] timeout-minutes: 90 strategy: fail-fast: false - # The shared pool has enough capacity for the three PR consumers. Keep - # the off-critical-path producer to one slot so a refresh cannot crowd - # those consumers out. + # Snapshot production is off the PR critical path. Serialize the network + # jobs to bound archive RPC load and shared-runner usage. max-parallel: 1 matrix: network: @@ -197,9 +183,20 @@ jobs: snapshot="${{ matrix.network.name }}.snap" manifest="${{ matrix.network.name }}.manifest.json" # The positional path must precede --uri because --uri is variadic. + # Each client creates four remote-externalities value-fetch workers; + # the archive endpoint is dedicated to this workload. + rpc_uri="${{ matrix.network.uri }}" + [[ "$SNAPSHOT_RPC_CLIENTS" =~ ^[1-9][0-9]*$ ]] || { + echo "invalid SNAPSHOT_RPC_CLIENTS: $SNAPSHOT_RPC_CLIENTS" >&2 + exit 2 + } + rpc_uris=() + for ((client = 0; client < SNAPSHOT_RPC_CLIENTS; client++)); do + rpc_uris+=("$rpc_uri") + done "$TRY_RUNTIME_BIN" create-snapshot \ "$snapshot" \ - --uri "${{ matrix.network.uri }}" \ + --uri "${rpc_uris[@]}" \ --at "${{ steps.source.outputs.block-hash }}" snapshot_size=$(stat -c '%s' "$snapshot") diff --git a/.github/workflows/release-train.yml b/.github/workflows/release-train.yml index 7869eb3bf5..ee96c6eda5 100644 --- a/.github/workflows/release-train.yml +++ b/.github/workflows/release-train.yml @@ -190,7 +190,7 @@ jobs: check-devnet: name: Smoke-check devnet needs: deploy-devnet - runs-on: [self-hosted, fireactions-tryruntime] + runs-on: ubuntu-latest timeout-minutes: 30 steps: - uses: actions/checkout@v4 @@ -278,7 +278,7 @@ jobs: check-testnet: name: Smoke-check testnet needs: deploy-testnet - runs-on: [self-hosted, fireactions-tryruntime] + runs-on: ubuntu-latest timeout-minutes: 30 steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/runtime-checks.yml b/.github/workflows/runtime-checks.yml index 33038f1d35..a5e9f100e6 100644 --- a/.github/workflows/runtime-checks.yml +++ b/.github/workflows/runtime-checks.yml @@ -16,19 +16,10 @@ on: description: "Bypass cached try-runtime snapshots and scrape all networks live" type: boolean default: false - snapshot_source_branch: - description: "Trusted snapshot artifact branch (manual measurement only)" - type: string - default: main try_runtime_only: description: "Run only the try-runtime build and cached network checks" type: boolean default: false - skip_devnet: - description: "Skip devnet when measuring a partially completed producer run" - type: boolean - default: false - concurrency: group: runtime-checks-${{ github.ref }} cancel-in-progress: true @@ -58,6 +49,7 @@ jobs: name: detect relevant changes runs-on: ubuntu-latest permissions: + contents: read pull-requests: read outputs: runtime: ${{ steps.filter.outputs.runtime }} @@ -66,6 +58,16 @@ jobs: sdk_drift: ${{ steps.filter.outputs.sdk_drift }} snapshot_ci: ${{ steps.filter.outputs.snapshot_ci }} steps: + - name: Checkout trusted path classifier + if: github.event_name == 'pull_request' + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.base.sha }} + sparse-checkout: .github/scripts/classify-runtime-changes.sh + sparse-checkout-cone-mode: false + path: .trusted-runtime-filter + persist-credentials: false + # Plain gh-api file listing instead of a marketplace action: the org's # Actions allowlist rejects unlisted third-party actions (startup_failure). - name: Filter changed paths @@ -74,30 +76,35 @@ jobs: env: GH_TOKEN: ${{ github.token }} PR_NUMBER: ${{ github.event.pull_request.number }} + CHANGED_FILES: ${{ github.event.pull_request.changed_files }} run: | set -euo pipefail - files=$(gh api "repos/$GITHUB_REPOSITORY/pulls/$PR_NUMBER/files" --paginate --jq '.[].filename') - # SDK-only changes are covered by sdk-checks and the Rust SDK e2e - # workflow; they should not force clone-upgrade or SDK drift. - runtime_pattern='^(common|node|pallets|precompiles|primitives|runtime|support|chain-extensions|src|vendor|clones)/|^(Cargo\.toml|build\.rs|rust-toolchain\.toml)$|^website/apps/bittensor-website/scripts/|^\.github/(workflows/(runtime-checks|refresh-mainnet-snapshot)\.yml|actions/(rust-setup|sccache-setup)/|scripts/(sccache-configure|snapshot-artifact|test-snapshot-artifact)\.sh$)' - docs_pattern='^website/|^sdk/python/|^\.github/workflows/runtime-checks\.yml$' - python_sdk_pattern='^sdk/(python|bittensor-core|bittensor-core-py|bittensor-core-wasm)/|^Cargo.lock$|^\.github/workflows/runtime-checks\.yml$' - sdk_drift_pattern='^(common|node|pallets|precompiles|primitives|runtime|support|chain-extensions|src|vendor)/|^(Cargo\.toml|build\.rs|rust-toolchain\.toml)$' - snapshot_pattern='^\.github/(workflows/(runtime-checks|refresh-mainnet-snapshot)\.yml|scripts/(snapshot-artifact|test-snapshot-artifact)\.sh)$' - runtime=false; docs=false; python_sdk=false; sdk_drift=false; snapshot_ci=false - grep -qE "$runtime_pattern" <<< "$files" && runtime=true - grep -qE "$docs_pattern" <<< "$files" && docs=true - grep -qE "$python_sdk_pattern" <<< "$files" && python_sdk=true - grep -qE "$sdk_drift_pattern" <<< "$files" && sdk_drift=true - grep -qE "$snapshot_pattern" <<< "$files" && snapshot_ci=true - echo "runtime=$runtime, docs=$docs, python_sdk=$python_sdk, sdk_drift=$sdk_drift, snapshot_ci=$snapshot_ci" - { - echo "runtime=$runtime" - echo "docs=$docs" - echo "python_sdk=$python_sdk" - echo "sdk_drift=$sdk_drift" - echo "snapshot_ci=$snapshot_ci" - } >> "$GITHUB_OUTPUT" + enable_all() { + { + echo "runtime=true" + echo "docs=true" + echo "python_sdk=true" + echo "sdk_drift=true" + echo "snapshot_ci=true" + } >> "$GITHUB_OUTPUT" + } + classifier=.trusted-runtime-filter/.github/scripts/classify-runtime-changes.sh + if [[ ! -f "$classifier" ]]; then + echo "::warning::Trusted base predates the path classifier; enabling every check." + enable_all + exit 0 + fi + pages=$(gh api "repos/$GITHUB_REPOSITORY/pulls/$PR_NUMBER/files" --paginate --slurp) + fetched_files=$(jq '[.[][]] | length' <<< "$pages") + if [[ "$fetched_files" -ne "$CHANGED_FILES" ]]; then + echo "::warning::PR file listing was incomplete ($fetched_files/$CHANGED_FILES); enabling every check." + enable_all + exit 0 + fi + # A rename can move a runtime file outside a matched directory, so + # classify both the current and previous paths. + jq -r '.[][] | .filename, (.previous_filename // empty)' <<< "$pages" \ + | bash "$classifier" "$GITHUB_OUTPUT" snapshot-artifact-tests: name: snapshot artifact contract tests @@ -107,6 +114,7 @@ jobs: steps: - uses: actions/checkout@v4 - run: .github/scripts/test-snapshot-artifact.sh + - run: .github/scripts/test-runtime-change-filter.sh # Build the try-runtime wasm independently so its consumers do not wait for # the unrelated release node build. The source check also makes the @@ -217,7 +225,8 @@ jobs: try-runtime: name: try-runtime ${{ matrix.network.name }} needs: build-try-runtime - runs-on: [self-hosted, fireactions-tryruntime] + runs-on: ubuntu-latest + timeout-minutes: 90 permissions: contents: read # Cross-run artifact download (the nightly try-runtime state snapshot). @@ -225,33 +234,21 @@ jobs: strategy: fail-fast: false matrix: - selected: [devnet, testnet, mainnet] - exclude: - - selected: ${{ github.event_name == 'workflow_dispatch' && inputs.skip_devnet && 'devnet' || '__none__' }} - include: + network: # NOTE: the hostnames are misleading — archive.dev.opentensor.ai # serves the TESTNET archive on :8443 and the MAINNET archive on # :443 (verified by genesis hash; try-runtime wants archive nodes, # which is why we don't use the entrypoint hostnames). The # "Verify endpoint" step below enforces each URI's identity. - - selected: devnet - network: - name: devnet - uri: wss://dev.chain.opentensor.ai:443 - genesis: "0x077899043eb684c5277b6814a39161f4ce072b45e782e12c81a521c63fb4f3e5" - - selected: testnet - network: - name: testnet - uri: wss://archive.dev.opentensor.ai:8443 - genesis: "0x8f9cf856bf558a14440e75569c9e58594757048d7b3a84b5d25f6bd978263105" - - selected: mainnet - network: - name: mainnet - uri: wss://archive.dev.opentensor.ai:443 - genesis: "0x2f0555cc76fc2840a25a6ea3b9637146806f1f44b090c175ffde2a7e5ab36c03" - # Keep endpoint-heavy checks near the archive RPCs. Measurements - # from BHS showed materially higher per-request latency, which - # compounds during try-runtime's state traversal. + - name: devnet + uri: wss://dev.chain.opentensor.ai:443 + genesis: "0x077899043eb684c5277b6814a39161f4ce072b45e782e12c81a521c63fb4f3e5" + - name: testnet + uri: wss://archive.dev.opentensor.ai:8443 + genesis: "0x8f9cf856bf558a14440e75569c9e58594757048d7b3a84b5d25f6bd978263105" + - name: mainnet + uri: wss://archive.dev.opentensor.ai:443 + genesis: "0x2f0555cc76fc2840a25a6ea3b9637146806f1f44b090c175ffde2a7e5ab36c03" steps: - uses: actions/checkout@v4 @@ -309,8 +306,9 @@ jobs: set -euo pipefail .github/scripts/snapshot-artifact.sh select \ "try-runtime-snap-v${TRY_RUNTIME_VERSION}-${{ matrix.network.name }}" \ - "${{ github.event_name == 'workflow_dispatch' && inputs.snapshot_source_branch || github.event.repository.default_branch }}" \ + "${{ github.event.repository.default_branch }}" \ "${{ github.repository_id }}" \ + .github/workflows/refresh-mainnet-snapshot.yml \ 72 "$GITHUB_OUTPUT" required - name: Mark snapshot restore start @@ -319,13 +317,14 @@ jobs: - name: Download selected try-runtime state snapshot if: steps.state-mode.outputs.fresh-state != 'true' - env: - GH_TOKEN: ${{ github.token }} - run: | - .github/scripts/snapshot-artifact.sh download \ - "${{ steps.snapshot.outputs.artifact-id }}" \ - "${{ steps.snapshot.outputs.digest }}" \ - snap + uses: actions/download-artifact@v4 + with: + artifact-ids: ${{ steps.snapshot.outputs.artifact-id }} + github-token: ${{ github.token }} + repository: ${{ github.repository }} + run-id: ${{ steps.snapshot.outputs.run-id }} + path: snap + merge-multiple: true - name: Validate try-runtime state snapshot if: steps.state-mode.outputs.fresh-state != 'true' @@ -336,7 +335,7 @@ jobs: .github/scripts/snapshot-artifact.sh validate \ "$manifest" "$snapshot" \ "${{ matrix.network.name }}" "${{ matrix.network.genesis }}" \ - "$TRY_RUNTIME_VERSION" + "$TRY_RUNTIME_VERSION" "${{ steps.snapshot.outputs.producer-sha }}" echo "TRY_RUNTIME_SNAP=$snapshot" >> "$GITHUB_ENV" restore_seconds=$(($(date -u +%s) - SNAPSHOT_RESTORE_STARTED)) block=$(jq -er '.finalized_block_number' "$manifest") @@ -458,12 +457,14 @@ jobs: # label is set). Auto-skips (via the `build-node-release` need) when the PR touches # nothing runtime-relevant. # - # The regression tests are wall-clock-bound (they wait on 12-second blocks) - # and mutate chain state, so they can't share one node concurrently. - # Instead each shard boots its own clone from the snapshot and runs a - # subset, preserving the suite's original relative order within each shard. + # Interval sealing advances runtime time by one 12-second slot every 250ms, + # so the block-driven regressions can run sequentially on one runner. The + # issuance-invariant test gets a pristine clone; the already-downloaded + # checkpoint is then restored locally for the remaining tests, before + # forceSetBalance-based fixtures make the issuance mirrors diverge. This + # avoids four runners and four downloads without sharing destructive state. clone-upgrade: - name: clone-upgrade (${{ matrix.shard.name }}) + name: clone-upgrade needs: [trusted-pr, changes, build-node-release] runs-on: [self-hosted, fireactions-turbo-8] if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip-clone-upgrade') }} @@ -472,35 +473,6 @@ jobs: contents: read # Cross-run artifact download (the nightly mainnet snapshot). actions: read - strategy: - fail-fast: false - matrix: - shard: - - name: balancer - tests: >- - test-balancer-operation.ts - test-balancer-edge-emission-issuance.ts - sdk: false - - name: locks - tests: >- - test-locks-conviction.ts - test-lock-dust-cleanup.ts - test-proxy-filter-security-regressions.ts - sdk: false - - name: stake-sdk - tests: >- - test-hotkey-swap-and-proxy-stake.ts - test-alpha-deprecated-stake-histogram.ts - sdk: true - # test-total-issuance-trackers legitimately waits ~6 min on chain - # state (see CLONE_REGRESSION_TIMEOUT_MS below), so the two - # emission/issuance tests get their own shard instead of - # serializing behind the stake-sdk pair. - - name: issuance - tests: >- - test-net-tao-flow-emission-allocation.ts - test-total-issuance-trackers.ts - sdk: false steps: - uses: actions/checkout@v4 @@ -528,21 +500,23 @@ jobs: set -euo pipefail .github/scripts/snapshot-artifact.sh select \ mainnet-snapshot \ - "${{ github.event_name == 'workflow_dispatch' && inputs.snapshot_source_branch || github.event.repository.default_branch }}" \ + "${{ github.event.repository.default_branch }}" \ "${{ github.repository_id }}" \ + .github/workflows/refresh-mainnet-snapshot.yml \ 168 "$GITHUB_OUTPUT" optional - name: Download selected mainnet clone snapshot id: download-clone-snapshot if: steps.clone-snapshot.outputs.found == 'true' continue-on-error: true - env: - GH_TOKEN: ${{ github.token }} - run: | - .github/scripts/snapshot-artifact.sh download \ - "${{ steps.clone-snapshot.outputs.artifact-id }}" \ - "${{ steps.clone-snapshot.outputs.digest }}" \ - /tmp/mainnet-snapshot + uses: actions/download-artifact@v4 + with: + artifact-ids: ${{ steps.clone-snapshot.outputs.artifact-id }} + github-token: ${{ github.token }} + repository: ${{ github.repository }} + run-id: ${{ steps.clone-snapshot.outputs.run-id }} + path: /tmp/mainnet-snapshot + merge-multiple: true - name: Restore mainnet clone snapshot id: restore-clone-snapshot @@ -550,12 +524,13 @@ jobs: continue-on-error: true run: | set -euo pipefail - tar -xzf /tmp/mainnet-snapshot/mainnet-snapshot.tar.gz - rm -rf /tmp/mainnet-snapshot + checkpoint=/tmp/mainnet-snapshot/mainnet-snapshot.tar.gz + ./clones/scripts/local-clone-checkpoint.sh restore "$checkpoint" echo "Restored artifact ${{ steps.clone-snapshot.outputs.artifact-id }} from run ${{ steps.clone-snapshot.outputs.run-id }}." # Tells start-local-clone.sh to keep the pre-initialized database # instead of wiping it and re-running genesis init. echo "KEEP_CLONE_DATA=1" >> "$GITHUB_ENV" + echo "CLONE_CHECKPOINT=$checkpoint" >> "$GITHUB_ENV" - name: Fall back after an unusable clone snapshot if: >- @@ -564,7 +539,8 @@ jobs: steps.restore-clone-snapshot.outcome != 'success') run: | echo "::warning::selected clone snapshot was unusable; falling back to a live mainnet scrape" - rm -rf /tmp/mainnet-snapshot clones/mainnet-clone clones/mainnet-clone-chainspec.json + ./clones/scripts/local-clone-checkpoint.sh clear + rm -rf /tmp/mainnet-snapshot - name: Fall back after clone snapshot lookup failure if: steps.clone-snapshot.outcome == 'failure' @@ -591,65 +567,76 @@ jobs: done exit 1 - - name: Start local clone + - name: Prepare a reusable checkpoint after live fallback + if: steps.restore-clone-snapshot.outcome != 'success' run: | - nohup ./clones/scripts/start-local-clone.sh > clone-node.log 2>&1 & - # Genesis init from the ~2 GB mainnet-clone chainspec takes several - # minutes before the RPC server comes up, so allow up to 15 minutes. - # (With a restored snapshot the db is pre-initialized and this is - # nearly instant.) - for i in $(seq 1 450); do - if curl -sf -H "Content-Type: application/json" \ - -d '{"id":1,"jsonrpc":"2.0","method":"system_health","params":[]}' \ - http://127.0.0.1:9944 > /dev/null; then - echo "Clone node is up." - exit 0 - fi - sleep 2 - done - echo "Clone node failed to start:" - cat clone-node.log - exit 1 + set -euo pipefail + ./clones/scripts/start-local-clone-and-wait.sh manual + checkpoint=/tmp/mainnet-clone-pristine.tar + ./clones/scripts/local-clone-checkpoint.sh create "$checkpoint" + echo "KEEP_CLONE_DATA=1" >> "$GITHUB_ENV" + echo "CLONE_CHECKPOINT=$checkpoint" >> "$GITHUB_ENV" + + - name: Install clone test dependencies + working-directory: clones/js-tests + run: npm ci + + - name: Start isolated issuance clone + run: ./clones/scripts/start-local-clone-and-wait.sh accelerated - - name: Sudo-upgrade clone with proposed runtime + - name: Sudo-upgrade issuance clone with proposed runtime working-directory: clones/js-tests run: | - npm ci # Right after genesis the node occasionally rejects the first - # extrinsic with "bad signature" (observed: 1 of 3 shards failed at - # +8s while the identical tx succeeded on the others seconds later). + # extrinsic with "bad signature" (observed in the former sharded + # workflow while identical transactions succeeded seconds later). # Setting the same runtime code twice is idempotent, so retry once. npm run runtime:update:alice || { sleep 15; npm run runtime:update:alice; } + - name: Run isolated issuance regression + working-directory: clones/js-tests + env: + CLONE_REGRESSION_PHASE: pristine + CLONE_REGRESSION_TIMEOUT_MS: 1800000 + run: npm run test:clone-regressions + + - name: Stop isolated issuance clone + run: ./clones/scripts/stop-local-clone.sh + + - name: Restore pristine clone for remaining regressions + run: ./clones/scripts/local-clone-checkpoint.sh restore "$CLONE_CHECKPOINT" + + - name: Start regression clone + run: ./clones/scripts/start-local-clone-and-wait.sh accelerated + + - name: Sudo-upgrade regression clone with proposed runtime + working-directory: clones/js-tests + run: npm run runtime:update:alice || { sleep 15; npm run runtime:update:alice; } + - name: Run clone smoke tests working-directory: clones/js-tests run: npm test - - name: Run clone regression tests (shard subset) + - name: Run remaining clone regression tests working-directory: clones/js-tests env: - CLONE_REGRESSION_TESTS: ${{ matrix.shard.tests }} - # test-total-issuance-trackers legitimately waits minutes for the - # queued replacement registration: register_network at the subnet - # limit prunes a subnet and only emits NetworkAdded from on_idle - # after the pruned subnet's chunked storage cleanup finishes - # (~6 min on mainnet-scale state). - CLONE_REGRESSION_TIMEOUT_MS: ${{ matrix.shard.name == 'issuance' && '1800000' || '900000' }} + CLONE_REGRESSION_PHASE: remaining + CLONE_REGRESSION_TIMEOUT_MS: 1800000 run: npm run test:clone-regressions - name: Install uv - if: ${{ matrix.shard.sdk }} + if: ${{ github.event_name != 'pull_request' || needs.changes.outputs.sdk_drift == 'true' }} run: | curl -LsSf https://astral.sh/uv/0.11.28/install.sh | sh echo "$HOME/.local/bin" >> $GITHUB_PATH - name: Sync SDK environment - if: ${{ matrix.shard.sdk }} + if: ${{ github.event_name != 'pull_request' || needs.changes.outputs.sdk_drift == 'true' }} working-directory: sdk/python run: uv sync --locked --all-extras --dev - name: Metadata drift gate (committed _generated vs upgraded clone) - if: ${{ matrix.shard.sdk && (github.event_name != 'pull_request' || needs.changes.outputs.sdk_drift == 'true') }} + if: ${{ github.event_name != 'pull_request' || needs.changes.outputs.sdk_drift == 'true' }} working-directory: sdk/python run: uv run python -m codegen.check --drift ${{ env.WS_ENDPOINT }} @@ -669,34 +656,33 @@ jobs: run: ./clones/scripts/stop-local-clone.sh # Branch protection requires a check named exactly "Sudo-upgrade mainnet - # clone and test"; the shard matrix above produces differently-named jobs, - # so this fan-in preserves the required-check name. Passes when every shard - # succeeded, or when the whole suite legitimately skipped (docs-only PR or - # the skip-clone-upgrade label). Anything else — including a build failure - # cascading into skipped shards — fails. + # clone and test"; this fan-in preserves the required-check name. It passes + # when the clone job succeeded, or when the whole suite legitimately skipped + # (docs-only PR or the skip-clone-upgrade label). Anything else — including + # a build failure cascading into a skipped clone job — fails. clone-upgrade-gate: name: Sudo-upgrade mainnet clone and test needs: [changes, build-node-release, clone-upgrade] if: always() runs-on: ubuntu-latest steps: - - name: Evaluate shard results + - name: Evaluate clone result env: CHANGES: ${{ needs.changes.result }} - SHARDS: ${{ needs.clone-upgrade.result }} + CLONE: ${{ needs.clone-upgrade.result }} BUILD: ${{ needs.build-node-release.result }} LABEL_SKIP: ${{ contains(github.event.pull_request.labels.*.name, 'skip-clone-upgrade') }} run: | - echo "changes=$CHANGES shards=$SHARDS build=$BUILD label-skip=$LABEL_SKIP" + echo "changes=$CHANGES clone=$CLONE build=$BUILD label-skip=$LABEL_SKIP" # A failed path-filter job must not read as "nothing runtime-related - # changed" — that would let build+shards skip and the gate pass. + # changed" — that would let build+clone skip and the gate pass. if [ "$CHANGES" != "success" ]; then exit 1 fi - if [ "$SHARDS" = "success" ]; then + if [ "$CLONE" = "success" ]; then exit 0 fi - if [ "$SHARDS" = "skipped" ] && { [ "$LABEL_SKIP" = "true" ] || [ "$BUILD" = "skipped" ]; }; then + if [ "$CLONE" = "skipped" ] && { [ "$LABEL_SKIP" = "true" ] || [ "$BUILD" = "skipped" ]; }; then exit 0 fi exit 1 diff --git a/.github/workflows/sccache-warm.yml b/.github/workflows/sccache-warm.yml index a30f0cc3e7..9ee10b4d21 100644 --- a/.github/workflows/sccache-warm.yml +++ b/.github/workflows/sccache-warm.yml @@ -2,7 +2,7 @@ name: Warm R2 sccache # Normal same-repository PR and main CI writes through while it compiles. This # workflow is maintenance only: refresh deployed-state branches when they move, -# repair expired/missing main entries at Amsterdam noon, or recover manually. +# repair expired/missing main entries at 12:00 UTC, or recover manually. on: push: @@ -11,7 +11,6 @@ on: - testnet schedule: - cron: "0 12 * * *" - timezone: "Europe/Amsterdam" workflow_dispatch: inputs: source_ref: diff --git a/.github/workflows/scheduled-smoke-tests.yml b/.github/workflows/scheduled-smoke-tests.yml index 7f436bb8d2..79b61ca9c5 100644 --- a/.github/workflows/scheduled-smoke-tests.yml +++ b/.github/workflows/scheduled-smoke-tests.yml @@ -1,12 +1,12 @@ name: Scheduled Smoke Tests -# Read-only smoke suites against the live networks, every 6 hours. The same +# Read-only smoke suites against the live networks, daily at 12:00 UTC. The same # suites gate promotion in release-train.yml; this catches drift between # trains. on: schedule: - - cron: "0 */6 * * *" + - cron: "0 12 * * *" workflow_dispatch: permissions: @@ -15,7 +15,10 @@ permissions: jobs: smoke: name: smoke-e2e-${{ matrix.test }} - runs-on: [self-hosted, fireactions-tryruntime] + # Periodic monitoring must not compete with state-heavy PR checks for the + # small try-runtime runner pool. This public repository can use standard + # GitHub-hosted Linux runners for these read-only public-RPC suites. + runs-on: ubuntu-latest timeout-minutes: 30 strategy: fail-fast: false diff --git a/.github/workflows/typescript-e2e.yml b/.github/workflows/typescript-e2e.yml index b46dbcb541..7328defcee 100644 --- a/.github/workflows/typescript-e2e.yml +++ b/.github/workflows/typescript-e2e.yml @@ -45,7 +45,7 @@ jobs: files=$(gh api "repos/$GITHUB_REPOSITORY/pulls/$PR_NUMBER/files" --paginate --jq '.[].filename') # SDK-only lockfile movement is covered by SDK checks; do not run # zombienet unless chain/runtime or ts-tests inputs changed. - pattern='^(ts-tests|common|node|pallets|precompiles|primitives|runtime|support|chain-extensions|src|vendor)/|^(Cargo\.toml|build\.rs|rust-toolchain\.toml)$|^\.github/workflows/typescript-e2e\.yml$' + pattern='^(ts-tests|common|node|pallets|precompiles|primitives|runtime|support|chain-extensions|src|vendor)/|^(Cargo\.toml|build\.rs|rust-toolchain\.toml)$|^\.github/(workflows/typescript-e2e\.yml|actions/run-typescript-e2e/)' if grep -qE "$pattern" <<< "$files"; then echo "e2e=true" >> "$GITHUB_OUTPUT" else @@ -69,12 +69,8 @@ jobs: with: node-version-file: ts-tests/.nvmrc - - name: Install system dependencies - run: | - sudo DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=a apt-get update - sudo DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=a apt-get install -y --no-install-recommends \ - -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" \ - build-essential clang curl git make libssl-dev llvm libudev-dev protobuf-compiler pkg-config + - name: Validate E2E configuration + run: node ts-tests/scripts/validate-e2e-config.mjs - name: Install e2e dependencies working-directory: ts-tests @@ -91,13 +87,13 @@ jobs: environment: name: sccache-writer deployment: false - needs: [trusted-pr, typescript-formatting, changes] + needs: [trusted-pr, changes] if: needs.changes.outputs.e2e == 'true' timeout-minutes: 60 strategy: matrix: include: - - variant: release + - variant: release flags: "" - variant: fast flags: "--features fast-runtime" @@ -141,6 +137,10 @@ jobs: path: target/release/node-subtensor if-no-files-found: error + # State-focused suites do not exercise multi-validator consensus. They use a + # single immediately-finalized node; Shield retains the production-like + # multi-node topology below. Two workers here plus two Shield workers cap the + # E2E phase at four self-hosted runners without extending its critical path. run-e2e-tests: needs: [trusted-pr, build] runs-on: [self-hosted, fireactions-turbo-8] @@ -148,20 +148,19 @@ jobs: strategy: fail-fast: false + max-parallel: 2 matrix: include: - - test: dev - binary: release - - test: zombienet_shield - binary: release + - test: zombienet_evm + binary: fast - test: zombienet_staking binary: fast - test: zombienet_coldkey_swap binary: fast + - test: dev + binary: release - test: zombienet_subnets binary: fast - - test: zombienet_evm - binary: fast name: "typescript-e2e-${{ matrix.test }}" @@ -169,36 +168,47 @@ jobs: - name: Check-out repository uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - - name: Download binary - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + - name: Run E2E suite + uses: ./.github/actions/run-typescript-e2e with: - name: node-subtensor-${{ matrix.binary }} - path: target/release + binary: ${{ matrix.binary }} + test: ${{ matrix.test }} - - name: Make binary executable - run: chmod +x target/release/node-subtensor - - - name: Setup Node.js - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 - with: - node-version-file: ts-tests/.nvmrc + # Shield validates consensus, key rotation, timing, and transaction + # mortality, so preserve its six-node release topology and parallelize only + # by running balanced file groups on independent networks. + run-shield-tests: + needs: [trusted-pr, build] + runs-on: [self-hosted, fireactions-turbo-8] + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + test: [zombienet_shield_a, zombienet_shield_b] + name: "typescript-e2e-${{ matrix.test }}" + steps: + - name: Check-out repository + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - - name: Setup pnpm - uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4 + - name: Run Shield shard + uses: ./.github/actions/run-typescript-e2e with: - version: 10 - - - name: Install e2e dependencies - working-directory: ts-tests - run: pnpm install --frozen-lockfile - - - name: Install lsof (dev foundation RPC port discovery) - if: matrix.test == 'dev' - run: | - sudo DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=a apt-get update - sudo DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=a apt-get install -y --no-install-recommends lsof + binary: release + test: ${{ matrix.test }} - - name: Run tests + shield-result: + name: typescript-e2e-zombienet_shield + if: always() + needs: [build, run-shield-tests] + runs-on: ubuntu-latest + steps: + - name: Check Shield shard results + env: + BUILD_RESULT: ${{ needs.build.result }} + SHIELD_RESULT: ${{ needs.run-shield-tests.result }} run: | - cd ts-tests - pnpm moonwall test ${{ matrix.test }} + if [[ "$BUILD_RESULT" == "failure" || "$BUILD_RESULT" == "cancelled" || \ + "$SHIELD_RESULT" == "failure" || "$SHIELD_RESULT" == "cancelled" ]]; then + echo "Shield E2E prerequisites failed: build=$BUILD_RESULT shards=$SHIELD_RESULT" >&2 + exit 1 + fi diff --git a/clones/js-tests/scripts/run-clone-regressions.ts b/clones/js-tests/scripts/run-clone-regressions.ts index 1328bb2a3c..f0205022e3 100644 --- a/clones/js-tests/scripts/run-clone-regressions.ts +++ b/clones/js-tests/scripts/run-clone-regressions.ts @@ -14,35 +14,63 @@ const testsDir = path.join(__dirname, "..", "tests"); // beats burning half an hour of runner time per stuck test. const TEST_TIMEOUT_MS = Number(process.env.CLONE_REGRESSION_TIMEOUT_MS ?? 15 * 60 * 1000); +const CLONE_REGRESSION_PHASES = ["pristine", "remaining"] as const; +type CloneRegressionPhase = (typeof CLONE_REGRESSION_PHASES)[number]; + const CLONE_REGRESSIONS = [ - "test-balancer-operation.ts", - "test-balancer-edge-emission-issuance.ts", - "test-locks-conviction.ts", - "test-lock-dust-cleanup.ts", - "test-proxy-filter-security-regressions.ts", - "test-hotkey-swap-and-proxy-stake.ts", - "test-net-tao-flow-emission-allocation.ts", - "test-total-issuance-trackers.ts", - "test-alpha-deprecated-stake-histogram.ts", -]; + // This test asserts the global issuance mirrors before and after each + // scenario, so it owns the pristine phase before forceSetBalance-based tests + // mutate the clone. + { name: "test-total-issuance-trackers.ts", phase: "pristine" }, + { name: "test-balancer-operation.ts", phase: "remaining" }, + { name: "test-balancer-edge-emission-issuance.ts", phase: "remaining" }, + { name: "test-locks-conviction.ts", phase: "remaining" }, + { name: "test-lock-dust-cleanup.ts", phase: "remaining" }, + { name: "test-proxy-filter-security-regressions.ts", phase: "remaining" }, + { name: "test-hotkey-swap-and-proxy-stake.ts", phase: "remaining" }, + { name: "test-net-tao-flow-emission-allocation.ts", phase: "remaining" }, + { name: "test-alpha-deprecated-stake-histogram.ts", phase: "remaining" }, +] as const satisfies ReadonlyArray<{ name: string; phase: CloneRegressionPhase }>; + +const regressionNames = new Set(CLONE_REGRESSIONS.map(({ name }) => name)); +const regressionPhases = new Set(CLONE_REGRESSION_PHASES); -// CI shards the suite across parallel clones; CLONE_REGRESSION_TESTS selects -// a subset (whitespace/comma separated). Unknown names fail loudly so a typo -// in a shard definition can't silently skip a regression test. +// CI selects a named phase, keeping suite membership canonical here. Explicit +// filename selection remains available for targeted local/debug runs. +const requestedPhase = (process.env.CLONE_REGRESSION_PHASE ?? "").trim(); const requested = (process.env.CLONE_REGRESSION_TESTS ?? "").split(/[\s,]+/).filter(Boolean); -const unknown = requested.filter((name) => !CLONE_REGRESSIONS.includes(name)); +const cliArgs = process.argv.slice(2); +const unknownArgs = cliArgs.filter((arg) => arg !== "--list"); +if (unknownArgs.length > 0) { + console.error(`Unknown argument(s): ${unknownArgs.join(", ")}`); + process.exit(1); +} +if (requestedPhase && requested.length > 0) { + console.error("Set either CLONE_REGRESSION_PHASE or CLONE_REGRESSION_TESTS, not both"); + process.exit(1); +} +if (requestedPhase && !regressionPhases.has(requestedPhase)) { + console.error(`Unknown regression phase: ${requestedPhase}`); + process.exit(1); +} +const unknown = requested.filter((name) => !regressionNames.has(name)); if (unknown.length > 0) { console.error(`Unknown regression test(s): ${unknown.join(", ")}`); process.exit(1); } const selected = requested.length > 0 - ? CLONE_REGRESSIONS.filter((name) => requested.includes(name)) - : CLONE_REGRESSIONS; + ? CLONE_REGRESSIONS.filter(({ name }) => requested.includes(name)) + : requestedPhase + ? CLONE_REGRESSIONS.filter(({ phase }) => phase === requestedPhase) + : CLONE_REGRESSIONS; -let failed = false; +if (cliArgs.includes("--list")) { + console.log(selected.map(({ name }) => name).join("\n")); + process.exit(0); +} -for (const name of selected) { +for (const { name } of selected) { const script = path.join(testsDir, name); console.log(`\n=== clone regression: ${name} (timeout ${TEST_TIMEOUT_MS}ms) ===`); const result = spawnSync(process.execPath, ["--import", "tsx", script], { @@ -51,14 +79,11 @@ for (const name of selected) { timeout: TEST_TIMEOUT_MS, }); if ((result.error as NodeJS.ErrnoException | undefined)?.code === "ETIMEDOUT") { - failed = true; console.error(`TIMEOUT: ${name} exceeded ${TEST_TIMEOUT_MS}ms`); - continue; + process.exit(1); } if (result.status !== 0) { - failed = true; console.error(`FAILED: ${name}`); + process.exit(1); } } - -process.exit(failed ? 1 : 0); diff --git a/clones/js-tests/tests/test-hotkey-swap-and-proxy-stake.ts b/clones/js-tests/tests/test-hotkey-swap-and-proxy-stake.ts index 1ce7ab8954..94282212f5 100644 --- a/clones/js-tests/tests/test-hotkey-swap-and-proxy-stake.ts +++ b/clones/js-tests/tests/test-hotkey-swap-and-proxy-stake.ts @@ -7,6 +7,7 @@ import { connectApi } from "../lib/api.js"; import { createTempLogger } from "../lib/file-log.js"; const WS_ENDPOINT = process.env.WS_ENDPOINT ?? "ws://127.0.0.1:9944"; +const BLOCK_PRODUCTION_POLL_MS = 1_000; const RUN_ID = process.env.HOTKEY_PROXY_RUN_ID ?? `run${Date.now()}p${process.pid}`; const FUND_SOURCE_URI = process.env.HOTKEY_PROXY_FUND_SOURCE_URI ?? "//Alice"; const FUND_AMOUNT = BigInt(process.env.HOTKEY_PROXY_FUND_AMOUNT ?? "5000000000000"); @@ -104,7 +105,7 @@ async function waitForBlockProduction() { const deadline = Date.now() + 180_000; while (Date.now() < deadline && observed.length < 2) { - await sleep(6_000); + await sleep(BLOCK_PRODUCTION_POLL_MS); const current = (await api.rpc.chain.getHeader()).number.toBigInt(); if (current > previous) { observed.push(current); diff --git a/clones/js-tests/tests/test-proxy-filter-security-regressions.ts b/clones/js-tests/tests/test-proxy-filter-security-regressions.ts index 2e3768d46e..1736632b13 100644 --- a/clones/js-tests/tests/test-proxy-filter-security-regressions.ts +++ b/clones/js-tests/tests/test-proxy-filter-security-regressions.ts @@ -6,6 +6,7 @@ import { connectApi } from "../lib/api.js"; import { createTempLogger } from "../lib/file-log.js"; const WS_ENDPOINT = process.env.WS_ENDPOINT ?? "ws://127.0.0.1:9944"; +const BLOCK_PRODUCTION_POLL_MS = 1_000; const RUN_ID = process.env.PROXY_FILTER_RUN_ID ?? `run${Date.now()}p${process.pid}`; const FUND_SOURCE_URI = process.env.PROXY_FILTER_FUND_SOURCE_URI ?? "//Alice"; const FUND_AMOUNT = BigInt(process.env.PROXY_FILTER_FUND_AMOUNT ?? "5000000000000"); @@ -115,7 +116,7 @@ async function waitForBlockProduction() { const deadline = Date.now() + 180_000; while (Date.now() < deadline && observed.length < 2) { - await sleep(6_000); + await sleep(BLOCK_PRODUCTION_POLL_MS); const current = (await api.rpc.chain.getHeader()).number.toBigInt(); if (current > previous) { observed.push(current); diff --git a/clones/js-tests/tests/test-total-issuance-trackers.ts b/clones/js-tests/tests/test-total-issuance-trackers.ts index 59f6f91e16..4b92eda849 100644 --- a/clones/js-tests/tests/test-total-issuance-trackers.ts +++ b/clones/js-tests/tests/test-total-issuance-trackers.ts @@ -53,7 +53,7 @@ async function main() { console.log("run id:", RUN_ID); assertMetadataAvailable(); - await repairIssuanceMirrorIfNeeded("pre-test setup"); + await assertIssuanceMatch("pre-test setup"); await fundTestAccounts(); await assertIssuanceMatch("initial"); @@ -408,30 +408,6 @@ async function ensureEvmWhitelistDisabled() { console.log("EVM whitelist check disabled"); } -async function repairIssuanceMirrorIfNeeded(label) { - for (let attempt = 1; attempt <= 3; attempt++) { - const balances = (await api.query.balances.totalIssuance()).toBigInt(); - const subtensor: bigint = (await api.query.subtensorModule.totalIssuance()).toBigInt(); - const diff = balances - subtensor; - if (diff === 0n) { - console.log(`${label}: issuance matched`, balances.toString()); - return; - } - - const target = balances + diff; - assert.ok(target > 0n, `cannot repair issuance mirror: computed target ${target}`); - await submitAndWait( - fundSource, - api.tx.sudo.sudo(api.tx.system.setStorage([ - [api.query.subtensorModule.totalIssuance.key(), storageValueHex("u64", target)], - ])), - `sudo repair Subtensor TotalIssuance mirror attempt ${attempt}` - ); - } - - await assertIssuanceMatch(`${label} repaired`); -} - async function assertIssuanceMatch(label) { const [balancesIssuance, subtensorIssuance] = await Promise.all([ api.query.balances.totalIssuance(), diff --git a/clones/scripts/local-clone-checkpoint.sh b/clones/scripts/local-clone-checkpoint.sh new file mode 100755 index 0000000000..6a09322665 --- /dev/null +++ b/clones/scripts/local-clone-checkpoint.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd -- "${SCRIPT_DIR}/../.." && pwd)" +CLONE_DIR="clones/mainnet-clone" +CHAIN_SPEC="clones/mainnet-clone-chainspec.json" + +usage() { + echo "usage: local-clone-checkpoint.sh create ARCHIVE | restore ARCHIVE | clear" >&2 + exit 2 +} + +stop_clone() { + "$SCRIPT_DIR/stop-local-clone.sh" +} + +clear_clone() { + rm -rf "$CLONE_DIR" "$CHAIN_SPEC" +} + +cd "$REPO_ROOT" +command="${1:-}" +shift || true + +case "$command" in + create) + [[ $# -eq 1 ]] || usage + archive="$1" + [[ -d "$CLONE_DIR" ]] || { echo "missing clone directory: $CLONE_DIR" >&2; exit 1; } + [[ -f "$CHAIN_SPEC" ]] || { echo "missing clone chainspec: $CHAIN_SPEC" >&2; exit 1; } + stop_clone + tar -cf "$archive" "$CHAIN_SPEC" "$CLONE_DIR" + echo "Created clone checkpoint $archive." + ;; + restore) + [[ $# -eq 1 ]] || usage + archive="$1" + [[ -f "$archive" ]] || { echo "missing clone checkpoint: $archive" >&2; exit 1; } + stop_clone + clear_clone + # GNU tar detects gzip automatically while reading, so callers do not need + # to carry the producer's archive format through the workflow. + tar -xf "$archive" + [[ -d "$CLONE_DIR" ]] || { echo "checkpoint did not contain $CLONE_DIR" >&2; exit 1; } + [[ -f "$CHAIN_SPEC" ]] || { echo "checkpoint did not contain $CHAIN_SPEC" >&2; exit 1; } + echo "Restored clone checkpoint $archive." + ;; + clear) + [[ $# -eq 0 ]] || usage + stop_clone + clear_clone + ;; + *) usage ;; +esac diff --git a/clones/scripts/start-local-clone-and-wait.sh b/clones/scripts/start-local-clone-and-wait.sh new file mode 100755 index 0000000000..c1e03108cd --- /dev/null +++ b/clones/scripts/start-local-clone-and-wait.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd -- "${SCRIPT_DIR}/../.." && pwd)" + +log_file="clone-node.log" +ready_attempts=450 +advance_timeout_seconds=60 + +usage() { + cat >&2 <<'EOF' +Usage: start-local-clone-and-wait.sh MODE + +Modes: + accelerated Seal every 250ms and require 20 blocks of advancement + manual Use manual sealing and require RPC health only + node-default Preserve the node's default sealing and require RPC health only +EOF + exit 2 +} + +[[ $# -eq 1 ]] || usage +mode="$1" + +cd "$REPO_ROOT" +case "$mode" in + accelerated) + nohup ./clones/scripts/start-local-clone.sh --sealing 250 > "$log_file" 2>&1 & + ;; + manual) + nohup ./clones/scripts/start-local-clone.sh --sealing manual > "$log_file" 2>&1 & + ;; + node-default) + nohup ./clones/scripts/start-local-clone.sh > "$log_file" 2>&1 & + ;; + *) usage ;; +esac +node_pid=$! + +cleanup_on_failure() { + local status=$? + (( status != 0 )) || return + echo "Clone node startup failed; last log lines:" + tail -n 200 "$log_file" 2>/dev/null || true + "$SCRIPT_DIR/stop-local-clone.sh" || true +} +trap cleanup_on_failure EXIT + +rpc() { + local method="$1" + curl -fsS -H "Content-Type: application/json" \ + -d "{\"id\":1,\"jsonrpc\":\"2.0\",\"method\":\"$method\",\"params\":[]}" \ + http://127.0.0.1:9944 +} + +ready=false +for ((attempt = 1; attempt <= ready_attempts; attempt++)); do + if rpc system_health > /dev/null; then + ready=true + break + fi + kill -0 "$node_pid" 2>/dev/null || break + sleep 2 +done +[[ "$ready" == true ]] || exit 1 + +if [[ "$mode" != accelerated ]]; then + echo "Clone node is healthy." + trap - EXIT + exit 0 +fi + +height() { + local hex + hex=$(rpc chain_getHeader | jq -er '.result.number') + echo $((16#${hex#0x})) +} + +first=$(height) +deadline=$(($(date +%s) + advance_timeout_seconds)) +while (( $(date +%s) < deadline )); do + current=$(height) + if (( current >= first + 20 )); then + echo "Clone advanced from block $first to $current." + trap - EXIT + exit 0 + fi + sleep 1 +done + +exit 1 diff --git a/clones/scripts/stop-local-clone.sh b/clones/scripts/stop-local-clone.sh index 1b8f47937f..8abac74a91 100755 --- a/clones/scripts/stop-local-clone.sh +++ b/clones/scripts/stop-local-clone.sh @@ -1,6 +1,38 @@ #!/usr/bin/env bash set -euo pipefail -pkill -f "node-subtensor.*--base-path clones/mainnet-clone" \ - || pkill -f "node-subtensor.*--base-path ../clones/mainnet-clone" \ - || true +timeout_seconds="${CLONE_STOP_TIMEOUT_SECONDS:-60}" +[[ "$timeout_seconds" =~ ^[0-9]+$ ]] || { + echo "invalid CLONE_STOP_TIMEOUT_SECONDS: $timeout_seconds" >&2 + exit 2 +} + +patterns=( + "node-subtensor.*--base-path clones/mainnet-clone" + "node-subtensor.*--base-path \.\./clones/mainnet-clone" +) + +clone_running() { + local pattern + for pattern in "${patterns[@]}"; do + pgrep -f "$pattern" > /dev/null && return 0 + done + return 1 +} + +for pattern in "${patterns[@]}"; do + pkill -f "$pattern" || true +done + +deadline=$(($(date +%s) + timeout_seconds)) +while clone_running && (( $(date +%s) < deadline )); do + sleep 1 +done + +if clone_running; then + echo "Clone node did not stop within ${timeout_seconds}s." >&2 + for pattern in "${patterns[@]}"; do + pgrep -af "$pattern" >&2 || true + done + exit 1 +fi diff --git a/node/src/service.rs b/node/src/service.rs index 624f63b968..f7d277eed2 100644 --- a/node/src/service.rs +++ b/node/src/service.rs @@ -24,9 +24,9 @@ use sp_runtime::key_types; use sp_runtime::traits::{Block as BlockT, NumberFor}; use stc_shield::{self, MemoryShieldKeystore}; use std::collections::HashSet; +use std::path::Path; use std::str::FromStr; -use std::sync::atomic::AtomicBool; -use std::{cell::RefCell, path::Path}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::{sync::Arc, time::Duration}; use stp_shield::ShieldKeystorePtr; use substrate_prometheus_endpoint::Registry; @@ -758,11 +758,20 @@ fn run_manual_seal_authorship( shield_keystore.clone(), ); - thread_local!(static TIMESTAMP: RefCell = const { RefCell::new(0) }); - - /// Provide a mock duration starting at 0 in millisecond for timestamp inherent. - /// Each call will increment timestamp by slot_duration making the consensus logic think time has passed. - struct MockTimestampInherentDataProvider; + let timestamp = Arc::new(AtomicU64::new( + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_millis() + .min(u64::MAX as u128) as u64, + )); + + /// Provide a synthetic timestamp starting after the current wall-clock slot. + /// Each call advances by one slot, allowing interval sealing to run faster + /// than real time while remaining ahead of cloned consensus state. + struct MockTimestampInherentDataProvider { + timestamp: Arc, + } #[async_trait::async_trait] impl sp_inherents::InherentDataProvider for MockTimestampInherentDataProvider { @@ -770,11 +779,15 @@ fn run_manual_seal_authorship( &self, inherent_data: &mut sp_inherents::InherentData, ) -> Result<(), sp_inherents::Error> { - TIMESTAMP.with(|x| { - let mut x_ref = x.borrow_mut(); - *x_ref = x_ref.saturating_add(subtensor_runtime_common::time::SLOT_DURATION); - inherent_data.put_data(sp_timestamp::INHERENT_IDENTIFIER, &*x_ref) - }) + let slot_duration = subtensor_runtime_common::time::SLOT_DURATION; + let timestamp = self + .timestamp + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| { + Some(current.saturating_add(slot_duration)) + }) + .map(|previous| previous.saturating_add(slot_duration)) + .unwrap_or(u64::MAX); + inherent_data.put_data(sp_timestamp::INHERENT_IDENTIFIER, ×tamp) } async fn try_handle_error( @@ -789,9 +802,10 @@ fn run_manual_seal_authorship( let create_inherent_data_providers = move |_, ()| { let keystore = shield_keystore.clone(); + let timestamp = timestamp.clone(); async move { Ok(( - MockTimestampInherentDataProvider, + MockTimestampInherentDataProvider { timestamp }, stc_shield::InherentDataProvider::new(keystore), )) } diff --git a/ts-tests/configs/zombie_node.json b/ts-tests/configs/zombie_single_node.json similarity index 72% rename from ts-tests/configs/zombie_node.json rename to ts-tests/configs/zombie_single_node.json index f073be80bd..e37ab34595 100644 --- a/ts-tests/configs/zombie_node.json +++ b/ts-tests/configs/zombie_single_node.json @@ -7,6 +7,7 @@ "chain_spec_path": "specs/chain-spec.json", "default_command": "../target/release/node-subtensor", "default_args": [ + "--sealing=100" ], "genesis": { "runtimeGenesis": { @@ -22,17 +23,7 @@ "nodes": [ { "name": "one", - "rpc_port": "9947", - "validator": true - }, - { - "name": "two", - "rpc_port": "9948", - "validator": true - }, - { - "name": "three", - "rpc_port": "9949", + "rpc_port": 9947, "validator": true } ] @@ -44,4 +35,4 @@ "post_state": "Hash" } } -} \ No newline at end of file +} diff --git a/ts-tests/moonwall.config.json b/ts-tests/moonwall.config.json index 951661ef9e..2b5a5072f2 100644 --- a/ts-tests/moonwall.config.json +++ b/ts-tests/moonwall.config.json @@ -52,7 +52,7 @@ "foundation": { "type": "zombie", "zombieSpec": { - "configPath": "./configs/zombie_node.json", + "configPath": "./configs/zombie_single_node.json", "skipBlockCheck": [] } }, @@ -99,6 +99,82 @@ } ] }, + { + "name": "zombienet_shield_a", + "timeout": 600000, + "testFileDir": ["suites/zombienet_shield"], + "include": [ + "suites/zombienet_shield/00.00-basic.test.ts", + "suites/zombienet_shield/00.01-basic.test.ts", + "suites/zombienet_shield/02-edge-cases.test.ts" + ], + "runScripts": [ + "generate-types.sh", + "build-spec.sh" + ], + "foundation": { + "type": "zombie", + "zombieSpec": { + "configPath": "./configs/zombie_extended.json", + "skipBlockCheck": [] + } + }, + "vitestArgs": { + "bail": 1 + }, + "connections": [ + { + "name": "Node", + "type": "papi", + "endpoints": ["ws://127.0.0.1:9947"], + "descriptor": "subtensor" + }, + { + "name": "NodeFull", + "type": "papi", + "endpoints": ["ws://127.0.0.1:9950"], + "descriptor": "subtensor" + } + ] + }, + { + "name": "zombienet_shield_b", + "timeout": 600000, + "testFileDir": ["suites/zombienet_shield"], + "include": [ + "suites/zombienet_shield/01-scaling.test.ts", + "suites/zombienet_shield/03-timing.test.ts", + "suites/zombienet_shield/04-mortality.test.ts" + ], + "runScripts": [ + "generate-types.sh", + "build-spec.sh" + ], + "foundation": { + "type": "zombie", + "zombieSpec": { + "configPath": "./configs/zombie_extended.json", + "skipBlockCheck": [] + } + }, + "vitestArgs": { + "bail": 1 + }, + "connections": [ + { + "name": "Node", + "type": "papi", + "endpoints": ["ws://127.0.0.1:9947"], + "descriptor": "subtensor" + }, + { + "name": "NodeFull", + "type": "papi", + "endpoints": ["ws://127.0.0.1:9950"], + "descriptor": "subtensor" + } + ] + }, { "name": "zombienet_coldkey_swap", "timeout": 600000, @@ -110,7 +186,7 @@ "foundation": { "type": "zombie", "zombieSpec": { - "configPath": "./configs/zombie_node.json", + "configPath": "./configs/zombie_single_node.json", "skipBlockCheck": [] } }, @@ -136,7 +212,7 @@ "foundation": { "type": "zombie", "zombieSpec": { - "configPath": "./configs/zombie_node.json", + "configPath": "./configs/zombie_single_node.json", "skipBlockCheck": [] } }, @@ -168,7 +244,7 @@ "foundation": { "type": "zombie", "zombieSpec": { - "configPath": "./configs/zombie_node.json", + "configPath": "./configs/zombie_single_node.json", "skipBlockCheck": [] } }, diff --git a/ts-tests/scripts/validate-e2e-config.mjs b/ts-tests/scripts/validate-e2e-config.mjs new file mode 100644 index 0000000000..505558085a --- /dev/null +++ b/ts-tests/scripts/validate-e2e-config.mjs @@ -0,0 +1,85 @@ +import { readdirSync, readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const tsTestsDir = join(dirname(fileURLToPath(import.meta.url)), ".."); +const config = JSON.parse(readFileSync(join(tsTestsDir, "moonwall.config.json"), "utf8")); +const singleNodeSpec = JSON.parse(readFileSync(join(tsTestsDir, "configs/zombie_single_node.json"), "utf8")); +const shieldSpec = JSON.parse(readFileSync(join(tsTestsDir, "configs/zombie_extended.json"), "utf8")); +const environments = new Map(config.environments.map((environment) => [environment.name, environment])); + +const shieldFiles = readdirSync(join(tsTestsDir, "suites/zombienet_shield")) + .filter((file) => file.endsWith(".test.ts")) + .map((file) => `suites/zombienet_shield/${file}`) + .sort(); +const shieldShardNames = ["zombienet_shield_a", "zombienet_shield_b"]; +const shieldShardIncludes = shieldShardNames.map((name) => { + const environment = environments.get(name); + if (!environment) { + throw new Error(`Missing Moonwall environment: ${name}`); + } + const includes = environment.include ?? []; + if (includes.length === 0) { + throw new Error(`${name} must include at least one Shield test`); + } + return includes; +}); +const shardSizes = shieldShardIncludes.map((includes) => includes.length); +if (Math.max(...shardSizes) - Math.min(...shardSizes) > 1) { + throw new Error(`Shield shards must stay balanced; found sizes ${shardSizes.join(" and ")}`); +} +const shieldIncludes = shieldShardIncludes.flat(); + +const duplicateShieldFiles = shieldIncludes.filter((file, index) => shieldIncludes.indexOf(file) !== index); +if (duplicateShieldFiles.length > 0) { + throw new Error(`Shield tests assigned to multiple shards: ${[...new Set(duplicateShieldFiles)].join(", ")}`); +} + +const sortedIncludes = [...shieldIncludes].sort(); +if (JSON.stringify(sortedIncludes) !== JSON.stringify(shieldFiles)) { + const missing = shieldFiles.filter((file) => !sortedIncludes.includes(file)); + const unknown = sortedIncludes.filter((file) => !shieldFiles.includes(file)); + throw new Error(`Shield shard coverage mismatch; missing=[${missing.join(", ")}] unknown=[${unknown.join(", ")}]`); +} + +const singleNodeConfig = "./configs/zombie_single_node.json"; +const singleNodes = singleNodeSpec.relaychain?.nodes ?? []; +if ( + singleNodes.length !== 1 || + singleNodes[0]?.validator !== true || + !singleNodeSpec.relaychain?.default_args?.includes("--sealing=100") +) { + throw new Error("Single-node state spec must contain one validator using --sealing=100"); +} + +for (const name of ["zombienet_staking", "zombienet_coldkey_swap", "zombienet_evm", "zombienet_subnets"]) { + const configPath = environments.get(name)?.foundation?.zombieSpec?.configPath; + if (configPath !== singleNodeConfig) { + throw new Error(`${name} must use ${singleNodeConfig}; found ${configPath ?? "no config"}`); + } +} + +const shieldConfig = "./configs/zombie_extended.json"; +const shieldNodes = shieldSpec.relaychain?.nodes ?? []; +const shieldValidators = shieldNodes.filter((node) => node.validator === true); +if (shieldNodes.length !== 6 || shieldValidators.length !== 3) { + throw new Error( + `Shield spec must retain six nodes and three validators; found ${shieldNodes.length} nodes and ${shieldValidators.length} validators` + ); +} + +for (const name of ["zombienet_shield", "zombienet_shield_a", "zombienet_shield_b"]) { + const environment = environments.get(name); + const configPath = environment?.foundation?.zombieSpec?.configPath; + const connectionNames = new Set((environment?.connections ?? []).map((connection) => connection.name)); + if (configPath !== shieldConfig) { + throw new Error(`${name} must use ${shieldConfig}; found ${configPath ?? "no config"}`); + } + if (!connectionNames.has("Node") || !connectionNames.has("NodeFull")) { + throw new Error(`${name} must expose authority and full-node connections`); + } +} + +console.log( + `Validated ${shieldFiles.length} Shield files, three multi-node Shield environments, and four single-node state suites.` +); diff --git a/ts-tests/suites/zombienet_evm/00-evm-substrate-transfer.test.ts b/ts-tests/suites/zombienet_evm/00-evm-substrate-transfer.test.ts index a21b61c777..e10967f1ce 100644 --- a/ts-tests/suites/zombienet_evm/00-evm-substrate-transfer.test.ts +++ b/ts-tests/suites/zombienet_evm/00-evm-substrate-transfer.test.ts @@ -24,6 +24,7 @@ import { ss58ToEthAddress, ss58ToH160, tao, + waitForEthBalance, waitForFinalizedBlocks, waitForTransactionWithRetry, WITHDRAW_CONTRACT_ABI, @@ -48,7 +49,7 @@ function expectWithinTxFee(actual: bigint, expected: bigint): void { async function transferAndGetFee( wallet: ethers.Wallet, wallet2: ethers.Wallet, - provider: ethers.Provider, + provider: ethers.JsonRpcProvider, maxFeePerGas: bigint, maxPriorityFeePerGas: bigint ): Promise { @@ -137,7 +138,11 @@ describeSuite({ }); await waitForTransactionWithRetry(api, tx, signer, "substrate_to_evm"); - const receiverBalanceAfter = await getEthBalance(provider, ethWallet.address); + const receiverBalanceAfter = await waitForEthBalance( + provider, + ethWallet.address, + receiverBalance + raoToEth(transferBalance) + ); expect(receiverBalanceAfter).toEqual(receiverBalance + raoToEth(transferBalance)); }, }); @@ -226,7 +231,11 @@ describeSuite({ await waitForTransactionWithRetry(api, tx, signer, "evm_call"); - const receiverBalanceAfterCall = await getEthBalance(provider, ethWallet.address); + const receiverBalanceAfterCall = await waitForEthBalance( + provider, + ethWallet.address, + receiverBalance + raoToEth(tao(1)) + ); expect(receiverBalanceAfterCall).toEqual(receiverBalance + raoToEth(tao(1))); }, }); diff --git a/ts-tests/suites/zombienet_evm/01-contract-deploy-call.test.ts b/ts-tests/suites/zombienet_evm/01-contract-deploy-call.test.ts index f23e9b5380..b713d582af 100644 --- a/ts-tests/suites/zombienet_evm/01-contract-deploy-call.test.ts +++ b/ts-tests/suites/zombienet_evm/01-contract-deploy-call.test.ts @@ -66,11 +66,17 @@ describeSuite({ let api: TypedApi; let provider: ethers.JsonRpcProvider; let ethWallet: ethers.Wallet; + let ethSigner: ethers.NonceManager; let stakeWallet: ethers.Wallet; let proxyWallet1: ethers.Wallet; let proxyWallet2: ethers.Wallet; let proxyWallet3: ethers.Wallet; let proxyWallet4: ethers.Wallet; + let stakeSigner: ethers.NonceManager; + let proxySigner1: ethers.NonceManager; + let proxySigner2: ethers.NonceManager; + let proxySigner3: ethers.NonceManager; + let proxySigner4: ethers.NonceManager; let pureProxyReceiver: KeyringPair; let delegateProxyReceiver: KeyringPair; let hotkey: KeyringPair; @@ -83,6 +89,7 @@ describeSuite({ api = context.papi("Node").getTypedApi(subtensor); provider = context.ethers("EVM").provider as ethers.JsonRpcProvider; ethWallet = createEthersWallet(provider); + ethSigner = new ethers.NonceManager(ethWallet); await forceSetBalance(api, convertH160ToSS58(ethWallet.address)); await disableWhiteListCheck(api, true); await waitForFinalizedBlocks(api, 1); @@ -117,6 +124,11 @@ describeSuite({ proxyWallet2 = createEthersWallet(provider); proxyWallet3 = createEthersWallet(provider); proxyWallet4 = createEthersWallet(provider); + stakeSigner = new ethers.NonceManager(stakeWallet); + proxySigner1 = new ethers.NonceManager(proxyWallet1); + proxySigner2 = new ethers.NonceManager(proxyWallet2); + proxySigner3 = new ethers.NonceManager(proxyWallet3); + proxySigner4 = new ethers.NonceManager(proxyWallet4); pureProxyReceiver = generateKeyringPair("sr25519"); delegateProxyReceiver = generateKeyringPair("sr25519"); @@ -127,7 +139,7 @@ describeSuite({ proxyWalletsReady = true; } - async function deployAndFundStakeWrap(wallet: ethers.Wallet): Promise { + async function deployAndFundStakeWrap(wallet: ethers.NonceManager): Promise { const contractFactory = new ethers.ContractFactory(STAKE_WRAP_ABI, STAKE_WRAP_BYTECODE, wallet); const contract = await contractFactory.deploy(); await contract.waitForDeployment(); @@ -198,7 +210,7 @@ describeSuite({ const contractFactory = new ethers.ContractFactory( BRIDGE_TOKEN_CONTRACT_ABI, BRIDGE_TOKEN_CONTRACT_BYTECODE, - ethWallet + ethSigner ); const contract = await contractFactory.deploy("name", "symbol", ethWallet.address); await contract.waitForDeployment(); @@ -216,7 +228,7 @@ describeSuite({ const contractFactory = new ethers.ContractFactory( BRIDGE_TOKEN_CONTRACT_ABI, BRIDGE_TOKEN_CONTRACT_BYTECODE, - ethWallet + ethSigner ); const contract = await contractFactory.deploy("name", "symbol", ethWallet.address, { gasLimit: 12_345_678, @@ -239,7 +251,7 @@ describeSuite({ const stakeBalance = tao(20); const stakeBefore = await getStake(api, hotkeySs58, walletSs58, netuid); - const stakingPrecompile = new ethers.Contract(ISTAKING_V2_ADDRESS, IStakingV2ABI, ethWallet); + const stakingPrecompile = new ethers.Contract(ISTAKING_V2_ADDRESS, IStakingV2ABI, ethSigner); const tx = await stakingPrecompile.addStake(hotkey.publicKey, stakeBalance.toString(), netuid); const receipt = await tx.wait(); expect(receipt?.status).toEqual(1); @@ -266,14 +278,14 @@ describeSuite({ await ensureSubnetReady(); const hotkeySs58 = convertPublicKeyToSs58(hotkey.publicKey); const walletSs58 = convertH160ToSS58(ethWallet.address); - const stakingPrecompile = new ethers.Contract(ISTAKING_V2_ADDRESS, IStakingV2ABI, ethWallet); + const stakingPrecompile = new ethers.Contract(ISTAKING_V2_ADDRESS, IStakingV2ABI, ethSigner); const stakeBeforeDeposit = await getStake(api, hotkeySs58, walletSs58, netuid); const contractFactory = new ethers.ContractFactory( ALPHA_POOL_CONTRACT_ABI, ALPHA_POOL_CONTRACT_BYTECODE, - ethWallet + ethSigner ); const contract = await contractFactory.deploy(hotkey.publicKey); await contract.waitForDeployment(); @@ -284,7 +296,7 @@ describeSuite({ await forceSetBalance(api, convertPublicKeyToSs58(contractPublicKey)); await expectDeployedContract(provider, contractAddress); - const contractForCall = new ethers.Contract(contractAddress, ALPHA_POOL_CONTRACT_ABI, ethWallet); + const contractForCall = new ethers.Contract(contractAddress, ALPHA_POOL_CONTRACT_ABI, ethSigner); const setContractColdkeyTx = await contractForCall.setContractColdkey(contractPublicKey); const setColdkeyReceipt = await setContractColdkeyTx.wait(); expect(setColdkeyReceipt?.status).toEqual(1); @@ -297,12 +309,13 @@ describeSuite({ const depositAlphaTx = await contractForCall.depositAlpha(netuid, tao(10).toString(), hotkey.publicKey); const depositReceipt = await depositAlphaTx.wait(); - expect(depositReceipt?.status).toEqual(1); + if (!depositReceipt) throw new Error("Missing depositAlpha receipt"); + expect(depositReceipt.status).toEqual(1); // Wait for the deposit's own block to finalize rather than a // fixed block count: when GRANDPA lags best by more than 2 // blocks, the finalized-state stake reads below see the // pre-deposit stake and the toBeLessThan assertion flakes. - await waitUntilBlockFinalized(api, depositReceipt!.blockNumber); + await waitUntilBlockFinalized(api, depositReceipt.blockNumber); const stakeAfterDeposit = await getStake(api, hotkeySs58, walletSs58, netuid); expect(stakeAfterDeposit).toBeLessThan(stakeBeforeDeposit); @@ -327,7 +340,7 @@ describeSuite({ await ensureSubnetReady(); await ensureProxyWalletsReady(); - const deployedContract = await deployAndFundStakeWrap(stakeWallet); + const deployedContract = await deployAndFundStakeWrap(stakeSigner); const stakeTx = await deployedContract.stake(hotkey.publicKey, netuid, tao(2)); const stakeReceipt = await stakeTx.wait(); @@ -347,7 +360,7 @@ describeSuite({ await ensureSubnetReady(); await ensureProxyWalletsReady(); - const deployedContract = await deployAndFundStakeWrap(stakeWallet); + const deployedContract = await deployAndFundStakeWrap(stakeSigner); const tx = await deployedContract.stakeLimit(hotkey.publicKey, netuid, tao(2000), tao(1000), true); const receipt = await tx.wait(); @@ -363,7 +376,7 @@ describeSuite({ await ensureProxyWalletsReady(); const proxies = await getProxies(api, convertH160ToSS58(proxyWallet1.address)); - const contract = new ethers.Contract(IPROXY_ADDRESS, IProxyABI, proxyWallet1); + const contract = new ethers.Contract(IPROXY_ADDRESS, IProxyABI, proxySigner1); const type = 0; const delay = 0; @@ -399,7 +412,7 @@ describeSuite({ test: async () => { await ensureProxyWalletsReady(); - const contract = new ethers.Contract(IPROXY_ADDRESS, IProxyABI, proxyWallet1); + const contract = new ethers.Contract(IPROXY_ADDRESS, IProxyABI, proxySigner1); const type = 0; const delay = 0; const index = 0; @@ -421,7 +434,11 @@ describeSuite({ test: async () => { await ensureProxyWalletsReady(); - const contract = new ethers.Contract(IPROXY_ADDRESS, IProxyABI, proxyWallet2); + // Use a throwaway manager for this expected rejection. NonceManager + // increments before submission, so a preflight rejection would + // otherwise leave the persistent signer used by T10 one nonce ahead. + const rejectingSigner = new ethers.NonceManager(proxyWallet2); + const contract = new ethers.Contract(IPROXY_ADDRESS, IProxyABI, rejectingSigner); const amount = 1_000_000_000; const wrongReceiver = generateKeyringPair("sr25519"); const callCode = await getTransferCallCode(api, wrongReceiver, amount); @@ -440,7 +457,7 @@ describeSuite({ await ensureProxyWalletsReady(); const proxies = await api.query.Proxy.Proxies.getValue(convertH160ToSS58(proxyWallet2.address)); - const contract = new ethers.Contract(IPROXY_ADDRESS, IProxyABI, proxyWallet2); + const contract = new ethers.Contract(IPROXY_ADDRESS, IProxyABI, proxySigner2); const proxiesFromContract = await contract.getProxies(convertH160ToPublicKey(proxyWallet2.address)); expect(proxiesFromContract.length).toEqual(proxies[0].length); @@ -479,7 +496,7 @@ describeSuite({ const balance = await getBalance(api, receiverSs58); const amount = 1_000_000_000; - const contract2 = new ethers.Contract(IPROXY_ADDRESS, IProxyABI, proxyWallet3); + const contract2 = new ethers.Contract(IPROXY_ADDRESS, IProxyABI, proxySigner3); const callCode = await getTransferCallCode(api, delegateProxyReceiver, amount); const tx2 = await contract2.proxyCall(convertH160ToPublicKey(proxyWallet2.address), [type], callCode); await tx2.wait(); @@ -495,7 +512,7 @@ describeSuite({ await ensureProxyWalletsReady(); const proxies = await api.query.Proxy.Proxies.getValue(convertH160ToSS58(proxyWallet4.address)); - const contract = new ethers.Contract(IPROXY_ADDRESS, IProxyABI, proxyWallet4); + const contract = new ethers.Contract(IPROXY_ADDRESS, IProxyABI, proxySigner4); expect(proxies[0].length).toEqual(0); const proxiesFromContract = await contract.getProxies(convertH160ToPublicKey(proxyWallet4.address)); diff --git a/ts-tests/suites/zombienet_shield/01-scaling.test.ts b/ts-tests/suites/zombienet_shield/01-scaling.test.ts index a83d5a3e2b..aa12f50d46 100644 --- a/ts-tests/suites/zombienet_shield/01-scaling.test.ts +++ b/ts-tests/suites/zombienet_shield/01-scaling.test.ts @@ -5,6 +5,7 @@ import { Keyring } from "@polkadot/keyring"; import { hexToU8a } from "@polkadot/util"; import type { PolkadotClient, TypedApi } from "polkadot-api"; import { beforeAll, expect } from "vitest"; +import { sleep } from "@zombienet/utils"; import { checkRuntime, getAccountNonce, @@ -15,13 +16,38 @@ import { waitForFinalizedBlocks, } from "../../utils"; +type IndexedEvmBlock = { + hash: string; + number: string; +}; + +async function waitForIndexedEvmBlock(client: PolkadotClient, blockNumber: number): Promise { + const deadline = Date.now() + 60_000; + const blockTag = `0x${blockNumber.toString(16)}`; + + while (Date.now() < deadline) { + try { + const block = (await client._request("eth_getBlockByNumber", [blockTag, false])) as IndexedEvmBlock | null; + if (block) return block; + } catch { + // The node may know the finalized Substrate block just before its + // asynchronous Frontier mapping is queryable. + } + await sleep(1_000); + } + + throw new Error(`Frontier did not index finalized block #${blockNumber}`); +} + describeSuite({ id: "01_scaling", title: "MEV Shield — 6 node scaling", foundationMethods: "zombie", testCases: ({ it, context }) => { let api: TypedApi; + let apiFull: TypedApi; let client: PolkadotClient; + let clientFull: PolkadotClient; let alice: KeyringPair; let bob: KeyringPair; @@ -35,6 +61,8 @@ describeSuite({ client = context.papi("Node"); api = client.getTypedApi(subtensor); + clientFull = context.papi("NodeFull"); + apiFull = clientFull.getTypedApi(subtensor); await checkRuntime(api); }, 120000); @@ -85,6 +113,37 @@ describeSuite({ const balanceAfter = await getBalance(api, bob.address); expect(balanceAfter).toBeGreaterThan(balanceBefore); + + // The state-oriented suites run one immediately-finalized node, + // so retain explicit GRANDPA propagation and Frontier indexing + // assertions on the production-like Shield topology. Pin every + // read to the same finalized block to avoid comparing independently + // advancing latest-state views. + const finalizedHash = (await client._request("chain_getFinalizedHead", [])) as string; + const finalizedNumber = await api.query.System.Number.getValue({ at: finalizedHash }); + const authorityAccount = await api.query.System.Account.getValue(bob.address, { at: finalizedHash }); + expect(authorityAccount.data.free).toBe(balanceAfter); + const deadline = Date.now() + 60_000; + let fullNodeBalance: bigint | undefined; + while (Date.now() < deadline) { + try { + const fullAccount = await apiFull.query.System.Account.getValue(bob.address, { + at: finalizedHash, + }); + fullNodeBalance = fullAccount.data.free; + break; + } catch { + await sleep(1_000); + } + } + expect(fullNodeBalance).toBe(authorityAccount.data.free); + + const [authorityEvmBlock, fullNodeEvmBlock] = await Promise.all([ + waitForIndexedEvmBlock(client, finalizedNumber), + waitForIndexedEvmBlock(clientFull, finalizedNumber), + ]); + expect(fullNodeEvmBlock.number).toBe(authorityEvmBlock.number); + expect(fullNodeEvmBlock.hash).toBe(authorityEvmBlock.hash); }, }); diff --git a/ts-tests/utils/evm.ts b/ts-tests/utils/evm.ts index c4f35c1fba..699b7a2c6f 100644 --- a/ts-tests/utils/evm.ts +++ b/ts-tests/utils/evm.ts @@ -21,8 +21,35 @@ export function createEthersWallet(provider: ethers.JsonRpcProvider): ethers.Wal return new ethers.Wallet(account.privateKey, provider); } -export async function getEthBalance(provider: ethers.Provider, address: string): Promise { - return provider.getBalance(address); +/** Read an uncached latest balance directly from the node. */ +export async function getEthBalance(provider: ethers.JsonRpcProvider, address: string): Promise { + return BigInt(await provider.send("eth_getBalance", [address, "latest"])); +} + +/** + * Wait for Frontier's latest-state view to expose an exact balance. + * + * Raw RPC is intentional: ethers caches some high-level reads briefly, which + * can return the pre-transaction value when development blocks are very fast. + */ +export async function waitForEthBalance( + provider: ethers.JsonRpcProvider, + address: string, + expected: bigint, + timeoutMs = 30_000 +): Promise { + const deadline = Date.now() + timeoutMs; + let actual = 0n; + + while (Date.now() < deadline) { + actual = await getEthBalance(provider, address); + if (actual === expected) { + return actual; + } + await new Promise((resolve) => setTimeout(resolve, 100)); + } + + throw new Error(`Timed out waiting for ${address} balance ${expected}; last observed ${actual}`); } /** Read chain ID via RPC without ethers' cached-network checks. */ diff --git a/ts-tests/utils/staking.ts b/ts-tests/utils/staking.ts index 79c432b17b..a43ed5000a 100644 --- a/ts-tests/utils/staking.ts +++ b/ts-tests/utils/staking.ts @@ -315,7 +315,7 @@ export async function waitForBlocks(api: TypedApi, numBlocks: if (currentBlock >= targetBlock) { break; } - await new Promise((resolve) => setTimeout(resolve, 1000)); + await new Promise((resolve) => setTimeout(resolve, 100)); } } diff --git a/ts-tests/utils/transactions.ts b/ts-tests/utils/transactions.ts index d57be282fb..d8c66dafae 100644 --- a/ts-tests/utils/transactions.ts +++ b/ts-tests/utils/transactions.ts @@ -112,6 +112,7 @@ export async function sendTransaction( } const SECOND = 1000; +const LOCAL_BLOCK_POLL_INTERVAL = 100; /** * Polls until the finalized head reaches `blockNumber` (inclusive). Use this @@ -124,7 +125,7 @@ const SECOND = 1000; export async function waitUntilBlockFinalized( api: TypedApi, blockNumber: number, - pollInterval = 1 * SECOND, + pollInterval = LOCAL_BLOCK_POLL_INTERVAL, timeout = 120 * SECOND ): Promise { const deadline = Date.now() + timeout; @@ -142,7 +143,7 @@ export async function waitUntilBlockFinalized( export async function waitForFinalizedBlocks( api: TypedApi, count: number, - pollInterval = 1 * SECOND, + pollInterval = LOCAL_BLOCK_POLL_INTERVAL, timeout = 120 * SECOND ): Promise { const startBlock = await api.query.System.Number.getValue({ at: "finalized" });