feat(runner): add the test runner, plans and tmachine Session API, and run a plan on CI

.agents/skills/watch-github-actions/SKILL.md

@@ -127,6 +127,8 @@ gh run list --json databaseId,status,headBranch,url --jq '.[] | {id: .databaseId
 
 For tmachine Kubernetes conformance failures, inspect the `tmachine-diagnostics-*` artifact. The K3s report includes Agent Sandbox controller state, leader leases, events, and current and previous controller logs. The conformance playbook checks reconciliation after guest boot with a temporary scheduling-gated Sandbox probe before running tests. Distinguish a failed controller probe from a failed conformance scenario; gateway connectivity and controller Deployment availability do not prove reconciliation works.
 
+For test plan lanes (`plan-cpu-tests` and other `e2e-suites.yml` callers), start with the runner's `Summary` block in the job log, which gives PASS, FAIL or ERROR per environment and family. Then inspect the `e2e-results-<plan>-<environment>` artifact: `suites.json` lists what the runner selected, and `<environment>/<family>/` holds `junit.xml`, `nextest.log` and, after a failure, the guest's `diagnostics/`.
+
 `Pre-release` temporarily omits Codex Security from its `Security Scan` job.
 `Release` uses the passing prerelease qualification result. Other scanners and
 qualification gates still run. Use the standalone

.github/workflows/branch-checks.yml

@@ -222,6 +222,9 @@ jobs:
           cargo fmt --manifest-path examples/supervisor-middleware-content-guard/Cargo.toml --all -- --check
           cargo fmt --manifest-path examples/isolation-backend-v8/Cargo.toml --all -- --check
           cargo fmt --manifest-path tests/testkit/Cargo.toml --all -- --check
+          cargo fmt --manifest-path tests/tmachine/Cargo.toml --all -- --check
+          cargo fmt --manifest-path tests/runner/Cargo.toml --all -- --check
+          cargo fmt --manifest-path tests/suites/conformance/Cargo.toml -p sandbox -- --check
 
   cargo-lockfiles:
     name: Cargo lockfiles
@@ -239,7 +242,9 @@ jobs:
 
       - name: Verify Cargo lockfiles
         shell: nix develop -c bash -euo pipefail {0}
-        run: tasks/scripts/check-cargo-lockfiles.sh
+        run: |
+          tasks/scripts/check-cargo-lockfiles.sh
+          uv run tasks/scripts/check_cargo_lockfile_alignment.py
 
   rust-lint:
     name: Rust lint (${{ matrix.system }})
@@ -279,6 +284,9 @@ jobs:
           cargo clippy --locked --manifest-path examples/supervisor-middleware-content-guard/Cargo.toml --all-targets -- -D warnings
           cargo clippy --locked --manifest-path examples/isolation-backend-v8/Cargo.toml --all-targets -- -D warnings
           cargo clippy --locked --manifest-path tests/testkit/Cargo.toml --all-targets -- -D warnings
+          cargo clippy --locked --manifest-path tests/tmachine/Cargo.toml --all-targets -- -D warnings
+          cargo clippy --locked --manifest-path tests/runner/Cargo.toml --all-targets -- -D warnings
+          cargo clippy --locked --manifest-path tests/suites/conformance/Cargo.toml -p sandbox --all-targets -- -D warnings
 
   rust-test:
     name: Rust test (${{ matrix.system }})
@@ -328,6 +336,8 @@ jobs:
           cargo nextest run --locked --profile ci --workspace --features openshell-server/test-support
           cargo nextest run --locked --config-file .config/nextest.toml --profile ci --manifest-path examples/supervisor-middleware-content-guard/Cargo.toml
           cargo nextest run --locked --config-file .config/nextest.toml --profile ci --manifest-path tests/testkit/Cargo.toml
+          cargo nextest run --locked --config-file .config/nextest.toml --profile ci --manifest-path tests/tmachine/Cargo.toml
+          cargo nextest run --locked --config-file .config/nextest.toml --profile ci --manifest-path tests/runner/Cargo.toml
 
   rust-postgres-test:
     name: Rust PostgreSQL tests

.github/workflows/branch-e2e.yml

@@ -303,6 +303,22 @@ jobs:
           {"environment":"fedora-podman-rootless","installer":"binaries","testsuite":"driver-podman"}
         ]
 
+  # Run the tests in tests/plans/cpu-tests.yaml. The plan decides which suites
+  # run in which environments, so adding coverage means editing the plan, not
+  # this workflow. Not yet in core-e2e-result.needs: it starts out advisory.
+  plan-cpu-tests:
+    needs: [pr_metadata, prepare-integration]
+    if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true'
+    permissions:
+      actions: read
+      contents: read
+      packages: read
+    uses: ./.github/workflows/e2e-suites.yml
+    with:
+      plan: cpu-tests
+      source-sha: ${{ needs.prepare-integration.outputs.source_sha }}
+      integration-inputs-artifact-id: ${{ needs.prepare-integration.outputs.integration_inputs_artifact_id }}
+
   docker-e2e:
     needs: [pr_metadata, build-binaries, build-images]
     if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true'

.github/workflows/e2e-suites.yml

@@ -0,0 +1,153 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# Runs a test plan (tests/plans/<plan>.yaml) on CI. The plan decides which
+# suites run in which environments; this workflow keeps no list of either.
+# `openshell-test-runner test --emit-github-matrix` turns the plan into one job per
+# environment, and each job runs the plan's tests in that environment with
+# `openshell-test-runner test --env <id> --strict`.
+#
+# Modeled on the E2E Suites workflow in NVIDIA/OpenShell#4629 by Evan Lezar.
+
+name: E2E Suites
+
+on:
+  workflow_call:
+    inputs:
+      plan:
+        description: Plan to run, the file stem of a plan in tests/plans/
+        required: true
+        type: string
+      source-sha:
+        description: Source revision of the prepared candidate artifacts
+        required: true
+        type: string
+      integration-inputs-artifact-id:
+        description: Immutable ID of the prepared integration inputs artifact
+        required: true
+        type: string
+
+permissions:
+  actions: read
+  contents: read
+  packages: read
+
+# github.workflow is the caller's name here, so scope the group to this plan to
+# avoid colliding with (and cancelling) the calling workflow's other jobs.
+concurrency:
+  group: ${{ github.workflow }}-e2e-suites-${{ inputs.plan }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  plan:
+    name: Plan ${{ inputs.plan }}
+    runs-on: ubuntu-24.04
+    timeout-minutes: 30
+    outputs:
+      matrix: ${{ steps.matrix.outputs.matrix }}
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          ref: ${{ inputs['source-sha'] }}
+          persist-credentials: false
+
+      - uses: ./.github/actions/setup-nix
+        with:
+          prepare-shell: "true"
+
+      - name: Generate the job matrix from the plan
+        id: matrix
+        shell: nix develop -c bash -euo pipefail {0}
+        env:
+          PLAN: ${{ inputs.plan }}
+        run: |
+          matrix=$(cargo run --locked --quiet \
+            --manifest-path tests/runner/Cargo.toml --bin openshell-test-runner -- \
+            test --plan "tests/plans/${PLAN}.yaml" --emit-github-matrix)
+          echo "$matrix"
+          echo "matrix=${matrix}" >> "$GITHUB_OUTPUT"
+
+  suites:
+    name: E2E suites (${{ matrix.plan }}, ${{ matrix.environment }})
+    needs: plan
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(needs.plan.outputs.matrix) }}
+    runs-on: ${{ matrix.runner }}
+    timeout-minutes: 90
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          ref: ${{ inputs['source-sha'] }}
+          persist-credentials: false
+
+      - name: Enable KVM access
+        run: |
+          set -euo pipefail
+          if [[ ! -c /dev/kvm ]]; then
+            echo "::error::The runner did not expose /dev/kvm"
+            exit 1
+          fi
+          sudo chmod 0666 /dev/kvm
+          exec 3<>/dev/kvm
+          exec 3>&-
+
+      - uses: ./.github/actions/setup-nix
+
+      - name: Cache tmachine disks
+        uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0
+        with:
+          path: ~/.cache/tmachine
+          # tmachine keys its disk layers on a hash of the Ansible tree.
+          # actions/cache doesn't save after an exact key hit, so a fixed key
+          # would rebuild the changed layers on every run. Key on the playbooks
+          # and fall back to the newest entry for the same runner.
+          key: tmachine-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('tests/tmachine/ansible/**', 'tests/tmachine/images.nix') }}
+          restore-keys: |
+            tmachine-${{ runner.os }}-${{ runner.arch }}-
+
+      - name: Download integration inputs
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          artifact-ids: ${{ inputs['integration-inputs-artifact-id'] }}
+          path: artifacts
+
+      - name: Run the plan in this environment
+        env:
+          PLAN: ${{ matrix.plan }}
+          ENVIRONMENT: ${{ matrix.environment }}
+        run: |
+          # Retry dependency preparation, then run the plan once. The runner
+          # builds the suites for the guest, so it needs no prepared archives.
+          # --strict fails the job instead of skipping an environment or a
+          # selection that would run no tests.
+          nix build --no-link .#openshell-test-runner || nix build --no-link .#openshell-test-runner
+          nix run .#openshell-test-runner -- test \
+            --plan "tests/plans/${PLAN}.yaml" \
+            --env "${ENVIRONMENT}" \
+            --strict \
+            --results artifacts/test-results
+
+      # The results directory holds each family's JUnit report (junit.xml),
+      # the nextest log, and the guest's diagnostics when a run failed.
+      - name: Upload results
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+        with:
+          name: e2e-results-${{ matrix.plan }}-${{ matrix.environment }}
+          path: artifacts/test-results
+          if-no-files-found: ignore
+          retention-days: 7
+
+  result:
+    name: E2E suites result (${{ inputs.plan }})
+    needs: [plan, suites]
+    if: always()
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - uses: ./.github/actions/check-job-results
+        with:
+          results: ${{ toJSON(needs) }}

AGENTS.md

@@ -62,7 +62,11 @@ Do not rely on this file for a full inventory. The detailed public and contribut
 | `python/openshell/` | Python SDK | Python bindings and CLI packaging |
 | `sdk/typescript/` | TypeScript SDK | Native Connect client, curated sandbox API, and generated protobuf types |
 | `proto/` | Protobuf definitions | gRPC service contracts |
-| `tests/testkit/` | E2E testkit | Support library and `#[testkit::test]` attribute for SDK-driven E2E suites |
+| `tests/testkit/` | E2E testkit | Support library, `#[testkit::test]` attribute, and `testkit-spec` parser for SDK-driven E2E suites |
+| `tests/suites/` | E2E suites | One Cargo workspace per test family; `conformance/sandbox` is the first testkit suite |
+| `tests/tmachine/` | Test machines | QEMU machines provisioned with Ansible, and the `Session` API that drives them |
+| `tests/runner/` | Test runner | `openshell-test-runner`: expands plans into jobs, runs them in tmachine environments, checks JUnit results, emits the CI matrix |
+| `tests/plans/`, `tests/environments/` | Test plans | Plans pair suite families with environments; environments say what each provides |
 | `deploy/` | Docker, Helm, K8s | Dockerfiles, Helm chart, manifests |
 | `docs/` | Published docs | MDX pages, navigation, and content assets |
 | `fern/` | Docs site config | Fern site config, components, and theme assets |

CONTRIBUTING.md

@@ -345,6 +345,10 @@ These are the primary `mise` tasks for day-to-day development:
 | `proto/`        | Protocol buffer definitions and [public API conventions](proto/README.md) |
 | `tasks/`        | `mise` task definitions and build scripts     |
 | `tests/testkit/` | Support library for SDK-driven end-to-end suites |
+| `tests/suites/` | End-to-end suites, one Cargo workspace per family (`conformance/sandbox` runs through a plan) |
+| `tests/tmachine/` | QEMU test machines provisioned with Ansible, and the `Session` API that drives them |
+| `tests/runner/` | `openshell-test-runner`: expands test plans into jobs, runs them in tmachine environments, and checks the results |
+| `tests/plans/`, `tests/environments/` | Test plans, and the environments they run in |
 | `deploy/`       | Dockerfiles, Helm chart, Kubernetes manifests |
 | `docs/`         | Published Fern docs source, navigation, and content assets |
 | `fern/`         | Fern site config, components, and theme assets |

TESTING.md

@@ -17,6 +17,11 @@ python/openshell/      # Python unit tests (*_test.py suffix)
 e2e/python/            # Python E2E tests (test_*.py prefix)
 e2e/rust/              # Rust CLI E2E tests
 tests/testkit/         # Support library for SDK-driven E2E suites
+tests/suites/conformance/  # Testkit suites (the `sandbox` crate) beside the older CLI conformance crate
+tests/tmachine/        # QEMU test machines, Ansible playbooks, and the Session API
+tests/runner/          # openshell-test-runner: runs test plans in environments
+tests/plans/           # Test plans: which suites run in which environments
+tests/environments/    # Environment definitions
 ```
 
 ## Rust Tests
@@ -140,6 +145,85 @@ by `OPENSHELL_BIN`. See
 [tests/testkit/README.md](tests/testkit/README.md) for how to write and run
 tests.
 
+Suites live under `tests/suites/<family>/`. Each family is a Cargo workspace
+with its own lockfile, and a family name is what a plan's `type` selects.
+`tests/suites/conformance/sandbox` is the first testkit suite; it has one smoke
+test. Run it against a local Docker gateway with:
+
+```shell
+mise run e2e:testkit
+```
+
+That task runs `e2e/with-docker-gateway.sh`, which starts a gateway and exports
+`OPENSHELL_GATEWAY` and `OPENSHELL_BIN`, so the suite runs the same way the
+runner runs it in a guest.
+
+Workspaces that build root crates through path dependencies must lock shared
+crates at the versions in the root `Cargo.lock`; otherwise CI can build against
+dependencies the Nix toolchain doesn't support. `mise run rust:lockfiles:check`
+verifies this, and `uv run tasks/scripts/check_cargo_lockfile_alignment.py --fix`
+re-locks a workspace from the root lockfile. Add new testkit workspaces to
+`ALIGNED_WORKSPACES` in that script.
+
+### Test plans and the runner (`tests/runner/`)
+
+`openshell-test-runner` runs testkit suites in provisioned environments. It is the
+runner and plan layer from NVIDIA/OpenShell#4629 by Evan Lezar, on top of the
+testkit and tmachine here. A plan selects which suites run in which environments. The plan is the only list of
+what runs where: CI reads it, so no workflow or playbook repeats it.
+
+- **Plans** (`tests/plans/*.yaml`) pair a family (a workspace under
+  `tests/suites/`, such as `conformance`) and optionally some of its suites with
+  environment IDs. See [tests/plans/README.md](tests/plans/README.md).
+- **Environments** (`tests/environments/*.yaml`) name the tmachine environment and
+  installer that provision them and the gateway name the installer registers. The
+  runner points the tests at that gateway and the installed CLI with
+  `OPENSHELL_GATEWAY` and `OPENSHELL_BIN`.
+- **Test IDs** are optional: `#[testkit::test(id = "t-0123abcd")]` gives a test
+  an identity that survives renames. There is no per-environment skipping. A test
+  that cannot pass everywhere belongs in a different family, and a flaky or
+  broken one is marked `#[ignore = "<issue link>"]`, which drops it everywhere.
+- **Results** are checked, not trusted. A job passes only if nextest succeeds and
+  its JUnit report has a passing case for exactly the tests the runner selected.
+  A test missing from the report, or skipped, fails the job. An `#[ignore]` test
+  is left out of what a lane expects, so it can't fail the lane on its own. That
+  is the only way to take a broken or flaky test out of a lane, and it does so in every
+  environment: give the reason as a link to the tracking issue,
+  `#[ignore = "https://github.com/NVIDIA/OpenShell/issues/<n>"]`.
+- **Strict mode** (`--strict`, or `OPENSHELL_TEST_STRICT=true`) is for CI. It fails
+  instead of skipping an environment the host can't provision, a family
+  with no tests, or a plan that selects nothing. A family whose selected tests
+  are all ignored has no tests, so strict mode fails it: the plan pairs that
+  environment with suites that have nothing to run. Without it the
+  runner reports those and exits successfully, so a laptop can run the part of a
+  plan it can boot.
+
+```shell
+nix run .#openshell-test-runner -- test --plan tests/plans/cpu-tests.yaml --dry-run
+nix run .#openshell-test-runner -- test --plan tests/plans/cpu-tests.yaml --strict
+```
+
+`nix run .#openshell-test-runner` cross-builds the suites for the guest, boots
+each environment through tmachine (this needs KVM), runs nextest in the guest
+over SSH, and writes JUnit reports, logs and gateway diagnostics to
+`artifacts/test-results/<plan>/`. Every SSH and nextest step has a timeout, and
+the runner kills its VMs if it receives SIGTERM. Two runs on one host need
+different `TMACHINE_SSH_PORT` values (default 2222). The installer inputs, such
+as the Debian package and the images, come from `artifacts/`, built by
+`nix run .#build-artifacts` or downloaded by CI.
+
+**CI runs the plan.** `.github/workflows/e2e-suites.yml` takes a plan name. It
+runs `openshell-test-runner test --plan tests/plans/<name>.yaml --emit-github-matrix`,
+which prints the `{"include": [...]}` a workflow reads with `fromJSON`: one job
+per environment with the runner label that can boot it. Each job then runs
+`nix run .#openshell-test-runner -- test --plan ... --env <id> --strict` and
+uploads its JUnit reports and logs. Branch E2E calls the workflow for `cpu-tests` when the
+`test:e2e` label is set. To change what CI runs, edit the plan.
+
+The older conformance, feature and driver lanes in `.github/workflows/` are still
+written by hand and run through the tmachine testsuites in
+`tests/tmachine/config.nix`. They move to plans as their suites move to testkit.
+
 ### Python E2E (`e2e/python/`)
 
 `mise run e2e:python` builds `openshell/e2e-python:dev` from

flake.nix

@@ -127,7 +127,7 @@
           firmwarePkgs = testGuestPkgs;
         };
         testMachines = import ./tests/tmachine/config.nix {
-          inherit pkgs toolchains;
+          inherit pkgs toolchains rustToolchain;
           qemuPkgs = tmachineRuntimePkgs;
           firmwarePkgs = tmachineRuntimePkgs;
         };
@@ -203,6 +203,7 @@
           tmachine = testMachines.package;
           tmachine-config = testMachines.config;
           tmachine-unwrapped = testMachines.unwrapped;
+          openshell-test-runner = testMachines.testRunner;
         };
 
         devShells = {