diff --git a/.github/workflows/runner-catalog-trusted.yml b/.github/workflows/runner-catalog-trusted.yml new file mode 100644 index 0000000..a68e8b8 --- /dev/null +++ b/.github/workflows/runner-catalog-trusted.yml @@ -0,0 +1,146 @@ +name: Trusted runner catalog provenance + +on: + pull_request_target: + paths: + - RUNNERS.md + - runner-catalog-manifest.json + - runner-profiles.json + - runner-profiles.yaml + - script/validate_runner_catalog.rb + - script/check_runner_catalog_lifecycle.rb + - script/check_runner_catalog_trust_boundary.rb + - .github/workflows/runner-catalog.yml + - .github/workflows/runner-catalog-trusted.yml + push: + branches: [main] + paths: + - RUNNERS.md + - runner-catalog-manifest.json + - runner-profiles.json + - runner-profiles.yaml + - script/validate_runner_catalog.rb + - script/check_runner_catalog_lifecycle.rb + - script/check_runner_catalog_trust_boundary.rb + - test/runner_catalog_test.rb + - test/runner_catalog_lifecycle_test.rb + - test/runner_catalog_trust_boundary_test.rb + - .github/workflows/runner-catalog.yml + - .github/workflows/runner-catalog-trusted.yml + +permissions: + contents: read + +jobs: + bootstrap: + if: github.event_name == 'push' + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v6 + - name: Exercise trusted validator contract + run: ruby test/runner_catalog_test.rb + - name: Exercise catalog lifecycle contract + run: ruby test/runner_catalog_lifecycle_test.rb + - name: Exercise trust boundary contract + run: ruby test/runner_catalog_trust_boundary_test.rb + + verify-published: + if: github.event_name == 'push' + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + - name: Materialize previous main tree + env: + BEFORE: ${{ github.event.before }} + run: | + mkdir -p .previous + if [[ -z "$BEFORE" || "$BEFORE" == "0000000000000000000000000000000000000000" ]]; then + exit 0 + fi + git cat-file -e "${BEFORE}^{commit}" + for path in RUNNERS.md runner-catalog-manifest.json runner-profiles.json runner-profiles.yaml; do + if git cat-file -e "${BEFORE}:${path}" 2>/dev/null; then + mkdir -p ".previous/$(dirname "$path")" + git show "${BEFORE}:${path}" > ".previous/$path" + fi + done + - name: Check catalog lifecycle + id: lifecycle + run: echo "action=$(ruby script/check_runner_catalog_lifecycle.rb --previous-root .previous --current-root .)" >> "$GITHUB_OUTPUT" + - name: Resolve canonical source revision + if: steps.lifecycle.outputs.action == 'validate' + id: catalog-provenance + shell: ruby {0} + run: | + require "json" + + catalog = JSON.parse(File.read("runner-profiles.json")) + provenance = catalog.fetch("metadata").fetch("provenance") + abort "missing source revision" unless provenance.fetch("revision", "").match?(/\A[0-9a-f]{40}\z/) + File.open(ENV.fetch("GITHUB_OUTPUT"), "a") { |output| output.puts("revision=#{provenance.fetch("revision")}") } + - name: Checkout canonical runner source + if: steps.lifecycle.outputs.action == 'validate' + uses: actions/checkout@v6 + with: + repository: akua-dev/gitops + ref: ${{ steps.catalog-provenance.outputs.revision }} + token: ${{ secrets.GITOPS_READ_TOKEN }} + path: .gitops-source + sparse-checkout: clusters/agentos/runner-platform/profiles.yaml + sparse-checkout-cone-mode: false + persist-credentials: false + - name: Validate generated catalog against canonical source + if: steps.lifecycle.outputs.action == 'validate' + run: ruby script/validate_runner_catalog.rb --candidate-root . --source-root .gitops-source + + verify: + if: github.event_name == 'pull_request_target' + runs-on: ubuntu-24.04 + steps: + - name: Checkout trusted validator from base + uses: actions/checkout@v6 + with: + ref: ${{ github.event.pull_request.base.sha }} + path: .trusted + persist-credentials: false + - name: Checkout candidate catalog + uses: actions/checkout@v6 + with: + repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.event.pull_request.head.sha }} + path: .candidate + persist-credentials: false + - name: Verify trusted boundary + run: ruby .trusted/script/check_runner_catalog_trust_boundary.rb --trusted-root .trusted --candidate-root .candidate + - name: Exercise base-owned infrastructure + run: ruby .trusted/test/runner_catalog_test.rb && ruby .trusted/test/runner_catalog_lifecycle_test.rb && ruby .trusted/test/runner_catalog_trust_boundary_test.rb + - name: Check catalog lifecycle + id: lifecycle + run: echo "action=$(ruby .trusted/script/check_runner_catalog_lifecycle.rb --previous-root .trusted --current-root .candidate)" >> "$GITHUB_OUTPUT" + - name: Resolve canonical source revision + if: steps.lifecycle.outputs.action == 'validate' + id: catalog-provenance + shell: ruby {0} + run: | + require "json" + + catalog = JSON.parse(File.read(".candidate/runner-profiles.json")) + provenance = catalog.fetch("metadata").fetch("provenance") + abort "missing source revision" unless provenance.fetch("revision", "").match?(/\A[0-9a-f]{40}\z/) + File.open(ENV.fetch("GITHUB_OUTPUT"), "a") { |output| output.puts("revision=#{provenance.fetch("revision")}") } + - name: Checkout canonical runner source + if: steps.lifecycle.outputs.action == 'validate' + uses: actions/checkout@v6 + with: + repository: akua-dev/gitops + ref: ${{ steps.catalog-provenance.outputs.revision }} + token: ${{ secrets.GITOPS_READ_TOKEN }} + path: .gitops-source + sparse-checkout: clusters/agentos/runner-platform/profiles.yaml + sparse-checkout-cone-mode: false + persist-credentials: false + - name: Validate generated catalog against canonical source + if: steps.lifecycle.outputs.action == 'validate' + run: ruby .trusted/script/validate_runner_catalog.rb --candidate-root .candidate --source-root .gitops-source diff --git a/.github/workflows/runner-catalog.yml b/.github/workflows/runner-catalog.yml new file mode 100644 index 0000000..6962f79 --- /dev/null +++ b/.github/workflows/runner-catalog.yml @@ -0,0 +1,50 @@ +name: Runner catalog validator bootstrap + +on: + pull_request: + paths: + - script/validate_runner_catalog.rb + - script/check_runner_catalog_lifecycle.rb + - script/check_runner_catalog_trust_boundary.rb + - test/runner_catalog_test.rb + - test/runner_catalog_lifecycle_test.rb + - test/runner_catalog_trust_boundary_test.rb + - .github/workflows/runner-catalog.yml + - .github/workflows/runner-catalog-trusted.yml + - RUNNERS.md + - runner-catalog-manifest.json + - runner-profiles.json + - runner-profiles.yaml + push: + branches: [main] + paths: + - script/validate_runner_catalog.rb + - script/check_runner_catalog_lifecycle.rb + - script/check_runner_catalog_trust_boundary.rb + - test/runner_catalog_test.rb + - test/runner_catalog_lifecycle_test.rb + - test/runner_catalog_trust_boundary_test.rb + - .github/workflows/runner-catalog.yml + - .github/workflows/runner-catalog-trusted.yml + - RUNNERS.md + - runner-catalog-manifest.json + - runner-profiles.json + - runner-profiles.yaml + +permissions: + contents: read + +jobs: + validate: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v6 + - name: Exercise trusted validator contract + run: ruby test/runner_catalog_test.rb + - name: Exercise catalog lifecycle contract + run: ruby test/runner_catalog_lifecycle_test.rb + - name: Exercise trust boundary contract + run: ruby test/runner_catalog_trust_boundary_test.rb + - name: Validate published catalog when present + if: hashFiles('runner-profiles.json') != '' + run: ruby script/validate_runner_catalog.rb diff --git a/README.md b/README.md index b8134a6..fba4147 100644 --- a/README.md +++ b/README.md @@ -4,3 +4,14 @@ This repository maintains the public GitHub organization profile for `akua-dev`. The rendered organization profile lives in [profile/README.md](profile/README.md). + +This change installs the base-owned, secret-free validation infrastructure for +the provider-neutral runner catalog. It intentionally publishes no runner +catalog yet. + +After this bootstrap merges to `main`, create the follow-up catalog publication +change from that merge commit. That change adds `RUNNERS.md`, +`runner-profiles.yaml`, `runner-profiles.json`, and +`runner-catalog-manifest.json`; its pull request is then checked by the +base-owned [`runner-catalog-trusted.yml`](.github/workflows/runner-catalog-trusted.yml) +workflow against the private `akua-dev/gitops` source. diff --git a/script/check_runner_catalog_lifecycle.rb b/script/check_runner_catalog_lifecycle.rb new file mode 100644 index 0000000..86da2e5 --- /dev/null +++ b/script/check_runner_catalog_lifecycle.rb @@ -0,0 +1,59 @@ +#!/usr/bin/env ruby + +require "optparse" + +class RunnerCatalogLifecycleError < StandardError; end + +class RunnerCatalogLifecycle + FILES = %w[RUNNERS.md runner-catalog-manifest.json runner-profiles.json runner-profiles.yaml].freeze + + def initialize(previous_root:, current_root:) + @previous_root = File.expand_path(previous_root) + @current_root = File.expand_path(current_root) + end + + def action + previous = state(@previous_root) + current = state(@current_root) + case [previous, current] + when %w[absent absent], %w[absent complete], %w[complete complete] + previous == "absent" && current == "absent" ? "skip" : "validate" + when ["partial", "absent"], ["partial", "partial"], ["partial", "complete"] + fail_with("previous catalog is partial") + when ["absent", "partial"] + fail_with("current catalog is partial") + when ["complete", "absent"], ["complete", "partial"] + fail_with("published catalog cannot be deleted or partial") + else + fail_with("unsupported catalog lifecycle transition") + end + end + + private + + def state(root) + present = FILES.count { |path| File.file?(File.join(root, path)) } + return "absent" if present.zero? + return "complete" if present == FILES.length + + "partial" + end + + def fail_with(message) + raise RunnerCatalogLifecycleError, message + end +end + +options = {} +OptionParser.new do |parser| + parser.on("--previous-root PATH") { |path| options[:previous_root] = path } + parser.on("--current-root PATH") { |path| options[:current_root] = path } +end.parse! + +begin + abort "missing lifecycle roots" unless options.keys.sort == %i[current_root previous_root] + puts RunnerCatalogLifecycle.new(**options).action +rescue RunnerCatalogLifecycleError => error + warn error.message + exit 1 +end diff --git a/script/check_runner_catalog_trust_boundary.rb b/script/check_runner_catalog_trust_boundary.rb new file mode 100644 index 0000000..600ca92 --- /dev/null +++ b/script/check_runner_catalog_trust_boundary.rb @@ -0,0 +1,112 @@ +#!/usr/bin/env ruby + +require "digest" +require "optparse" +require "yaml" + +class RunnerCatalogTrustBoundaryError < StandardError; end + +class RunnerCatalogTrustBoundary + TRUST_FILES = [ + ".github/workflows/runner-catalog-trusted.yml", + "script/check_runner_catalog_trust_boundary.rb", + "script/check_runner_catalog_lifecycle.rb", + "script/validate_runner_catalog.rb" + ].freeze + CATALOG_FILES = %w[RUNNERS.md runner-catalog-manifest.json runner-profiles.json runner-profiles.yaml].freeze + TRUSTED_WORKFLOW_PATHS = [ + *CATALOG_FILES, + "script/validate_runner_catalog.rb", + "script/check_runner_catalog_lifecycle.rb", + "script/check_runner_catalog_trust_boundary.rb", + ".github/workflows/runner-catalog.yml", + ".github/workflows/runner-catalog-trusted.yml" + ].freeze + + def initialize(trusted_root:, candidate_root:) + @trusted_root = File.expand_path(trusted_root) + @candidate_root = File.expand_path(candidate_root) + end + + def validate! + trusted_workflow = parse_workflow(@trusted_root) + candidate_workflow = parse_workflow(@candidate_root) + validate_workflow_model!(trusted_workflow) + fail_with("trusted workflow semantic drift") unless workflow_model(trusted_workflow) == workflow_model(candidate_workflow) + TRUST_FILES.each do |path| + trusted = File.join(@trusted_root, path) + candidate = File.join(@candidate_root, path) + fail_with("trusted boundary file missing: #{path}") unless File.file?(trusted) && File.file?(candidate) + fail_with("trusted boundary file modified: #{path}") unless Digest::SHA256.file(trusted).hexdigest == Digest::SHA256.file(candidate).hexdigest + end + true + rescue KeyError, Psych::Exception, Errno::ENOENT => error + raise RunnerCatalogTrustBoundaryError, error.message + end + + private + + def parse_workflow(root) + path = File.join(root, TRUST_FILES.first) + fail_with("trusted workflow missing") unless File.file?(path) + YAML.safe_load(File.read(path), aliases: false) + end + + def workflow_model(workflow) + trigger = workflow.fetch(true) + verify = workflow.fetch("jobs").fetch("verify") + { + "pullRequestTargetPaths" => trigger.fetch("pull_request_target").fetch("paths").sort, + "pushPaths" => trigger.fetch("push").fetch("paths").sort, + "verifyIf" => verify.fetch("if"), + "verifySteps" => verify.fetch("steps").map { |step| step_model(step) } + } + end + + def step_model(step) + { + "name" => step.fetch("name", nil), + "uses" => step.fetch("uses", nil), + "run" => step.fetch("run", nil), + "if" => step.fetch("if", nil), + "with" => step.fetch("with", {}).slice("ref", "path", "repository", "token", "persist-credentials", "sparse-checkout", "sparse-checkout-cone-mode") + } + end + + def validate_workflow_model!(workflow) + model = workflow_model(workflow) + expected_paths = TRUSTED_WORKFLOW_PATHS.sort + fail_with("trusted workflow catalog paths drift") unless model.fetch("pullRequestTargetPaths") == expected_paths + fail_with("trusted workflow push paths drift") unless model.fetch("pushPaths") == (expected_paths + ["test/runner_catalog_lifecycle_test.rb", "test/runner_catalog_test.rb", "test/runner_catalog_trust_boundary_test.rb"]).sort + fail_with("trusted workflow verify trigger drift") unless model.fetch("verifyIf") == "github.event_name == 'pull_request_target'" + steps = workflow.fetch("jobs").fetch("verify").fetch("steps") + boundary_index = steps.index { |step| step.fetch("name", "") == "Verify trusted boundary" } + tests_index = steps.index { |step| step.fetch("name", "") == "Exercise base-owned infrastructure" } + source_index = steps.index { |step| step.fetch("with", {}).fetch("repository", nil) == "akua-dev/gitops" } + fail_with("trusted workflow boundary step missing") unless boundary_index + fail_with("base-owned infrastructure tests missing") unless tests_index + fail_with("trusted source checkout missing") unless source_index + fail_with("trusted workflow boundary ordering drift") unless boundary_index < source_index && tests_index < source_index + fail_with("trusted boundary must use base-owned files") unless steps.fetch(boundary_index).fetch("run") == "ruby .trusted/script/check_runner_catalog_trust_boundary.rb --trusted-root .trusted --candidate-root .candidate" + fail_with("base-owned tests must use trusted files") unless steps.fetch(tests_index).fetch("run") == "ruby .trusted/test/runner_catalog_test.rb && ruby .trusted/test/runner_catalog_lifecycle_test.rb && ruby .trusted/test/runner_catalog_trust_boundary_test.rb" + end + + def fail_with(message) + raise RunnerCatalogTrustBoundaryError, message + end +end + +options = {} +OptionParser.new do |parser| + parser.on("--trusted-root PATH") { |path| options[:trusted_root] = path } + parser.on("--candidate-root PATH") { |path| options[:candidate_root] = path } +end.parse! + +begin + abort "missing trust-boundary roots" unless options.keys.sort == %i[candidate_root trusted_root] + RunnerCatalogTrustBoundary.new(**options).validate! + puts "trusted" +rescue RunnerCatalogTrustBoundaryError => error + warn error.message + exit 1 +end diff --git a/script/validate_runner_catalog.rb b/script/validate_runner_catalog.rb new file mode 100644 index 0000000..8771d36 --- /dev/null +++ b/script/validate_runner_catalog.rb @@ -0,0 +1,439 @@ +#!/usr/bin/env ruby + +require "digest" +require "json" +require "optparse" +require "yaml" + +class RunnerCatalogValidationError < StandardError; end + +class RunnerCatalogValidator + SOURCE_RELATIVE_PATH = "clusters/agentos/runner-platform/profiles.yaml" + DOCUMENTATION = "https://github.com/akua-dev/.github/blob/main/RUNNERS.md" + BASELINE_CAPABILITY = "ordinary build and test tooling" + CAPABILITY_NAMES = { + "docker" => "Docker", + "buildx" => "Buildx", + "serviceContainers" => "service containers", + "privilegedContainers" => "privileged containers", + "ordinaryBuildAndTestTooling" => BASELINE_CAPABILITY + }.freeze + PUBLIC_CAPABILITIES = (CAPABILITY_NAMES.values + [BASELINE_CAPABILITY]).freeze + PROFILE_DEFINITIONS = { + "akua-x64-ci-v2" => { + "sourceId" => "linux-x64-standard-v2", + "sourceClass" => "linux-x64", + "sourceDisplayName" => "Linux x64 standard", + "id" => "standard-v2", + "class" => "standard", + "displayName" => "Standard" + }, + "akua-docker-ci-v2" => { + "sourceId" => "linux-x64-docker-v2", + "sourceClass" => "docker-x64", + "sourceDisplayName" => "Linux x64 Docker", + "id" => "docker-v2", + "class" => "docker", + "displayName" => "Docker" + }, + "akua-heavy-ci-v2" => { + "sourceId" => "linux-x64-heavy-v2", + "sourceClass" => "heavy-x64", + "sourceDisplayName" => "Linux x64 heavy", + "id" => "heavy-v2", + "class" => "heavy", + "displayName" => "Heavy" + } + }.freeze + SOURCE_WORKLOAD_NORMALIZATIONS = { + "akua-x64-ci-v2" => { + "source" => { + "recommended" => [ + "linting, formatting, unit tests and ordinary compilation", + "jobs that do not start containers or require large local caches" + ], + "exclusions" => [ + "Docker, Buildx and GitHub Actions service containers", + "nested virtualization, KVM and architecture-specific non-x64 builds" + ] + }, + "public" => { + "recommended" => [ + "linting, formatting, unit tests and ordinary compilation", + "jobs that do not start containers or require large local caches" + ], + "exclusions" => [ + "Docker, Buildx and service containers", + "workloads whose requirements exceed the Standard guarantees" + ] + } + }, + "akua-docker-ci-v2" => { + "source" => { + "recommended" => [ + "Docker and Buildx image builds", + "integration tests using Docker or GitHub Actions service containers" + ], + "exclusions" => [ + "nested virtualization, KVM and architecture-specific non-x64 builds", + "workloads declaring resources above this profile; use the heavy profile" + ] + }, + "public" => { + "recommended" => [ + "Docker and Buildx image builds", + "integration tests using Docker or service containers" + ], + "exclusions" => [ + "workloads whose requirements exceed the Docker guarantees; use Heavy only when all Heavy bounds fit" + ] + } + }, + "akua-heavy-ci-v2" => { + "source" => { + "recommended" => [ + "memory-heavy compilation, packaging and browser or integration suites", + "Docker jobs whose declared requirements exceed the Docker profile" + ], + "exclusions" => [ + "nested virtualization, KVM and architecture-specific non-x64 builds", + "workloads requiring more than the stated minimum resource contract" + ] + }, + "public" => { + "recommended" => [ + "memory-heavy compilation, packaging and browser or integration suites that fit the Heavy guarantees", + "Docker jobs whose declared requirements exceed Docker but fit Heavy" + ], + "exclusions" => [ + "workloads requiring more than %{vcpu} vCPU, %{memoryMiB} MiB memory or %{usableDiskMiB} MiB usable disk; use an external runner or reduce requirements" + ] + } + } + }.freeze + SELECTION = { + "safeMatch" => { + "resources" => "required-at-most-guaranteed-minimum", + "capabilities" => "required-subset-of-guaranteed" + }, + "order" => %w[akua-x64-ci-v2 akua-docker-ci-v2 akua-heavy-ci-v2], + "noMatch" => "external-runner-or-reduce-requirements" + }.freeze + PROFILE_KEYS = %w[capabilities class deprecation displayName id label minimumResources status workload].freeze + METADATA_KEYS = %w[contractVersion documentation name provenance].freeze + PROVENANCE_KEYS = %w[path repository revision sha256].freeze + PUBLIC_API_VERSION = "runners.akua.dev/v1alpha1" + PUBLIC_KIND = "RunnerProfileCatalog" + REQUIREMENT_ENVIRONMENT = { + "cpu" => "AKUA_CI_REQUIRED_VCPU", + "memoryMiB" => "AKUA_CI_REQUIRED_MEMORY_MIB", + "diskMiB" => "AKUA_CI_REQUIRED_DISK_MIB" + }.freeze + MARKDOWN_TABLE_HEADER = "| Label | Profile | Minimum CPU | Minimum memory | Minimum usable disk | Guaranteed capabilities | Status | Deprecation |".freeze + MARKDOWN_TABLE_SEPARATOR = "| --- | --- | ---: | ---: | ---: | --- | --- | --- |".freeze + + def initialize(candidate_root:, source_root: nil) + @candidate_root = File.expand_path(candidate_root) + @source_root = source_root && File.expand_path(source_root) + end + + def validate! + catalog = load_catalog + validate_public_contract!(catalog) + validate_markdown!(catalog) + validate_source!(catalog) if @source_root + catalog + rescue KeyError, JSON::ParserError, Psych::Exception, Errno::ENOENT => error + raise RunnerCatalogValidationError, error.message + end + + private + + def load_catalog + json = JSON.parse(File.read(File.join(@candidate_root, "runner-profiles.json"))) + yaml = YAML.safe_load(File.read(File.join(@candidate_root, "runner-profiles.yaml")), aliases: false) + fail_with("JSON and YAML catalogs differ") unless json == yaml + manifest = JSON.parse(File.read(File.join(@candidate_root, "runner-catalog-manifest.json"))) + fail_with("manifest schema drift") unless manifest.keys.sort == %w[catalog manifestVersion source] + fail_with("manifest version drift") unless manifest.fetch("manifestVersion") == 1 + fail_with("manifest provenance drift") unless manifest.fetch("source") == json.dig("metadata", "provenance") + fail_with("manifest catalog drift") unless manifest.fetch("catalog") == json + fail_with("catalog schema drift") unless json.keys.sort == %w[apiVersion capacity kind metadata policy profiles] + json + end + + def validate_public_contract!(catalog) + metadata = catalog.fetch("metadata") + fail_with("provider-neutral metadata drift") unless metadata.keys.sort == METADATA_KEYS + fail_with("catalog api schema drift") unless catalog.fetch("apiVersion") == PUBLIC_API_VERSION && catalog.fetch("kind") == PUBLIC_KIND + fail_with("catalog contract version drift") unless metadata.fetch("contractVersion") == "2.0.0" + fail_with("catalog name drift") unless metadata.fetch("name") == "akua-ci-catalog" + fail_with("documentation drift") unless metadata.fetch("documentation") == DOCUMENTATION + + provenance = metadata.fetch("provenance") + fail_with("provenance schema drift") unless provenance.keys.sort == PROVENANCE_KEYS + fail_with("non-canonical provenance") unless provenance.slice("repository", "path") == { + "repository" => "akua-dev/gitops", + "path" => SOURCE_RELATIVE_PATH + } + fail_with("missing source revision") unless provenance.fetch("revision", "").match?(/\A[0-9a-f]{40}\z/) + fail_with("missing source SHA-256") unless provenance.fetch("sha256", "").match?(/\A[0-9a-f]{64}\z/) + capacity = catalog.fetch("capacity") + expected_capacity = { + "scope" => "organization", + "allocation" => "shared", + "maxConcurrentJobs" => 4, + "queueSlo" => nil, + "notes" => [ + "Capacity is shared by all three profiles; a profile label does not reserve a private slot.", + "Four concurrent jobs are the current safe contract. Six was only a short load experiment." + ] + } + fail_with("unsafe capacity contract") unless capacity == expected_capacity + + policy = catalog.fetch("policy") + fail_with("selection policy drift") unless policy.keys.sort == %w[requirementEnvironment selection] + fail_with("requirement environment drift") unless policy.fetch("requirementEnvironment") == REQUIREMENT_ENVIRONMENT + fail_with("selection policy drift") unless policy.fetch("selection") == SELECTION + + profiles = catalog.fetch("profiles") + fail_with("stable labels drift") unless profiles.map { |profile| profile.fetch("label") } == SELECTION.fetch("order") + profiles.each do |profile| + fail_with("provider-specific profile fields") unless profile.keys.sort == PROFILE_KEYS + definition = PROFILE_DEFINITIONS.fetch(profile.fetch("label")) { fail_with("stable labels drift") } + fail_with("public profile identity drift") unless profile.slice("id", "class", "displayName") == definition.slice("id", "class", "displayName") + fail_with("public profile resources drift") unless profile.fetch("minimumResources").keys.sort == %w[memoryMiB usableDiskMiB vcpu] + fail_with("public profile resources drift") unless profile.fetch("minimumResources").values.all? { |value| value.is_a?(Integer) && value.positive? } + capabilities = profile.fetch("capabilities") + fail_with("public capability schema drift") unless capabilities.keys == ["guaranteed"] + fail_with("public capability vocabulary drift") unless capabilities.fetch("guaranteed").all? { |capability| PUBLIC_CAPABILITIES.include?(capability) } + fail_with("missing baseline capability") unless profile.dig("capabilities", "guaranteed").include?(BASELINE_CAPABILITY) + fail_with("public workload semantics drift") unless profile.fetch("workload") == public_workload_for(profile.fetch("label"), profile.fetch("minimumResources")) + fail_with("profile status drift") unless profile.fetch("status") == "active" + fail_with("profile deprecation drift") unless profile.fetch("deprecation") == { + "deprecated" => false, + "announcedAt" => nil, + "sunsetAt" => nil, + "replacementLabel" => nil + } + end + end + + def validate_markdown!(catalog) + markdown = File.read(File.join(@candidate_root, "RUNNERS.md")) + contract_match = markdown.match(//m) + fail_with("missing structured catalog contract") unless contract_match + markdown_contract = YAML.safe_load(contract_match[1], aliases: false) + fail_with("README contract schema drift") unless markdown_contract.keys.sort == %w[capacity contractVersion provenance selection] + fail_with("README contract version drift") unless markdown_contract.fetch("contractVersion") == catalog.dig("metadata", "contractVersion") + fail_with("README capacity drift") unless markdown_contract.fetch("capacity") == catalog.fetch("capacity") + fail_with("README selection drift") unless markdown_contract.fetch("selection") == catalog.dig("policy", "selection") + fail_with("README provenance drift") unless markdown_contract.fetch("provenance") == catalog.dig("metadata", "provenance") + + lines = markdown.lines.map(&:chomp) + contract_matches = markdown.to_enum(:scan, //m).map { Regexp.last_match } + fail_with("README contract multiplicity drift") unless contract_matches.length == 1 + header_index = lines.index { |line| line.start_with?("| Label | Profile |") } + fail_with("missing profile table") unless header_index + fail_with("README table schema drift") unless lines.fetch(header_index) == MARKDOWN_TABLE_HEADER && lines.fetch(header_index + 1) == MARKDOWN_TABLE_SEPARATOR + table_lines = lines[(header_index + 2)..].take_while { |line| line.start_with?("| `akua-") } + table = table_lines.map do |line| + cells = line.strip.split("|", -1)[1...-1].map(&:strip) + fail_with("malformed profile table") unless cells.length == 8 + cpu = cells.fetch(2).match?(/\A\d+ vCPU\z/) && cells.fetch(2).to_i + memory = cells.fetch(3).match?(/\A\d+ MiB\z/) && cells.fetch(3).to_i + disk = cells.fetch(4).match?(/\A\d+ MiB\z/) && cells.fetch(4).to_i + fail_with("malformed profile resources") unless cpu && memory && disk + { + "label" => cells.fetch(0).delete("`"), + "displayName" => cells.fetch(1), + "minimumResources" => { "vcpu" => cpu, "memoryMiB" => memory, "usableDiskMiB" => disk }, + "guaranteed" => cells.fetch(5).split(/,\s*/), + "status" => cells.fetch(6), + "deprecation" => cells.fetch(7) + } + end + expected_table = catalog.fetch("profiles").map do |profile| + { + "label" => profile.fetch("label"), + "displayName" => profile.fetch("displayName"), + "minimumResources" => profile.fetch("minimumResources"), + "guaranteed" => profile.dig("capabilities", "guaranteed"), + "status" => profile.fetch("status"), + "deprecation" => profile.dig("deprecation", "deprecated") ? "deprecated" : "not deprecated" + } + end + fail_with("README profile semantics drift") unless table == expected_table + fail_with("README document semantics drift") unless markdown_text_model(lines, header_index, table_lines, contract_matches.fetch(0)) == expected_markdown_text_model(catalog) + end + + def markdown_text_model(lines, header_index, table_lines, contract_match) + contract_start = lines.index { |line| line == "" } + table_end = header_index + 2 + table_lines.length + ignored = (contract_start..contract_end).to_a + (header_index...table_end).to_a + lines.each_with_index.reject { |_line, index| ignored.include?(index) }.map(&:first).reject(&:empty?) + end + + def expected_markdown_text_model(catalog) + heavy = catalog.fetch("profiles").find { |profile| profile.fetch("label") == "akua-heavy-ci-v2" } + heavy_resources = heavy.fetch("minimumResources") + lines = [ + "# Akua GitHub Actions runner profiles", + "This provider-neutral catalog defines the stable runner labels and their conservative guarantees.", + "Workflows select a label by declared resources and capabilities; the implementation behind a label may change without repository edits.", + "Capacity is shared across all profiles and capped at #{catalog.dig("capacity", "maxConcurrentJobs")} concurrent jobs. A label does not reserve a private slot.", + "## Selection rules", + "1. Select the first profile whose guaranteed resources meet the declared requirements and whose capabilities contain every required capability.", + "2. Use the stable label in workflow configuration; do not infer implementation details from the label.", + "3. Use `akua-heavy-ci-v2` only when requirements exceed Docker but fit Heavy: #{heavy_resources.fetch("vcpu")} vCPU, #{heavy_resources.fetch("memoryMiB")} MiB memory and #{heavy_resources.fetch("usableDiskMiB")} MiB usable disk.", + "Requirements above Heavy, or capabilities absent from every profile, require an external runner or reduced requirements.", + "Required-resource inputs are supplied through AKUA_CI_REQUIRED_VCPU, AKUA_CI_REQUIRED_MEMORY_MIB and AKUA_CI_REQUIRED_DISK_MIB.", + "## Profile guarantees" + ] + catalog.fetch("profiles").each do |profile| + workload = profile.fetch("workload") + lines << "## `#{profile.fetch("label")}`" + lines << "Recommended uses:" + lines.concat(workload.fetch("recommended").map { |item| "- #{item}" }) + lines << "Exclusions:" + lines.concat(workload.fetch("exclusions").map { |item| "- #{item}" }) + end + lines.concat([ + "## Versioning and deprecation", + "The public contract version is #{catalog.dig("metadata", "contractVersion")}. Profiles are active and not deprecated unless the structured catalog says otherwise.", + "The machine-readable catalogs and provenance manifest are the canonical serialized projections of this document." + ]) + lines + end + + def validate_source!(catalog) + source_path = File.join(@source_root, SOURCE_RELATIVE_PATH) + fail_with("canonical source missing") unless File.file?(source_path) + provenance = catalog.dig("metadata", "provenance") + fail_with("canonical source hash mismatch") unless Digest::SHA256.file(source_path).hexdigest == provenance.fetch("sha256") + source = YAML.safe_load(File.read(source_path), aliases: false) + expected = normalize_source(source, provenance) + fail_with("canonical source normalization drift") unless catalog == expected + end + + def normalize_source(source, provenance) + fail_with("canonical source api schema drift") unless source.fetch("apiVersion") == PUBLIC_API_VERSION && source.fetch("kind") == PUBLIC_KIND + fail_with("canonical source contract version drift") unless source.dig("metadata", "contractVersion") == "2.0.0" + { + "apiVersion" => PUBLIC_API_VERSION, + "kind" => PUBLIC_KIND, + "metadata" => { + "name" => "akua-ci-catalog", + "contractVersion" => source.dig("metadata", "contractVersion"), + "documentation" => DOCUMENTATION, + "provenance" => provenance + }, + "capacity" => normalize_capacity(source.fetch("capacity")), + "policy" => { + "requirementEnvironment" => normalize_requirement_environment(source.dig("policy", "requirementEnvironment")), + "selection" => SELECTION + }, + "profiles" => source.fetch("profiles").map { |profile| normalize_profile(profile) } + } + end + + def normalize_capacity(capacity) + expected = { + "scope" => "organization", + "allocation" => "shared", + "maxConcurrentJobs" => 4, + "queueSlo" => nil, + "notes" => [ + "Capacity is shared by all three profiles; a profile label does not reserve a private slot.", + "Four concurrent jobs are the current safe contract. Six was only a short load experiment." + ] + } + fail_with("canonical source capacity drift") unless capacity == expected + expected + end + + def normalize_requirement_environment(environment) + fail_with("canonical source requirement environment drift") unless environment == REQUIREMENT_ENVIRONMENT + REQUIREMENT_ENVIRONMENT + end + + def normalize_profile(source_profile) + label = source_profile.fetch("label") + definition = PROFILE_DEFINITIONS.fetch(label) { fail_with("canonical source profile label drift") } + expected_source_identity = { + "id" => definition.fetch("sourceId"), + "class" => definition.fetch("sourceClass"), + "displayName" => definition.fetch("sourceDisplayName") + } + fail_with("canonical source profile identity drift") unless source_profile.slice(*expected_source_identity.keys) == expected_source_identity + resources = source_profile.fetch("minimumResources") + fail_with("canonical source resource schema drift") unless resources.is_a?(Hash) && resources.keys.sort == %w[memoryMiB usableDiskMiB vcpu] + fail_with("canonical source resource schema drift") unless resources.values.all? { |value| value.is_a?(Integer) && value.positive? } + { + "id" => definition.fetch("id"), + "label" => label, + "class" => definition.fetch("class"), + "displayName" => definition.fetch("displayName"), + "status" => source_profile.fetch("status"), + "minimumResources" => resources, + "capabilities" => { "guaranteed" => normalize_capabilities(source_profile.fetch("capabilities")) }, + "workload" => normalize_workload(source_profile.fetch("workload"), label, resources), + "deprecation" => source_profile.fetch("deprecation").transform_values { |value| value == "" ? nil : value } + } + end + + def normalize_capabilities(capabilities) + fail_with("source capability schema drift") unless capabilities.is_a?(Hash) + fail_with("source capability schema drift") unless capabilities.keys.all? { |key| CAPABILITY_NAMES.key?(key) } + result = CAPABILITY_NAMES.each_with_object([]) do |(key, name), values| + value = capabilities.fetch(key, false) + fail_with("source capability schema drift") unless value == true || value == false + values << name if value + end + fail_with("source capability not allowlisted") unless result.all? { |capability| PUBLIC_CAPABILITIES.include?(capability) } + fail_with("source missing baseline capability") unless result.include?(BASELINE_CAPABILITY) + result + end + + def normalize_workload(workload, label, resources) + normalization = SOURCE_WORKLOAD_NORMALIZATIONS.fetch(label) { fail_with("source profile label not allowlisted") } + fail_with("canonical source workload drift") unless workload == normalization.fetch("source") + public_workload_for(label, resources) + end + + def public_workload_for(label, resources) + normalization = SOURCE_WORKLOAD_NORMALIZATIONS.fetch(label) { fail_with("source profile label not allowlisted") } + public_workload = normalization.fetch("public").each_with_object({}) do |(key, values), copy| + copy[key] = values.dup + end + return public_workload unless label == "akua-heavy-ci-v2" + + public_workload["exclusions"] = public_workload.fetch("exclusions").map do |exclusion| + format( + exclusion, + vcpu: resources.fetch("vcpu"), + memoryMiB: resources.fetch("memoryMiB"), + usableDiskMiB: resources.fetch("usableDiskMiB") + ) + end + public_workload + end + + def fail_with(message) + raise RunnerCatalogValidationError, message + end +end + +options = { candidate_root: ".", source_root: nil } +OptionParser.new do |parser| + parser.on("--candidate-root PATH") { |path| options[:candidate_root] = path } + parser.on("--source-root PATH") { |path| options[:source_root] = path } +end.parse! + +begin + RunnerCatalogValidator.new(**options).validate! +rescue RunnerCatalogValidationError => error + warn error.message + exit 1 +end diff --git a/test/runner_catalog_lifecycle_test.rb b/test/runner_catalog_lifecycle_test.rb new file mode 100644 index 0000000..fa5caff --- /dev/null +++ b/test/runner_catalog_lifecycle_test.rb @@ -0,0 +1,91 @@ +require "fileutils" +require "minitest/autorun" +require "open3" +require "rbconfig" +require "tmpdir" + +class RunnerCatalogLifecycleTest < Minitest::Test + SCRIPT = File.expand_path("../script/check_runner_catalog_lifecycle.rb", __dir__) + FILES = %w[RUNNERS.md runner-catalog-manifest.json runner-profiles.json runner-profiles.yaml].freeze + + def test_bootstrap_without_catalog_skips_source_validation + with_roots do |previous, current| + assert_action(previous, current, "skip") + end + end + + def test_partial_catalog_fails_closed_before_publication + with_roots do |previous, current| + File.write(File.join(current, "runner-profiles.json"), "{}") + assert_failure(previous, current, "current catalog is partial") + end + end + + def test_complete_publication_is_validated + with_roots do |previous, current| + write_catalog(current) + assert_action(previous, current, "validate") + end + end + + def test_published_catalog_remains_validated + with_roots do |previous, current| + write_catalog(previous) + write_catalog(current) + assert_action(previous, current, "validate") + end + end + + def test_published_catalog_cannot_be_fully_deleted + with_roots do |previous, current| + write_catalog(previous) + assert_failure(previous, current, "published catalog cannot be deleted or partial") + end + end + + def test_published_catalog_cannot_become_partial + with_roots do |previous, current| + write_catalog(previous) + write_catalog(current) + File.delete(File.join(current, "RUNNERS.md")) + assert_failure(previous, current, "published catalog cannot be deleted or partial") + end + end + + private + + def with_roots + Dir.mktmpdir do |directory| + previous = File.join(directory, "previous") + current = File.join(directory, "current") + FileUtils.mkdir_p(previous) + FileUtils.mkdir_p(current) + yield previous, current + end + end + + def write_catalog(root) + FILES.each { |path| File.write(File.join(root, path), "catalog\n") } + end + + def assert_action(previous, current, expected) + stdout, stderr, status = execute_lifecycle(previous, current) + assert status.success?, stderr + assert_equal expected, stdout.strip + end + + def assert_failure(previous, current, message) + stdout, stderr, status = execute_lifecycle(previous, current) + refute status.success?, stdout + assert_includes stderr, message + end + + def execute_lifecycle(previous, current) + Open3.capture3( + RbConfig.ruby, + SCRIPT, + "--previous-root", previous, + "--current-root", current + ) + end +end diff --git a/test/runner_catalog_test.rb b/test/runner_catalog_test.rb new file mode 100644 index 0000000..33c46f0 --- /dev/null +++ b/test/runner_catalog_test.rb @@ -0,0 +1,452 @@ +require "digest" +require "fileutils" +require "json" +require "minitest/autorun" +require "open3" +require "rbconfig" +require "tmpdir" +require "yaml" + +class RunnerCatalogTest < Minitest::Test + SCRIPT = File.expand_path("../script/validate_runner_catalog.rb", __dir__) + SOURCE_PATH = "clusters/agentos/runner-platform/profiles.yaml" + DOCUMENTATION = "https://github.com/akua-dev/.github/blob/main/RUNNERS.md" + BASELINE = "ordinary build and test tooling" + REVISION = "a" * 40 + SHA256 = "0" * 64 + PROVENANCE = { + "repository" => "akua-dev/gitops", + "path" => SOURCE_PATH, + "revision" => REVISION, + "sha256" => SHA256 + }.freeze + CAPACITY = { + "scope" => "organization", + "allocation" => "shared", + "maxConcurrentJobs" => 4, + "queueSlo" => nil, + "notes" => [ + "Capacity is shared by all three profiles; a profile label does not reserve a private slot.", + "Four concurrent jobs are the current safe contract. Six was only a short load experiment." + ] + }.freeze + SELECTION = { + "safeMatch" => { + "resources" => "required-at-most-guaranteed-minimum", + "capabilities" => "required-subset-of-guaranteed" + }, + "order" => %w[akua-x64-ci-v2 akua-docker-ci-v2 akua-heavy-ci-v2], + "noMatch" => "external-runner-or-reduce-requirements" + }.freeze + REQUIREMENT_ENVIRONMENT = { + "cpu" => "AKUA_CI_REQUIRED_VCPU", + "memoryMiB" => "AKUA_CI_REQUIRED_MEMORY_MIB", + "diskMiB" => "AKUA_CI_REQUIRED_DISK_MIB" + }.freeze + SOURCE_IDENTITIES = { + "akua-x64-ci-v2" => ["linux-x64-standard-v2", "linux-x64", "Linux x64 standard", "standard-v2", "standard", "Standard"], + "akua-docker-ci-v2" => ["linux-x64-docker-v2", "docker-x64", "Linux x64 Docker", "docker-v2", "docker", "Docker"], + "akua-heavy-ci-v2" => ["linux-x64-heavy-v2", "heavy-x64", "Linux x64 heavy", "heavy-v2", "heavy", "Heavy"] + }.freeze + SOURCE_WORKLOADS = { + "akua-x64-ci-v2" => { + "recommended" => [ + "linting, formatting, unit tests and ordinary compilation", + "jobs that do not start containers or require large local caches" + ], + "exclusions" => [ + "Docker, Buildx and GitHub Actions service containers", + "nested virtualization, KVM and architecture-specific non-x64 builds" + ] + }, + "akua-docker-ci-v2" => { + "recommended" => [ + "Docker and Buildx image builds", + "integration tests using Docker or GitHub Actions service containers" + ], + "exclusions" => [ + "nested virtualization, KVM and architecture-specific non-x64 builds", + "workloads declaring resources above this profile; use the heavy profile" + ] + }, + "akua-heavy-ci-v2" => { + "recommended" => [ + "memory-heavy compilation, packaging and browser or integration suites", + "Docker jobs whose declared requirements exceed the Docker profile" + ], + "exclusions" => [ + "nested virtualization, KVM and architecture-specific non-x64 builds", + "workloads requiring more than the stated minimum resource contract" + ] + } + }.freeze + + def test_public_validator_accepts_fixture_contract + with_candidate { |candidate| assert_validator_success(candidate) } + end + + def test_public_validator_rejects_stale_markdown + with_candidate do |candidate| + path = File.join(candidate, "RUNNERS.md") + File.write(path, File.read(path).sub("| `akua-heavy-ci-v2` | Heavy | 4 vCPU | 7168 MiB", "| `akua-heavy-ci-v2` | Heavy | 4 vCPU | 7169 MiB")) + assert_validator_failure(candidate, "README profile semantics drift") + end + end + + def test_public_validator_rejects_extra_markdown_contract_keys + with_candidate do |candidate| + path = File.join(candidate, "RUNNERS.md") + File.write(path, File.read(path).sub("contractVersion: 2.0.0\n", "contractVersion: 2.0.0\nprovider: example\n")) + assert_validator_failure(candidate, "README contract schema drift") + end + end + + def test_public_validator_rejects_markdown_table_header_drift + with_candidate do |candidate| + path = File.join(candidate, "RUNNERS.md") + File.write(path, File.read(path).sub("| Label | Profile | Minimum CPU |", "| Label | Profile | Backend |")) + assert_validator_failure(candidate, "README table schema drift") + end + end + + def test_public_validator_rejects_markdown_table_separator_drift + with_candidate do |candidate| + path = File.join(candidate, "RUNNERS.md") + File.write(path, File.read(path).sub("| --- | --- | ---: |", "| --- | --- | --- |")) + assert_validator_failure(candidate, "README table schema drift") + end + end + + def test_public_validator_rejects_unmodeled_markdown_prose + with_candidate do |candidate| + path = File.join(candidate, "RUNNERS.md") + File.open(path, "a") { |file| file.puts("\nProvider-backed execution details") } + assert_validator_failure(candidate, "README document semantics drift") + end + end + + def test_public_validator_rejects_stale_manifest + with_candidate do |candidate| + path = File.join(candidate, "runner-catalog-manifest.json") + manifest = JSON.parse(File.read(path)) + manifest.fetch("catalog").fetch("profiles").last.fetch("minimumResources")["vcpu"] = 5 + File.write(path, JSON.pretty_generate(manifest) + "\n") + assert_validator_failure(candidate, "manifest catalog drift") + end + end + + def test_public_validator_rejects_extra_provenance_fields + with_candidate do |candidate| + catalog = read_catalog(candidate) + catalog.fetch("metadata").fetch("provenance")["provider"] = "example" + write_candidate(candidate, catalog) + assert_validator_failure(candidate, "provenance schema drift") + end + end + + def test_public_validator_rejects_provider_specific_public_workload_values + with_candidate do |candidate| + catalog = read_catalog(candidate) + catalog.fetch("profiles").last.fetch("workload").fetch("recommended") << "provider-backed builds" + write_candidate(candidate, catalog) + assert_validator_failure(candidate, "public workload semantics drift") + end + end + + def test_trusted_validator_rejects_truthy_non_boolean_capabilities + with_source_candidate do |candidate, source| + source_file = File.join(source, SOURCE_PATH) + source_catalog = YAML.safe_load(File.read(source_file), aliases: false) + source_catalog.fetch("profiles").last.fetch("capabilities")["docker"] = "false" + File.write(source_file, YAML.dump(source_catalog)) + update_source_hash(candidate, source_file) + assert_validator_failure(candidate, "source capability schema drift", source) + end + end + + def test_trusted_validator_rejects_missing_baseline_capability + with_source_candidate do |candidate, source| + source_file = File.join(source, SOURCE_PATH) + source_catalog = YAML.safe_load(File.read(source_file), aliases: false) + source_catalog.fetch("profiles").last.fetch("capabilities").delete("ordinaryBuildAndTestTooling") + File.write(source_file, YAML.dump(source_catalog)) + update_source_hash(candidate, source_file) + assert_validator_failure(candidate, "source missing baseline capability", source) + end + end + + def test_trusted_validator_rejects_source_workload_drift + with_source_candidate do |candidate, source| + source_file = File.join(source, SOURCE_PATH) + source_catalog = YAML.safe_load(File.read(source_file), aliases: false) + source_catalog.fetch("profiles").last.fetch("workload").fetch("recommended")[0] = "provider-backed compilation" + File.write(source_file, YAML.dump(source_catalog)) + update_source_hash(candidate, source_file) + assert_validator_failure(candidate, "canonical source workload drift", source) + end + end + + def test_trusted_validator_rejects_provider_specific_source_identity + with_source_candidate do |candidate, source| + source_file = File.join(source, SOURCE_PATH) + source_catalog = YAML.safe_load(File.read(source_file), aliases: false) + source_catalog.fetch("profiles").last["class"] = "aws-x64" + File.write(source_file, YAML.dump(source_catalog)) + update_source_hash(candidate, source_file) + assert_validator_failure(candidate, "canonical source profile identity drift", source) + end + end + + def test_trusted_validator_rejects_stale_source_resources + with_source_candidate do |candidate, source| + source_file = File.join(source, SOURCE_PATH) + source_catalog = YAML.safe_load(File.read(source_file), aliases: false) + source_catalog.fetch("profiles").last.fetch("minimumResources")["vcpu"] = 5 + File.write(source_file, YAML.dump(source_catalog)) + update_source_hash(candidate, source_file) + assert_validator_failure(candidate, "canonical source normalization drift", source) + end + end + + def test_trusted_validator_derives_heavy_markdown_bound_from_source_resources + with_source_candidate do |candidate, source| + source_file = File.join(source, SOURCE_PATH) + source_catalog = YAML.safe_load(File.read(source_file), aliases: false) + source_catalog.fetch("profiles").last.fetch("minimumResources")["vcpu"] = 5 + File.write(source_file, YAML.dump(source_catalog)) + catalog = read_catalog(candidate) + catalog.fetch("profiles").last.fetch("minimumResources")["vcpu"] = 5 + catalog.fetch("profiles").last["workload"] = public_workload("akua-heavy-ci-v2", catalog.fetch("profiles").last.fetch("minimumResources")) + write_candidate(candidate, catalog) + update_source_hash(candidate, source_file) + assert_validator_success(candidate, source) + end + end + + private + + def with_candidate + Dir.mktmpdir do |directory| + candidate = File.join(directory, "candidate") + FileUtils.mkdir_p(candidate) + write_candidate(candidate, public_catalog) + yield candidate + end + end + + def with_source_candidate + with_candidate do |candidate| + source = File.join(candidate, "source") + source_file = File.join(source, SOURCE_PATH) + FileUtils.mkdir_p(File.dirname(source_file)) + File.write(source_file, YAML.dump(source_catalog)) + update_source_hash(candidate, source_file) + yield candidate, source + end + end + + def public_catalog(provenance = PROVENANCE) + { + "apiVersion" => "runners.akua.dev/v1alpha1", + "kind" => "RunnerProfileCatalog", + "metadata" => { + "name" => "akua-ci-catalog", + "contractVersion" => "2.0.0", + "documentation" => DOCUMENTATION, + "provenance" => provenance + }, + "capacity" => CAPACITY, + "policy" => { + "requirementEnvironment" => REQUIREMENT_ENVIRONMENT, + "selection" => SELECTION + }, + "profiles" => public_profiles + } + end + + def public_profiles + SOURCE_IDENTITIES.map do |label, values| + id, profile_class, display_name = values.last(3) + capabilities = if label == "akua-x64-ci-v2" + [BASELINE] + else + ["Docker", "Buildx", "service containers", "privileged containers", BASELINE] + end + resources = { + "akua-x64-ci-v2" => { "vcpu" => 2, "memoryMiB" => 4096, "usableDiskMiB" => 10240 }, + "akua-docker-ci-v2" => { "vcpu" => 4, "memoryMiB" => 6144, "usableDiskMiB" => 15360 }, + "akua-heavy-ci-v2" => { "vcpu" => 4, "memoryMiB" => 7168, "usableDiskMiB" => 20480 } + }.fetch(label) + { + "id" => id, + "label" => label, + "class" => profile_class, + "displayName" => display_name, + "status" => "active", + "minimumResources" => resources, + "capabilities" => { "guaranteed" => capabilities }, + "workload" => public_workload(label, resources), + "deprecation" => { "deprecated" => false, "announcedAt" => nil, "sunsetAt" => nil, "replacementLabel" => nil } + } + end + end + + def public_workload(label, resources) + { + "akua-x64-ci-v2" => { + "recommended" => [ + "linting, formatting, unit tests and ordinary compilation", + "jobs that do not start containers or require large local caches" + ], + "exclusions" => [ + "Docker, Buildx and service containers", + "workloads whose requirements exceed the Standard guarantees" + ] + }, + "akua-docker-ci-v2" => { + "recommended" => [ + "Docker and Buildx image builds", + "integration tests using Docker or service containers" + ], + "exclusions" => [ + "workloads whose requirements exceed the Docker guarantees; use Heavy only when all Heavy bounds fit" + ] + }, + "akua-heavy-ci-v2" => { + "recommended" => [ + "memory-heavy compilation, packaging and browser or integration suites that fit the Heavy guarantees", + "Docker jobs whose declared requirements exceed Docker but fit Heavy" + ], + "exclusions" => [ + format( + "workloads requiring more than %d vCPU, %d MiB memory or %d MiB usable disk; use an external runner or reduce requirements", + vcpu: resources.fetch("vcpu"), + memoryMiB: resources.fetch("memoryMiB"), + usableDiskMiB: resources.fetch("usableDiskMiB") + ) + ] + } + }.fetch(label) + end + + def source_catalog + catalog = public_catalog + catalog.fetch("metadata").delete("name") + catalog.fetch("metadata").delete("documentation") + catalog.fetch("metadata").delete("provenance") + catalog.fetch("policy").delete("selection") + catalog["profiles"] = catalog.fetch("profiles").map do |profile| + label = profile.fetch("label") + source_id, source_class, source_name = SOURCE_IDENTITIES.fetch(label) + capability_hash = { + "docker" => profile.dig("capabilities", "guaranteed").include?("Docker"), + "buildx" => profile.dig("capabilities", "guaranteed").include?("Buildx"), + "serviceContainers" => profile.dig("capabilities", "guaranteed").include?("service containers"), + "privilegedContainers" => profile.dig("capabilities", "guaranteed").include?("privileged containers"), + "ordinaryBuildAndTestTooling" => true + } + profile.merge( + "id" => source_id, + "class" => source_class, + "displayName" => source_name, + "capabilities" => capability_hash, + "workload" => SOURCE_WORKLOADS.fetch(label) + ) + end + catalog + end + + def write_candidate(candidate, catalog) + File.write(File.join(candidate, "runner-profiles.json"), JSON.pretty_generate(catalog) + "\n") + File.write(File.join(candidate, "runner-profiles.yaml"), YAML.dump(catalog)) + manifest = { "manifestVersion" => 1, "source" => catalog.dig("metadata", "provenance"), "catalog" => catalog } + File.write(File.join(candidate, "runner-catalog-manifest.json"), JSON.pretty_generate(manifest) + "\n") + File.write(File.join(candidate, "RUNNERS.md"), markdown_for(catalog)) + end + + def markdown_for(catalog) + rows = catalog.fetch("profiles").map do |profile| + resources = profile.fetch("minimumResources") + capabilities = profile.dig("capabilities", "guaranteed").join(", ") + deprecated = profile.dig("deprecation", "deprecated") ? "deprecated" : "not deprecated" + "| `#{profile.fetch("label")}` | #{profile.fetch("displayName")} | #{resources.fetch("vcpu")} vCPU | #{resources.fetch("memoryMiB")} MiB | #{resources.fetch("usableDiskMiB")} MiB | #{capabilities} | #{profile.fetch("status")} | #{deprecated} |" + end + contract = { + "contractVersion" => catalog.dig("metadata", "contractVersion"), + "capacity" => catalog.fetch("capacity"), + "selection" => catalog.dig("policy", "selection"), + "provenance" => catalog.dig("metadata", "provenance") + } + contract_yaml = YAML.dump(contract).sub("---\n", "").lines.map { |line| " #{line}" }.join + heavy = catalog.fetch("profiles").find { |profile| profile.fetch("label") == "akua-heavy-ci-v2" } + heavy_resources = heavy.fetch("minimumResources") + profile_text = catalog.fetch("profiles").map do |profile| + workload = profile.fetch("workload") + [ + "## `#{profile.fetch("label")}`", + "Recommended uses:", + *workload.fetch("recommended").map { |item| "- #{item}" }, + "Exclusions:", + *workload.fetch("exclusions").map { |item| "- #{item}" } + ] + end.flatten.join("\n") + <<~MARKDOWN + # Akua GitHub Actions runner profiles + + This provider-neutral catalog defines the stable runner labels and their conservative guarantees. + Workflows select a label by declared resources and capabilities; the implementation behind a label may change without repository edits. + Capacity is shared across all profiles and capped at #{catalog.dig("capacity", "maxConcurrentJobs")} concurrent jobs. A label does not reserve a private slot. + + ## Selection rules + + 1. Select the first profile whose guaranteed resources meet the declared requirements and whose capabilities contain every required capability. + 2. Use the stable label in workflow configuration; do not infer implementation details from the label. + 3. Use `akua-heavy-ci-v2` only when requirements exceed Docker but fit Heavy: #{heavy_resources.fetch("vcpu")} vCPU, #{heavy_resources.fetch("memoryMiB")} MiB memory and #{heavy_resources.fetch("usableDiskMiB")} MiB usable disk. + Requirements above Heavy, or capabilities absent from every profile, require an external runner or reduced requirements. + Required-resource inputs are supplied through AKUA_CI_REQUIRED_VCPU, AKUA_CI_REQUIRED_MEMORY_MIB and AKUA_CI_REQUIRED_DISK_MIB. + + ## Profile guarantees + + #{profile_text} + + ## Versioning and deprecation + + The public contract version is #{catalog.dig("metadata", "contractVersion")}. Profiles are active and not deprecated unless the structured catalog says otherwise. + The machine-readable catalogs and provenance manifest are the canonical serialized projections of this document. + + | Label | Profile | Minimum CPU | Minimum memory | Minimum usable disk | Guaranteed capabilities | Status | Deprecation | + | --- | --- | ---: | ---: | ---: | --- | --- | --- | + #{rows.join("\n")} + + + MARKDOWN + end + + def read_catalog(candidate) + JSON.parse(File.read(File.join(candidate, "runner-profiles.json"))) + end + + def update_source_hash(candidate, source_file) + catalog = read_catalog(candidate) + provenance = catalog.fetch("metadata").fetch("provenance").merge("sha256" => Digest::SHA256.file(source_file).hexdigest) + catalog.fetch("metadata")["provenance"] = provenance + write_candidate(candidate, catalog) + end + + def assert_validator_success(candidate, source = nil) + args = [RbConfig.ruby, SCRIPT, "--candidate-root", candidate] + args.concat(["--source-root", source]) if source + stdout, stderr, status = Open3.capture3(*args) + assert status.success?, "#{stdout}\n#{stderr}" + end + + def assert_validator_failure(candidate, message, source = nil) + args = [RbConfig.ruby, SCRIPT, "--candidate-root", candidate] + args.concat(["--source-root", source]) if source + stdout, stderr, status = Open3.capture3(*args) + refute status.success?, stdout + assert_includes stderr, message + end +end diff --git a/test/runner_catalog_trust_boundary_test.rb b/test/runner_catalog_trust_boundary_test.rb new file mode 100644 index 0000000..05ade53 --- /dev/null +++ b/test/runner_catalog_trust_boundary_test.rb @@ -0,0 +1,124 @@ +require "digest" +require "fileutils" +require "minitest/autorun" +require "open3" +require "rbconfig" +require "tmpdir" +require "yaml" + +class RunnerCatalogTrustBoundaryTest < Minitest::Test + SCRIPT = File.expand_path("../script/check_runner_catalog_trust_boundary.rb", __dir__) + TRUST_FILES = [ + ".github/workflows/runner-catalog-trusted.yml", + "script/check_runner_catalog_trust_boundary.rb", + "script/check_runner_catalog_lifecycle.rb", + "script/validate_runner_catalog.rb" + ].freeze + CATALOG_FILES = %w[RUNNERS.md runner-catalog-manifest.json runner-profiles.json runner-profiles.yaml].freeze + WORKFLOW_PATHS = [ + *CATALOG_FILES, + "script/validate_runner_catalog.rb", + "script/check_runner_catalog_lifecycle.rb", + "script/check_runner_catalog_trust_boundary.rb", + ".github/workflows/runner-catalog.yml", + ".github/workflows/runner-catalog-trusted.yml" + ].freeze + + def test_current_trust_root_accepts_itself + stdout, stderr, status = execute(Dir.pwd, Dir.pwd) + assert status.success?, stderr + assert_equal "trusted", stdout.strip + end + + def test_unchanged_fixture_trust_root_passes + with_roots do |trusted, candidate| + write_trust_root(trusted) + FileUtils.cp_r("#{trusted}/.", candidate) + assert_success(trusted, candidate) + end + end + + def test_weakened_fixture_workflow_fails_semantically + with_roots do |trusted, candidate| + write_trust_root(trusted) + FileUtils.cp_r("#{trusted}/.", candidate) + workflow_path = File.join(candidate, ".github/workflows/runner-catalog-trusted.yml") + workflow = YAML.safe_load(File.read(workflow_path), aliases: false) + workflow.fetch("jobs").fetch("verify").fetch("steps").delete_if { |step| step.fetch("name", "") == "Verify trusted boundary" } + File.write(workflow_path, YAML.dump(workflow)) + assert_failure(trusted, candidate, "trusted workflow semantic drift") + end + end + + def test_modified_fixture_verifier_fails_integrity_check + with_roots do |trusted, candidate| + write_trust_root(trusted) + FileUtils.cp_r("#{trusted}/.", candidate) + File.open(File.join(candidate, "script/validate_runner_catalog.rb"), "a") { |file| file.puts("changed") } + assert_failure(trusted, candidate, "trusted boundary file modified: script/validate_runner_catalog.rb") + end + end + + def test_deleted_fixture_trust_workflow_fails_integrity_check + with_roots do |trusted, candidate| + write_trust_root(trusted) + FileUtils.cp_r("#{trusted}/.", candidate) + File.delete(File.join(candidate, ".github/workflows/runner-catalog-trusted.yml")) + assert_failure(trusted, candidate, "trusted workflow missing") + end + end + + private + + def workflow + { + true => { + "pull_request_target" => { "paths" => WORKFLOW_PATHS }, + "push" => { "branches" => ["main"], "paths" => WORKFLOW_PATHS + ["test/runner_catalog_test.rb", "test/runner_catalog_lifecycle_test.rb", "test/runner_catalog_trust_boundary_test.rb"] } + }, + "jobs" => { + "verify" => { + "if" => "github.event_name == 'pull_request_target'", + "steps" => [ + { "name" => "Verify trusted boundary", "run" => "ruby .trusted/script/check_runner_catalog_trust_boundary.rb --trusted-root .trusted --candidate-root .candidate" }, + { "name" => "Exercise base-owned infrastructure", "run" => "ruby .trusted/test/runner_catalog_test.rb && ruby .trusted/test/runner_catalog_lifecycle_test.rb && ruby .trusted/test/runner_catalog_trust_boundary_test.rb" }, + { "name" => "Checkout canonical runner source", "uses" => "actions/checkout@v6", "with" => { "repository" => "akua-dev/gitops", "token" => "${{ secrets.GITOPS_READ_TOKEN }}" } } + ] + } + } + } + end + + def with_roots + Dir.mktmpdir do |directory| + trusted = File.join(directory, "trusted") + candidate = File.join(directory, "candidate") + FileUtils.mkdir_p(trusted) + FileUtils.mkdir_p(candidate) + yield trusted, candidate + end + end + + def write_trust_root(root) + FileUtils.mkdir_p(File.join(root, ".github/workflows")) + FileUtils.mkdir_p(File.join(root, "script")) + File.write(File.join(root, ".github/workflows/runner-catalog-trusted.yml"), YAML.dump(workflow)) + TRUST_FILES.drop(1).each { |path| File.write(File.join(root, path), "base-owned\n") } + end + + def execute(trusted, candidate) + Open3.capture3(RbConfig.ruby, SCRIPT, "--trusted-root", trusted, "--candidate-root", candidate) + end + + def assert_success(trusted, candidate) + stdout, stderr, status = execute(trusted, candidate) + assert status.success?, stderr + assert_equal "trusted", stdout.strip + end + + def assert_failure(trusted, candidate, message) + stdout, stderr, status = execute(trusted, candidate) + refute status.success?, stdout + assert_includes stderr, message + end +end