diff --git a/.github/workflows/check_software_layer_scripts_commit.yml b/.github/workflows/check_software_layer_scripts_commit.yml new file mode 100644 index 0000000000..a345205ce6 --- /dev/null +++ b/.github/workflows/check_software_layer_scripts_commit.yml @@ -0,0 +1,70 @@ +# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions +# +# Our goal is to ensure that the bot/software_layer_scripts_commit in software-layer/main always +# points to the latest software-layer-scripts/main. +# +# This workflow checks if the commit SHA in bot/software_layer_scripts_commit +# matches the latest commit on the main branch of EESSI/software-layer-scripts. +# It only runs when the 'bot:deploy' label is present on the PR AND if the bot/software_layer_scripts_commit +# has been changed. This ensures that: +# - Contributors that don't know about bot/software_layer_scripts_commit (i.e.: haven't updated it) don't get +# a CI that complains about the version not being the latest. That's fine, we will update that file in other PRs +# and if users don't change the file in their PR, there's no risk of overwriting with an older version +# - Contributors that have changed bot/software_layer_scripts_commit can build with a fixed version (commit) +# of EESSI/software-layer-scripts, ensuring that builds for all targets are done in a coherent manner. +# After all builds are deployed, the CI is run, checking that this is the latest commit. If not, contributors +# will have to update that before merging. This ensures that mergin their PR does not cause software-layer/main's +# bot/software_layer_scripts_commit to start pointing to an _older_ commit. +name: Check software-layer-scripts commit is up to date +on: + pull_request: + types: [opened, synchronize, reopened, labeled] + paths: + - 'bot/software_layer_scripts_commit' +permissions: + contents: read # to fetch code (actions/checkout) +jobs: + check_latest_commit: + if: > + ( + contains(github.event.pull_request.labels.*.name, 'bot:deploy') + || contains(github.event.pull_request.labels.*.name, 'force-ci-checks') + ) + runs-on: ubuntu-24.04 + steps: + - name: Check out software-layer repository (shallow) + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + fetch-depth: 1 + + - name: Read commit SHA from bot/software_layer_scripts_commit + id: read_sha + run: | + if [[ ! -f bot/software_layer_scripts_commit ]]; then + echo "ERROR: bot/software_layer_scripts_commit not found!" + exit 1 + fi + SHA=$(cat bot/software_layer_scripts_commit | tr -d '[:space:]') + echo "sha=$SHA" >> $GITHUB_OUTPUT + echo "Found SHA: $SHA" + + - name: Get latest commit on main of EESSI/software-layer-scripts + id: latest_sha + env: + GH_TOKEN: ${{ github.token }} + run: | + SHA=$(gh api repos/EESSI/software-layer-scripts/commits/main --jq .sha) + echo "sha=$SHA" >> $GITHUB_OUTPUT + echo "Latest commit on main: $SHA" + + - name: Compare SHAs + run: | + if [[ "${{ steps.read_sha.outputs.sha }}" == "${{ steps.latest_sha.outputs.sha }}" ]]; then + echo "OK: bot/software_layer_scripts_commit matches the latest commit on main." + else + echo "ERROR: bot/software_layer_scripts_commit does not match the latest commit on main of EESSI/software-layer-scripts." + echo "Stored commit: ${{ steps.read_sha.outputs.sha }}" + echo "Latest commit: ${{ steps.latest_sha.outputs.sha }}" + echo "Please update bot/software_layer_scripts_commit to ${{ steps.latest_sha.outputs.sha }} and push the change." + exit 1 + fi diff --git a/.github/workflows/test_software_layer_scripts.yml b/.github/workflows/test_software_layer_scripts.yml new file mode 100644 index 0000000000..c422bfc0da --- /dev/null +++ b/.github/workflows/test_software_layer_scripts.yml @@ -0,0 +1,127 @@ +# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions +# +# This workflow verifies that the correct version of software-layer-scripts is used. +# +# First, check_bot_build_checksums checks if the bot/build.sh code that clones software-layer-scripts is untouched, +# as this normally shouldn't change (a change could mean a contributor is trying to inject something +# malicious). Having this CI means that a change in bot/build.sh should at least be accompanied by +# a change in this CI, making it stand out to reviewers and increasing the likelihood of this being caught. +# +# Second, check-software_layer_scripts_commit checks if the commit used in bot/software_layer_scripts_commit is a merge-commit for a +# merge into the default branch of software-layer-scripts. This guarantees that everything that is associated with +# that commit was approved by a reviewer (and deployed, if needed) +name: Verify software-layer-scripts +on: + push: + branches: [ "main" ] + pull_request: + workflow_dispatch: +permissions: + contents: read # to fetch code (actions/checkout) +jobs: + check_bot_build_checksum: + runs-on: ubuntu-24.04 + steps: + - name: Check out software-layer repository (shallow) + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + fetch-depth: 1 # We only need the current revision to read bot/software_layer_scripts_commit + + - name: Compute bot/build.sh checksum and verify it + run: | + # Print clear error if file doesn't exist at all + if [[ ! -f bot/build.sh ]]; then + echo "ERROR: File bot/build.sh not found!" + exit 1 + fi + + # Reference checksum + # UPDATE THIS CHECKSUM IF AND ONLY IF WE ACTUALLY WANT TO CHANGE bot/build.sh + EXPECTED_CHECKSUM="94df53fe4af9c4f28f5c3dc16bf1377e2ba5c7c0df65699d50984b538a1eeab1" + + # Compute checksum + COMPUTED_CHECKSUM=$(sha256sum bot/build.sh | awk '{print $1}') + echo "Computed checksum: $COMPUTED_CHECKSUM" + echo "Reference checksum: $EXPECTED_CHECKSUM" + + # Compare checksums + if [[ "$COMPUTED_CHECKSUM" != "$EXPECTED_CHECKSUM" ]]; then + echo "ERROR: Checksum mismatch! The file bot/build.sh has been modified." + exit 1 + else + echo "Checksum for bot/build.sh matches the reference value" + fi + check_software_layer_scripts_commit: + runs-on: ubuntu-24.04 + steps: + - name: Check out software-layer repository (shallow) + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + fetch-depth: 1 # We only need the current revision to read bot/software_layer_scripts_commit + - name: Checkout software-layer-scripts (full history) + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + repository: EESSI/software-layer-scripts + path: EESSI-software-layer-scripts + fetch-depth: 0 # full history → required for ancestry checks + + - name: Read commit SHA + id: read_sha + run: | + SHA=$(cat bot/software_layer_scripts_commit | tr -d '[:space:]') + echo "sha=$SHA" >> $GITHUB_OUTPUT + echo "Found SHA: $SHA" + + - name: Verify SHA exists in software‑layer‑scripts + working-directory: EESSI-software-layer-scripts + run: | + SHA="${{ steps.read_sha.outputs.sha }}" + + echo "Checking out commit ${SHA} from software-layer-scripts" + git fetch --depth=1 origin ${SHA} + git checkout --detach ${SHA} + + # Validate that this object is _actually_ a commit + if ! git cat-file -e "${SHA}^{commit}" 2>/dev/null; then + echo "Commit $SHA not found in software‑layer‑scripts." + exit 1 + fi + echo "Commit $SHA exists in software‑layer‑scripts." + + - name: Check that SHA is merged into the default branch + working-directory: EESSI-software-layer-scripts + run: | + SHA="${{ steps.read_sha.outputs.sha }}" + + # git merge‑base --is‑ancestor returns 0 if $SHA is an ancestor of origin/main + if git merge-base --is-ancestor "$SHA" origin/main; then + echo "Commit $SHA is merged into origin/main." + else + echo "Commit $SHA is NOT merged into origin/main." + exit 1 + fi + + - name: Verify commit is signed by GitHub’s web‑flow key + working-directory: EESSI-software-layer-scripts + env: + GIT_TRACE: 1 # extra debug output if something goes wrong + run: | + SHA="${{ steps.read_sha.outputs.sha }}" + + # Import the public key that GitHub uses for UI‑generated merges + echo "Importing GitHub web‑flow GPG key…" + curl -sSfL https://github.com/web-flow.gpg | gpg --dearmor > web-flow.gpg + gpg --import web-flow.gpg + # (optional) show the fingerprint for debugging + echo "Fingerprint of the web-flow GPG key:" + gpg --list-keys --fingerprint | grep -i "web-flow" -A1 + + # Verify the commit’s GPG signature + echo "Verifying the signature of commit $SHA…" + if git verify-commit "$SHA"; then + echo "Commit $SHA is signed and the signature validates with the web‑flow key." + echo "All verification steps succeeded." + else + echo "Commit $SHA is either unsigned or not signed by the web‑flow key." + exit 1 + fi diff --git a/.github/workflows/update_software_layer_scripts_commit.yml b/.github/workflows/update_software_layer_scripts_commit.yml new file mode 100644 index 0000000000..16ba4f418d --- /dev/null +++ b/.github/workflows/update_software_layer_scripts_commit.yml @@ -0,0 +1,104 @@ +# documentation: https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions +# +# Keeps bot/software_layer_scripts_commit in software-layer/main in sync with the latest +# commit on main of EESSI/software-layer-scripts. +# +# Triggers: +# - schedule (hourly): safety net; needs no credentials beyond the ephemeral GITHUB_TOKEN. +# - workflow_dispatch: manual runs, and remote triggering from EESSI/software-layer-scripts +# (see EESSI/.github/workflows/dispatch_software_layer_update.yml) via the workflow_dispatch API, +# using a fine-grained PAT with only "Actions: read & write" on this repo. +# +# Behavior: +# - Only acts when the stored SHA differs from the latest commit on software-layer-scripts main. +# - Recreates branch 'gh_action_update_software_layer_commit_sha' from software-layer's latest main and force-pushes it, +# so the PR never accumulates merge conflicts and the existing PR (if any) is updated in place. +# The force push is safe: the branch is bot-owned and fully regenerated on every run. +# - A PR is created only if none exists for the branch; on later updates a comment posts the new SHA. +# +# Permissions: +# - contents: write -> push to gh_action_update_software_layer_commit_sha +# - pull-requests: write -> create PR / comment on it +# main is branch-protected, so this token cannot touch main. +# +# Caveat: the pinned SHA is the raw tip of software-layer-scripts main. The check in +# test_software_layer_scripts.yml additionally requires the commit to be web-flow signed; +# if someone ever pushes directly to software-layer-scripts main (not via PR), the bot PR +# may fail that check until the next PR merge arrives. That should never happen because that branch +# is protected though. +name: Update bot/software_layer_scripts_commit +on: + schedule: + - cron: '0 * * * *' # hourly safety net + workflow_dispatch: {} # manual, or via API from software-layer-scripts +permissions: + contents: write # Needs to create a feature branch and push to it + pull-requests: write # Needs to create a PR, add comments on updates, etc +concurrency: # Prevent simultaneous runs from cron and workflow_dispatch + group: update-software-layer-scripts-commit + cancel-in-progress: true # Older run is cancelled, makes sure we get the most up-to-date SHA +jobs: + update: + runs-on: ubuntu-24.04 + steps: + - name: Check out software-layer (shallow) + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + + - name: Get latest commit on main of EESSI/software-layer-scripts + id: latest + env: + GH_TOKEN: ${{ github.token }} + run: | + SHA=$(gh api repos/EESSI/software-layer-scripts/commits/main --jq .sha) + echo "sha=$SHA" >> "$GITHUB_OUTPUT" + + - name: Compare with stored SHA + id: cmp + run: | + STORED=$(tr -d '[:space:]' < bot/software_layer_scripts_commit) + if [[ "$STORED" == "${{ steps.latest.outputs.sha }}" ]]; then + echo "changed=false" >> "$GITHUB_OUTPUT" + echo "Already up to date: $STORED" + else + echo "changed=true" >> "$GITHUB_OUTPUT" + echo "Stored: $STORED" + echo "Latest: ${{ steps.latest.outputs.sha }}" + fi + + - name: Update file and force-push gh_action_update_software_layer_commit_sha + if: steps.cmp.outputs.changed == 'true' + run: | + git checkout -B gh_action_update_software_layer_commit_sha origin/main + printf '%s\n' "${{ steps.latest.outputs.sha }}" > bot/software_layer_scripts_commit + git -c user.name='github-actions[bot]' \ + -c user.email='41898282+github-actions[bot]@users.noreply.github.com' \ + commit -am "Update bot/software_layer_scripts_commit to ${{ steps.latest.outputs.sha }}" + git push --force origin gh_action_update_software_layer_commit_sha + + - name: Create PR, or comment with the new SHA + if: steps.cmp.outputs.changed == 'true' + env: + GH_TOKEN: ${{ github.token }} + PR_BODY: | + This PR is auto-generated by the + [`update_software_layer_scripts_commit` workflow](https://github.com/EESSI/software-layer/actions/workflows/update_software_layer_scripts_commit.yml) + ([workflow source](https://github.com/EESSI/software-layer/blob/main/.github/workflows/update_software_layer_scripts_commit.yml)). + + It pins `bot/software_layer_scripts_commit` to `${{ steps.latest.outputs.sha }}`, + the current tip of [`EESSI/software-layer-scripts`](https://github.com/EESSI/software-layer-scripts) `main`. + + If `bot/software_layer_scripts_commit` in this PR looks outdated, **DO NOT push updates to this branch manually**. + The bot keeps it up to date automatically; you can also rerun the workflow manually from + [the workflow page](https://github.com/EESSI/software-layer/actions/workflows/update_software_layer_scripts_commit.yml). + run: | + NEW_SHA="${{ steps.latest.outputs.sha }}" + PR_URL=$(gh pr list --head gh_action_update_software_layer_commit_sha --state open --json url --jq '.[0].url') + if [[ -n "$PR_URL" ]]; then + echo "PR already exists: $PR_URL" + gh pr comment "$PR_URL" --body \ + "Updated \`bot/software_layer_scripts_commit\` to \`$NEW_SHA\` (commit \`$(git rev-parse HEAD)\`)." + else + gh pr create --base main --head gh_action_update_software_layer_commit_sha \ + --title "Update bot/software_layer_scripts_commit" \ + --body "$PR_BODY" + fi diff --git a/bot/build.sh b/bot/build.sh index 2884db8de4..01f3be57af 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -3,7 +3,19 @@ # give up as soon as any error occurs set -e -git clone https://github.com/EESSI/software-layer-scripts +TOPDIR=$(dirname $(realpath $0)) + +# Clone a the commit from software-layer-script that corresponds to `bot/software_layer_scripts_commit` +commit_sha=$(cat ${TOPDIR}/software_layer_scripts_commit) + +# Get a shallow clone first +git clone --depth 1 --filter=blob:none --no-checkout https://github.com/EESSI/software-layer-scripts + +# Fetch the relevant commit & check it out +cd software-layer-scripts +git fetch --depth=1 origin ${commit_sha} +git checkout --detach ${commit_sha} +cd .. # symlink everything, except for: # - common files like LICENSE and README.md diff --git a/bot/software_layer_scripts_commit b/bot/software_layer_scripts_commit new file mode 100644 index 0000000000..f81bd2b100 --- /dev/null +++ b/bot/software_layer_scripts_commit @@ -0,0 +1 @@ +89e670f2336a610827ef899ecad520af78b0a848