Repository navigation
deploy #18
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Deploys lambda code and the agent image from an uploaded working tree: a version of source.zip, the | |
| # dirs its deploy-dirs.txt names, to one gerp, a list or every active gerp — one job per gerp at once. | |
| # From a session: `bash scripts/workflow.sh run deploy.yaml --dirs <src-dir>… [-f gerp=…] [-f image=build]`, | |
| # which zips and uploads the tree, dispatches on that version and waits on the run. Every job runs the | |
| # tree's own scripts. | |
| # | |
| # gerps the gerp list and the source's dirs, for the jobs after it | |
| # bff the BFF, once: it lives in the operator account, and a push per gerp would race | |
| # lambdas one job over the fleet: fleet.py push the dirs once, then the fleet pipe (scripts/fleet.py, the same | |
| # pipe `deploy.sh fleet` runs on a laptop), ten gerps at a time through GNU parallel | |
| # playbooks playbooks.yaml, the same gerps, after the pushes — with -f playbooks=true | |
| # integ the deployed modules' integ tests, once the pushes are done | |
| # build the agent image, on an arm64 runner (native, no emulation): docker.sh --build, upload.sh image | |
| # runtimes one job per gerp: its runtime onto the tag build pushed (or the top of ECR, image: no-build) | |
| # | |
| # The fleet's lambdas are one job: the pipe moves every gerp's functions in about a minute. The fan-out is per | |
| # gerp only where a runtime waits minutes for READY. Runners cost nothing on this public repo; a job's price is its | |
| # minute of setup. | |
| name: deploy | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| gerp: | |
| description: a gerp id, a comma list, or all (every active gerp) | |
| required: true | |
| default: gradienterp | |
| image: | |
| description: the agent image — none, build (build, push the next tag, move the runtimes) or no-build (move the runtimes onto the top of ECR) | |
| required: false | |
| type: choice | |
| default: none | |
| options: [none, build, no-build] | |
| playbooks: | |
| description: after the pushes, ingest the playbooks (every modules/**/kb.md) into each gerp's knowledge base | |
| type: boolean | |
| default: false | |
| source_version: | |
| description: a version of source.zip (empty is its latest) | |
| required: false | |
| default: "" | |
| permissions: | |
| contents: read | |
| id-token: write | |
| # one deploy per gerp input at a time, and a second one waits rather than cancelling the first | |
| concurrency: | |
| group: deploy-${{ inputs.gerp }} | |
| cancel-in-progress: false | |
| defaults: | |
| run: | |
| shell: bash | |
| jobs: | |
| gerps: | |
| runs-on: ubuntu-latest | |
| environment: prod | |
| timeout-minutes: 10 | |
| outputs: | |
| list: ${{ steps.gerps.outputs.list }} | |
| lambdas_timeout: ${{ steps.gerps.outputs.lambdas_timeout }} | |
| version: ${{ steps.source.outputs.version }} | |
| fleet_dirs: ${{ steps.dirs.outputs.fleet }} | |
| bff: ${{ steps.dirs.outputs.bff }} | |
| steps: | |
| - uses: actions/checkout@v7 | |
| - uses: ./.github/actions/aws | |
| with: | |
| role: ${{ secrets.AWS_DEPLOY_ROLE_ARN }} | |
| - id: source | |
| uses: ./.github/actions/source | |
| with: | |
| version: ${{ inputs.source_version }} | |
| - id: gerps | |
| env: | |
| GERP: ${{ inputs.gerp }} | |
| run: | | |
| if [ "$GERP" = all ]; then | |
| # every gerp that runs: an active row with a real account, as deploy.py's _active_rows reads them | |
| list="$(aws dynamodb scan --table-name gerp-customers --profile operator-org --no-cli-pager \ | |
| --filter-expression '#s = :a AND attribute_exists(aws_account_id)' \ | |
| --expression-attribute-names '{"#s": "status"}' --expression-attribute-values '{":a": {"S": "active"}}' \ | |
| --projection-expression gerp_id --output json | jq -c '[.Items[].gerp_id.S] | sort')" | |
| else | |
| list="$(jq -Rc 'split(",") | map(gsub("^ +| +$"; "")) | map(select(length > 0))' <<< "$GERP")" | |
| fi | |
| echo "list=$list" >> "$GITHUB_OUTPUT" | |
| echo "lambdas_timeout=$(( 30 + $(jq length <<< "$list") ))" >> "$GITHUB_OUTPUT" # a minute a gerp, ten at a time | |
| echo "gerps: $(jq -r 'join(", ")' <<< "$list")" >> "$GITHUB_STEP_SUMMARY" | |
| - id: dirs | |
| working-directory: ${{ steps.source.outputs.tree }} | |
| run: | | |
| echo "fleet=$(grep -vx 'prod/gradienterp_cloud/bff' deploy-dirs.txt | jq -Rsc 'split("\n") | map(select(length > 0))')" >> "$GITHUB_OUTPUT" | |
| if grep -qx 'prod/gradienterp_cloud/bff' deploy-dirs.txt; then echo "bff=true" >> "$GITHUB_OUTPUT"; else echo "bff=false" >> "$GITHUB_OUTPUT"; fi | |
| bff: | |
| needs: gerps | |
| if: needs.gerps.outputs.bff == 'true' | |
| runs-on: ubuntu-latest | |
| environment: prod | |
| timeout-minutes: 20 | |
| steps: | |
| - uses: actions/checkout@v7 | |
| - uses: actions/setup-python@v7 | |
| with: | |
| python-version: "3.12" | |
| cache: pip | |
| cache-dependency-path: scripts/requirements.txt | |
| - run: pip install -r scripts/requirements.txt | |
| - uses: ./.github/actions/aws | |
| with: | |
| role: ${{ secrets.AWS_DEPLOY_ROLE_ARN }} | |
| - id: source | |
| uses: ./.github/actions/source | |
| with: | |
| version: ${{ needs.gerps.outputs.version }} | |
| summary: "false" | |
| - working-directory: ${{ steps.source.outputs.tree }} | |
| run: bash scripts/deploy.sh push --dirs prod/gradienterp_cloud/bff --notes "$(jq -r .commit source.json)" | |
| lambdas: | |
| needs: gerps | |
| if: needs.gerps.outputs.fleet_dirs != '[]' | |
| runs-on: ubuntu-latest | |
| environment: prod | |
| timeout-minutes: ${{ fromJSON(needs.gerps.outputs.lambdas_timeout) }} | |
| steps: | |
| - uses: actions/checkout@v7 | |
| - uses: actions/setup-python@v7 | |
| with: | |
| python-version: "3.12" | |
| cache: pip | |
| cache-dependency-path: scripts/requirements.txt | |
| - run: pip install -r scripts/requirements.txt | |
| - uses: actions/setup-node@v7 | |
| with: | |
| node-version: "22" | |
| - uses: ./.github/actions/aws | |
| with: | |
| role: ${{ secrets.AWS_DEPLOY_ROLE_ARN }} | |
| - id: source | |
| uses: ./.github/actions/source | |
| with: | |
| version: ${{ needs.gerps.outputs.version }} | |
| summary: "false" | |
| - name: push | |
| working-directory: ${{ steps.source.outputs.tree }} | |
| env: | |
| DIRS: ${{ needs.gerps.outputs.fleet_dirs }} | |
| run: | | |
| mapfile -t dirs < <(jq -r '.[]' <<< "$DIRS") | |
| # push refuses a Node lambda without node_modules, and the zip leaves them out | |
| for d in "${dirs[@]}"; do | |
| if [ -f "$d/package-lock.json" ]; then (cd "$d" && npm ci --no-audit --no-fund); fi | |
| done | |
| # the artifacts once, with provenance; the fleet pipe then moves every gerp's functions onto them | |
| { echo "### pushed"; echo '```'; python3 scripts/fleet.py push "${dirs[@]}" --notes "$(jq -r .commit source.json)"; echo '```'; } | tee -a "$GITHUB_STEP_SUMMARY" | |
| - name: fleet | |
| working-directory: ${{ steps.source.outputs.tree }} | |
| env: | |
| LIST: ${{ needs.gerps.outputs.list }} | |
| DIRS: ${{ needs.gerps.outputs.fleet_dirs }} | |
| run: | | |
| # the fleet pipe, the runner's composition: identical in logic to `deploy.sh fleet` (modules/terraform | |
| # AGENTS), the same pieces in the same order; GNU parallel in the xargs slot, the log group and the | |
| # summary are the runner's | |
| mapfile -t dirs < <(jq -r '.[]' <<< "$DIRS") | |
| snapshot="$RUNNER_TEMP/fleet-snapshot.tsv" | |
| python3 scripts/fleet.py listzipversions > "$snapshot" | |
| echo "::group::fleet" | |
| rc=0 | |
| python3 scripts/fleet.py listgerps | cut -f1 | grep -Fx -f <(jq -r '.[]' <<< "$LIST") \ | |
| | parallel -k -j 10 --halt never 'python3 scripts/fleet.py listzipfnsconf --gerp {}' \ | |
| | python3 scripts/fleet.py status "$snapshot" --dirs "${dirs[@]}" \ | |
| | tee "$RUNNER_TEMP/fleet.tsv" \ | |
| | python3 scripts/fleet.py update \ | |
| || rc=$? | |
| echo "::endgroup::" | |
| { echo "### fleet"; echo '```'; cat "$RUNNER_TEMP/fleet.tsv"; echo '```'; } >> "$GITHUB_STEP_SUMMARY" | |
| exit "$rc" | |
| playbooks: | |
| # playbooks.yaml composed: the same gerps, after their pushes; only when the dispatch asks, since a | |
| # kb.md change reaching main runs it on its own | |
| needs: [gerps, lambdas] | |
| if: ${{ !cancelled() && inputs.playbooks }} | |
| permissions: | |
| contents: read | |
| id-token: write | |
| uses: ./.github/workflows/playbooks.yaml | |
| with: | |
| gerp: ${{ inputs.gerp }} | |
| secrets: inherit | |
| integ: | |
| needs: [gerps, bff, lambdas] | |
| if: ${{ !cancelled() && !contains(needs.*.result, 'failure') && (needs.lambdas.result == 'success' || needs.bff.result == 'success') }} | |
| runs-on: ubuntu-latest | |
| environment: prod | |
| timeout-minutes: 30 | |
| steps: | |
| - uses: actions/checkout@v7 | |
| - uses: actions/setup-python@v7 | |
| with: | |
| python-version: "3.12" | |
| cache: pip | |
| cache-dependency-path: scripts/requirements.txt | |
| - run: pip install -r scripts/requirements.txt | |
| - uses: ./.github/actions/aws | |
| with: | |
| role: ${{ secrets.AWS_DEPLOY_ROLE_ARN }} | |
| - id: source | |
| uses: ./.github/actions/source | |
| with: | |
| version: ${{ needs.gerps.outputs.version }} | |
| summary: "false" | |
| - working-directory: ${{ steps.source.outputs.tree }} | |
| run: | | |
| for m in $(sed -E 's#^(modules|prod)/([^/]+)/.*#\2#' deploy-dirs.txt | sort -u); do | |
| if [ -d "tests/$m/integ" ]; then bash scripts/test.sh --env integ --module "$m"; fi | |
| done | |
| build: | |
| needs: gerps | |
| if: inputs.image == 'build' | |
| runs-on: ubuntu-24.04-arm | |
| environment: prod | |
| timeout-minutes: 45 | |
| outputs: | |
| tag: ${{ steps.push.outputs.tag }} | |
| steps: | |
| - uses: actions/checkout@v7 | |
| - uses: actions/setup-python@v7 | |
| with: | |
| python-version: "3.12" | |
| cache: pip | |
| cache-dependency-path: scripts/requirements.txt | |
| - run: pip install -r scripts/requirements.txt | |
| - uses: ./.github/actions/aws | |
| with: | |
| role: ${{ secrets.AWS_DEPLOY_ROLE_ARN }} | |
| - id: source | |
| uses: ./.github/actions/source | |
| with: | |
| version: ${{ needs.gerps.outputs.version }} | |
| summary: "false" | |
| - name: build | |
| working-directory: ${{ steps.source.outputs.tree }} | |
| run: bash scripts/docker.sh --build | |
| - id: push | |
| name: push | |
| working-directory: ${{ steps.source.outputs.tree }} | |
| run: | | |
| bash scripts/upload.sh image | tee "$RUNNER_TEMP/image.out" | |
| tag="$(sed -n 's/^==> \(v[0-9]*\) = .*/\1/p' "$RUNNER_TEMP/image.out" | tail -1)" | |
| [ -n "$tag" ] || { echo "upload.sh image printed no tag"; exit 1; } | |
| echo "tag=$tag" >> "$GITHUB_OUTPUT" | |
| echo "image: $(grep '^==> v' "$RUNNER_TEMP/image.out")" >> "$GITHUB_STEP_SUMMARY" | |
| runtimes: | |
| needs: [gerps, build] | |
| if: ${{ !cancelled() && inputs.image != 'none' && needs.gerps.result == 'success' && (needs.build.result == 'success' || (inputs.image == 'no-build' && needs.build.result == 'skipped')) }} | |
| runs-on: ubuntu-latest | |
| environment: prod | |
| timeout-minutes: 30 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| gerp: ${{ fromJSON(needs.gerps.outputs.list) }} | |
| steps: | |
| - uses: actions/checkout@v7 | |
| - uses: actions/setup-python@v7 | |
| with: | |
| python-version: "3.12" | |
| cache: pip | |
| cache-dependency-path: scripts/requirements.txt | |
| - run: pip install -r scripts/requirements.txt | |
| - uses: ./.github/actions/aws | |
| with: | |
| role: ${{ secrets.AWS_DEPLOY_ROLE_ARN }} | |
| - id: source | |
| uses: ./.github/actions/source | |
| with: | |
| version: ${{ needs.gerps.outputs.version }} | |
| summary: "false" | |
| - working-directory: ${{ steps.source.outputs.tree }} | |
| env: | |
| GERP: ${{ matrix.gerp }} | |
| TAG: ${{ needs.build.outputs.tag }} | |
| run: | | |
| # the tag build pushed holds still; without one (image: no-build), the top of ECR | |
| bash scripts/deploy.sh image --no-build --gerp "$GERP" ${TAG:+--tag "$TAG"} | tee "$RUNNER_TEMP/image.out" | |
| { echo "### $GERP"; echo '```'; cat "$RUNNER_TEMP/image.out"; echo '```'; } >> "$GITHUB_STEP_SUMMARY" |