feat: migrate e2e tests from bats to go

.github/workflows/test-e2e.yml

@@ -173,11 +173,11 @@ jobs:
       - build-tools
       - read-aggkit-args
       - get-kurtosis-cdk-commit
-    uses: agglayer/e2e/.github/workflows/aggkit-e2e-single-chain.yml@76b1ff29e2ab38697aa2fb8d51fd8563bca2bb5f
+    uses: agglayer/e2e/.github/workflows/aggkit-e2e-single-chain.yml@752c305822c458ad15b1a23a1e8de00418af0929
     secrets: inherit
     with:
       kurtosis-cdk-ref: ${{ needs.get-kurtosis-cdk-commit.outputs.kurtosis-commit }}
-      agglayer-e2e-ref: 76b1ff29e2ab38697aa2fb8d51fd8563bca2bb5f # main
+      agglayer-e2e-ref: 752c305822c458ad15b1a23a1e8de00418af0929 # main
       kurtosis-cdk-enclave-name: op
       kurtosis-cdk-args: ${{ needs.read-aggkit-args.outputs.kurtosis-cdk-args-single-op-pessimistic }}
       test-name: "test-single-l2-network-op-pessimistic"
@@ -201,11 +201,11 @@ jobs:
       - build-tools
       - read-aggkit-args
       - get-kurtosis-cdk-commit
-    uses: agglayer/e2e/.github/workflows/aggkit-e2e-single-chain.yml@76b1ff29e2ab38697aa2fb8d51fd8563bca2bb5f
+    uses: agglayer/e2e/.github/workflows/aggkit-e2e-single-chain.yml@752c305822c458ad15b1a23a1e8de00418af0929
     secrets: inherit
     with:
       kurtosis-cdk-ref: ${{ needs.get-kurtosis-cdk-commit.outputs.kurtosis-commit }}
-      agglayer-e2e-ref: 76b1ff29e2ab38697aa2fb8d51fd8563bca2bb5f # main
+      agglayer-e2e-ref: 752c305822c458ad15b1a23a1e8de00418af0929 # main
       kurtosis-cdk-enclave-name: op
       kurtosis-cdk-args: ${{ needs.read-aggkit-args.outputs.kurtosis-cdk-args-single-op-succinct }}
       aggsender-find-imported-bridge-artifact: aggsender_find_imported_bridge
@@ -229,12 +229,12 @@ jobs:
       - build-tools
       - read-aggkit-args
       - get-kurtosis-cdk-commit
-    uses: agglayer/e2e/.github/workflows/aggkit-e2e-single-chain.yml@76b1ff29e2ab38697aa2fb8d51fd8563bca2bb5f
+    uses: agglayer/e2e/.github/workflows/aggkit-e2e-single-chain.yml@752c305822c458ad15b1a23a1e8de00418af0929
     if: always() && github.event_name == 'schedule' && github.ref == 'refs/heads/develop'
     secrets: inherit
     with:
       kurtosis-cdk-ref: ${{ needs.get-kurtosis-cdk-commit.outputs.kurtosis-commit }}
-      agglayer-e2e-ref: 76b1ff29e2ab38697aa2fb8d51fd8563bca2bb5f # main
+      agglayer-e2e-ref: 752c305822c458ad15b1a23a1e8de00418af0929 # main
       kurtosis-cdk-enclave-name: op
       kurtosis-cdk-args: ${{ needs.read-aggkit-args.outputs.kurtosis-cdk-args-single-op-succinct-aggoracle-committee }}
       test-name: "test-single-l2-network-op-succinct-aggoracle-committee"
@@ -258,11 +258,11 @@ jobs:
       - build-tools
       - read-aggkit-args
       - get-kurtosis-cdk-commit
-    uses: agglayer/e2e/.github/workflows/aggkit-e2e-multi-chains.yml@76b1ff29e2ab38697aa2fb8d51fd8563bca2bb5f
+    uses: agglayer/e2e/.github/workflows/aggkit-e2e-multi-chains.yml@752c305822c458ad15b1a23a1e8de00418af0929
     secrets: inherit
     with:
       kurtosis-cdk-ref: ${{ needs.get-kurtosis-cdk-commit.outputs.kurtosis-commit }}
-      agglayer-e2e-ref: 76b1ff29e2ab38697aa2fb8d51fd8563bca2bb5f # main
+      agglayer-e2e-ref: 752c305822c458ad15b1a23a1e8de00418af0929 # main
       kurtosis-cdk-enclave-name: op
       aggsender-find-imported-bridge-artifact: aggsender_find_imported_bridge
       kurtosis-cdk-args-1: ${{ needs.read-aggkit-args.outputs.kurtosis-cdk-args-1 }}
@@ -284,11 +284,11 @@ jobs:
       - build-tools
       - read-aggkit-args
       - get-kurtosis-cdk-commit
-    uses: agglayer/e2e/.github/workflows/aggkit-e2e-multi-chains.yml@76b1ff29e2ab38697aa2fb8d51fd8563bca2bb5f
+    uses: agglayer/e2e/.github/workflows/aggkit-e2e-multi-chains.yml@752c305822c458ad15b1a23a1e8de00418af0929
     secrets: inherit
     with:
       kurtosis-cdk-ref: ${{ needs.get-kurtosis-cdk-commit.outputs.kurtosis-commit }}
-      agglayer-e2e-ref: 76b1ff29e2ab38697aa2fb8d51fd8563bca2bb5f # main
+      agglayer-e2e-ref: 752c305822c458ad15b1a23a1e8de00418af0929 # main
       kurtosis-cdk-enclave-name: aggkit
       aggsender-find-imported-bridge-artifact: aggsender_find_imported_bridge
       kurtosis-cdk-args-1: ${{ needs.read-aggkit-args.outputs.kurtosis-cdk-args-3 }}

.github/workflows/test-go-e2e.yml

@@ -3,15 +3,44 @@ name: Go E2E Tests
 on:
   pull_request: {}
   workflow_dispatch: {}
-  # Uncomment to run on schedule (e.g., nightly)
-  # schedule:
-  #   - cron: "0 2 * * *"  # Run at 2 AM UTC daily
+  # Heavy envs (op-fep, op-fep-committee, op-pp-2chains, cdk-erigon-3chains) run
+  # on this nightly schedule and on manual dispatch, not on every PR (see the
+  # setup-matrix job's env-name gating below).
+  schedule:
+    - cron: "0 2 * * *"  # Run at 2 AM UTC daily
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
 jobs:
+  # P11: Emit the env_name matrix dynamically so heavy envs are gated by trigger.
+  # On pull_request only the light, fully-published env (op-pp) runs so PRs stay
+  # fast; on schedule (nightly) and workflow_dispatch all five envs run. The same
+  # JSON feeds both the pull-docker-images and test-go-e2e matrices so their legs
+  # stay one-to-one (test-go-e2e needs: the matching pull-docker-images artifact).
+  setup-matrix:
+    name: Compute env matrix
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    outputs:
+      env_names: ${{ steps.set.outputs.env_names }}
+    steps:
+      - name: Select env_name list by trigger
+        id: set
+        run: |
+          # op-pp uses fully-published images and is the fast PR gate. The four
+          # heavy envs (FEP prover/committee, multi-chain) are heavier and depend
+          # on env snapshot images, so they run only on nightly schedule / manual
+          # dispatch to keep PR runs fast.
+          if [ "${{ github.event_name }}" = "pull_request" ]; then
+            env_names='["op-pp"]'
+          else
+            env_names='["op-pp","op-fep","op-fep-committee","op-pp-2chains","cdk-erigon-3chains"]'
+          fi
+          echo "env_names=${env_names}" >> "$GITHUB_OUTPUT"
+          echo "Selected env_names (${{ github.event_name }}): ${env_names}"
+
   build-docker-image:
     name: Build Docker Image
     runs-on: ubuntu-latest
@@ -34,38 +63,64 @@ jobs:
           retention-days: 1
 
   pull-docker-images:
-    name: Pull Docker Images
+    name: Pull Docker Images (${{ matrix.env_name }})
     runs-on: ubuntu-latest
     timeout-minutes: 20
+    needs: setup-matrix
+    strategy:
+      fail-fast: false
+      matrix:
+        # P11: env_name list comes from setup-matrix (op-pp on PR; all five on
+        # schedule/dispatch). Kept identical to the test-go-e2e job matrix.
+        env_name: ${{ fromJSON(needs.setup-matrix.outputs.env_names) }}
     steps:
       - name: Checkout code
         uses: actions/checkout@v5
 
       - name: Pull Docker images from compose file
+        working-directory: test/e2e/envs/${{ matrix.env_name }}
         run: |
-          cd test/e2e/envs/op-pp
-          docker compose pull geth beacon validator agglayer op-geth-001 op-node-001
+          # Derive the image list dynamically from the selected env's compose file
+          # (excluding the locally-built aggkit image) so no service names are hardcoded.
+          images=$(docker compose config --images | grep -v 'aggkit:local')
+          echo "Images to pull: $images"
+          docker compose pull $images
 
       - name: Save pulled Docker images
+        working-directory: test/e2e/envs/${{ matrix.env_name }}
         run: |
-          cd test/e2e/envs/op-pp
-          # Extract image names from docker compose file
+          # Re-derive the same image list and save exactly those images (never aggkit:local).
           images=$(docker compose config --images | grep -v 'aggkit:local')
           echo "Images to save: $images"
           docker save $images -o /tmp/docker-images.tar
 
       - name: Upload Docker images artifact
         uses: actions/upload-artifact@v4
         with:
-          name: pulled-docker-images
+          # Per-env artifact name so multiple matrix entries (added in P11) never collide.
+          name: pulled-docker-images-${{ matrix.env_name }}
           path: /tmp/docker-images.tar
           retention-days: 1
 
   test-go-e2e:
-    name: Run Go E2E Tests
+    name: Run Go E2E Tests (${{ matrix.env_name }})
     runs-on: ubuntu-latest
-    timeout-minutes: 45
-    needs: [build-docker-image, pull-docker-images]
+    # 120 min: the go test -timeout is 90m (make test-e2e), leaving 30m for runner setup,
+    # docker image loading, and the artifact-upload step so GH Actions never kills the job
+    # before go test exits cleanly and logs are collected.
+    timeout-minutes: 120
+    needs: [setup-matrix, build-docker-image, pull-docker-images]
+    strategy:
+      fail-fast: false
+      matrix:
+        # P11: same env_name list as the pull-docker-images job matrix (from
+        # setup-matrix) so each test-go-e2e leg needs: the matching
+        # pull-docker-images artifact.
+        env_name: ${{ fromJSON(needs.setup-matrix.outputs.env_names) }}
+    env:
+      # Selects the env directory under test/e2e/envs/<env_name> and is consumed by
+      # make test-e2e -> TestMain (envs.ParseENVName). Unset defaults to op-pp.
+      E2E_ENV: ${{ matrix.env_name }}
     steps:
       - name: Checkout code
         uses: actions/checkout@v5
@@ -75,10 +130,13 @@ jobs:
         with:
           go-version: 1.25.7
 
-      - name: Install Docker Compose
+      - name: Verify Docker Compose plugin
         run: |
-          sudo apt-get update
-          sudo apt-get install -y docker-compose
+          # The test code uses `docker compose` (v2 plugin syntax via exec.Command("docker", "compose", ...)).
+          # Ubuntu runners ship with the v2 plugin pre-installed; this step verifies it is available
+          # and does NOT install the legacy v1 docker-compose standalone binary (which would shadow the
+          # plugin and break the compose commands in loader.go).
+          docker compose version
 
       - name: Download aggkit Docker image
         uses: actions/download-artifact@v4
@@ -89,7 +147,8 @@ jobs:
       - name: Download pulled Docker images
         uses: actions/download-artifact@v4
         with:
-          name: pulled-docker-images
+          # Match the per-env artifact name produced by pull-docker-images.
+          name: pulled-docker-images-${{ matrix.env_name }}
           path: /tmp
 
       - name: Load Docker images
@@ -104,7 +163,8 @@ jobs:
         if: always()
         uses: actions/upload-artifact@v4
         with:
-          name: e2e-test-results
+          # Per-env results artifact so matrix entries do not collide.
+          name: e2e-test-results-${{ matrix.env_name }}
           path: |
             test/e2e/*.log
             test/e2e/envs/**/logs/

Makefile

@@ -107,7 +107,15 @@ test-unit: ## Runs the unit tests
 
 .PHONY: test-e2e
 test-e2e: ## Runs the e2e tests
-	go test -v -timeout 30m ./test/e2e/...
+	# Timeout raised to 90m to accommodate heavier multi-container envs (op-pp-2chains,
+	# cdk-erigon-3chains) so a heavier env is not killed by the Go test timeout before the
+	# CI job timeout. Env selection is via E2E_ENV (inherited from the environment); unset = op-pp-2chains.
+	# -p 1 serializes package execution so the test/e2e suite (TestMain) and the test/e2e/envs probe
+	# tests never boot their docker envs concurrently and collide on host ports (e.g. :8545).
+	# Optional E2E_RUN restricts to a -run group (used by the CI matrix to shard op-pp-2chains into separate,
+	# fresh-env legs for the non-destructive bridge tests vs the destructive BFL/removeGER/committee
+	# tests, which manipulate aggsender/GER state and must not share an env with the rest).
+	go test -v -timeout 90m -p 1 $(if $(E2E_RUN),-run '$(E2E_RUN)',) ./test/e2e/...
 
 .PHONY: lint
 lint: ## Runs the linter

claimsync/storage/storage.go

@@ -22,12 +22,6 @@ import (
 
 var _ claimsynctypes.ClaimStorager = (*claimStorage)(nil)
 
-// blockRecord is the meddler-tagged struct for the block table.
-type blockRecord struct {
-	Num  uint64 `meddler:"num"`
-	Hash string `meddler:"hash"`
-}
-
 const (
 	// claimColumnsSQL is the list of all claim columns
 	claimColumnsSQL = `block_num,
@@ -146,11 +140,17 @@ func (s *claimStorage) getQuerier(tx dbtypes.Querier) dbtypes.Querier {
 	return s.database
 }
 
-// InsertBlock inserts a block row using meddler.
+// InsertBlock inserts a block row idempotently (upsert), mirroring bridgesync's block insert.
+// The claim syncer's bootstrap (SetClaimSyncerNextRequiredBlock -> SyncNextBlock) can race the main
+// Sync loop on the genesis block, and a restart re-processes the configured starting block; a plain
+// INSERT then hits a UNIQUE constraint on block.num and wedges the driver's infinite retry loop, so
+// the aggsender never starts and no certificate ever settles. ON CONFLICT DO UPDATE makes a duplicate
+// insert of the same block a harmless no-op instead.
 func (s *claimStorage) InsertBlock(
 	_ context.Context, tx dbtypes.Querier, blockNum uint64, blockHash common.Hash,
 ) error {
-	if err := meddler.Insert(s.getQuerier(tx), "block", &blockRecord{Num: blockNum, Hash: blockHash.Hex()}); err != nil {
+	const query = `INSERT INTO block (num, hash) VALUES ($1, $2) ON CONFLICT (num) DO UPDATE SET hash = $2`
+	if _, err := s.getQuerier(tx).Exec(query, blockNum, blockHash.Hex()); err != nil {
 		return fmt.Errorf("InsertBlock %d: %w", blockNum, err)
 	}
 	return nil

claimsync/storage/storage_test.go

@@ -62,6 +62,32 @@ func insertBlockAndClaim(t *testing.T, ctx context.Context, s claimsynctypes.Cla
 	require.NoError(t, tx.Commit())
 }
 
+// TestInsertBlock_Idempotent verifies that re-inserting the same block is a harmless no-op rather
+// than a UNIQUE-constraint error. This happens when the claim syncer's bootstrap races the main Sync
+// loop on the genesis block, or when the configured starting block is re-processed after a restart;
+// without idempotency the driver's infinite retry loop wedges and the aggsender never starts.
+func TestInsertBlock_Idempotent(t *testing.T) {
+	s, _ := newTestStorage(t)
+	ctx := context.Background()
+	hash := common.HexToHash("0x1234")
+
+	insert := func() error {
+		tx, err := s.NewTx(ctx)
+		require.NoError(t, err)
+		require.NoError(t, s.InsertBlock(ctx, tx, 0, hash))
+		return tx.Commit()
+	}
+
+	require.NoError(t, insert())
+	// Re-inserting the same block must not error (idempotent upsert).
+	require.NoError(t, insert())
+
+	last, found, err := s.GetLastProcessedBlock(ctx, nil)
+	require.NoError(t, err)
+	require.True(t, found)
+	require.Equal(t, uint64(0), last)
+}
+
 func TestInsertAndGetClaim(t *testing.T) {
 	s, _ := newTestStorage(t)
 	ctx := context.Background()

cmd/run.go

@@ -584,6 +584,26 @@ func isNeeded(casesWhereNeeded, actualCases []string) bool {
 	return false
 }
 
+// shouldAutoStartL2ClaimSync reports whether the L2 claim syncer should auto-start its main sync
+// loop from InitialBlockNum. It must NOT auto-start when an aggsender-family component is present:
+// those components bootstrap the claim syncer's starting block via SetClaimSyncerNextRequiredBlock,
+// and auto-starting the main loop in parallel would race that bootstrap on the genesis block insert
+// (UNIQUE constraint on block.num), wedging the aggsender during startup so it never settles a
+// certificate. With auto-start disabled, the main loop waits until the aggsender bootstrap has set
+// the starting block, then resumes from there.
+func shouldAutoStartL2ClaimSync(components []string) bool {
+	bootstrappedByAggsender := isNeeded([]string{
+		aggkitcommon.AGGSENDER,
+		aggkitcommon.AGGSENDERVALIDATOR,
+		aggkitcommon.AGGCHAINPROOFGEN,
+	}, components)
+	if bootstrappedByAggsender {
+		return false
+	}
+
+	return isNeeded([]string{aggkitcommon.BRIDGE, aggkitcommon.L2BRIDGESYNC}, components)
+}
+
 func l1InfoTreeMustRun(components []string) bool {
 	return isNeeded([]string{
 		aggkitcommon.AGGORACLE, aggkitcommon.AGGSENDER, aggkitcommon.AGGSENDERVALIDATOR,
@@ -968,7 +988,7 @@ func runClaimSyncL2IfNeeded(
 		return nil
 	}
 
-	cfg.AutoStart.Resolve(isNeeded([]string{aggkitcommon.BRIDGE, aggkitcommon.L2BRIDGESYNC}, components))
+	cfg.AutoStart.Resolve(shouldAutoStartL2ClaimSync(components))
 
 	res, err := claimsync.NewStandaloneClaimSync(
 		ctx,