diff --git a/.github/scripts/smoke-cluster.sh b/.github/scripts/smoke-cluster.sh new file mode 100755 index 00000000000..dbe54b7abfa --- /dev/null +++ b/.github/scripts/smoke-cluster.sh @@ -0,0 +1,179 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# smoke-cluster.sh -- boot a computing-unit-master and a computing-unit-worker +# from their unpacked dist and assert the worker actually JOINS the master's +# amber Pekko cluster, not merely that its process survives. +# +# Why this can't reuse smoke-boot.sh: ComputingUnitWorker.main only calls +# AmberRuntime.startActorWorker, which binds pekko.remote.artery.canonical.port=0 +# (a random ephemeral port) and starts no HTTP server -- so there is no fixed +# port to probe. A worker booted standalone just retries its seed forever and +# stays up, so "did not crash" proves nothing. The real health signal is cluster +# membership, which ClusterListener prints on every membership change: +# ---------Now we have N nodes in the cluster--------- (N = cluster.state.members.size) +# master-alone => 1, master+worker joined => 2. We wait for the "2" line -- a +# deliberate readiness signal the app emits, not a crash-scan (contrast #6332). +# See issue #6523 (raised from the #6377 review) and #6220 for the rationale. +# +# Single host, no external network: the master runs in non-cluster mode, which +# self-seeds pekko://Amber@localhost:2552; the worker is launched with NO +# --serverAddr, so it defaults its seed to localhost:2552 and skips the +# getNodeIpAddress -> http://checkip.amazonaws.com lookup (that only fires when +# --serverAddr is supplied). +# +# Usage: +# smoke-cluster.sh [join_timeout] +# +# Requirements: run from the checkout root -- the master resolves its config via +# Utils.amberHomePath -> ./amber/src/main/resources/computing-unit-master-config.yml +# -- with postgres already provisioned (the master calls SqlServer.initConnection). + +set -euo pipefail + +master_glob="${1:?usage: smoke-cluster.sh [join_timeout]}" +worker_glob="${2:?worker launcher glob required}" +http_port="${3:?master http port required}" +join_timeout="${4:-40}" +master_ready_timeout="${SMOKE_MASTER_READY_TIMEOUT:-60}" + +PHRASE_JOINED='Now we have 2 nodes in the cluster' + +# Resolve a (globbed) launcher to exactly one executable -- same contract as +# smoke-boot.sh: erroring on >1 match avoids booting the wrong binary. +RESOLVED="" +resolve_launcher() { + local glob="$1" role="$2" matches count + # shellcheck disable=SC2086 # intentional: $glob must glob-expand here + matches="$(ls -d $glob 2>/dev/null || true)" + count="$(printf '%s' "$matches" | grep -c . || true)" + if [[ "$count" -eq 0 ]]; then + echo "::error::smoke-cluster: $role launcher not found: $glob"; return 1 + fi + if [[ "$count" -gt 1 ]]; then + echo "::error::smoke-cluster: $role launcher glob matched $count files, expected exactly 1: $glob"; return 1 + fi + if [[ ! -x "$matches" ]]; then + echo "::error::smoke-cluster: $role launcher not executable: $matches"; return 1 + fi + RESOLVED="$matches" +} +resolve_launcher "$master_glob" master || exit 1 +master_bin="$RESOLVED" +resolve_launcher "$worker_glob" worker || exit 1 +worker_bin="$RESOLVED" + +port_open() { + # Probe 127.0.0.1 explicitly (not "localhost", which can resolve to ::1 first + # while the JVM binds IPv4, giving a false "not listening"). + if command -v nc >/dev/null 2>&1; then + nc -z 127.0.0.1 "$http_port" >/dev/null 2>&1 + else + (exec 3<>"/dev/tcp/127.0.0.1/$http_port") 2>/dev/null + fi +} +if port_open; then + echo "::error::smoke-cluster: port $http_port is already in use before launching the master" + exit 1 +fi + +master_log="$(mktemp)" +worker_log="$(mktemp)" +master_pid="" +worker_pid="" + +alive() { kill -0 "$1" 2>/dev/null; } + +dump_logs() { + echo "----- master (last 120 lines) -----"; tail -n 120 "$master_log" 2>/dev/null || true + echo "----- worker (last 120 lines) -----"; tail -n 120 "$worker_log" 2>/dev/null || true +} + +# Reap both JVMs (SIGTERM, grace, SIGKILL) and remove temp logs on any exit -- +# including a set -e abort -- so a failing run can't orphan a JVM or leak files. +# shellcheck disable=SC2329 # invoked indirectly via `trap cleanup EXIT` below +cleanup() { + for p in "$worker_pid" "$master_pid"; do [[ -n "$p" ]] && kill "$p" 2>/dev/null || true; done + for _ in $(seq 1 10); do + { [[ -z "$master_pid" ]] || ! alive "$master_pid"; } && + { [[ -z "$worker_pid" ]] || ! alive "$worker_pid"; } && break + sleep 1 + done + for p in "$worker_pid" "$master_pid"; do [[ -n "$p" ]] && kill -9 "$p" 2>/dev/null || true; done + wait 2>/dev/null || true + rm -f "$master_log" "$worker_log" +} +trap cleanup EXIT + +# 1) Master: non-cluster mode self-seeds localhost:2552 and serves Dropwizard on +# :http_port. Launch from the current dir (checkout root) so its config resolves. +echo "smoke-cluster: launching master '$master_bin'" +"$master_bin" >"$master_log" 2>&1 & +master_pid=$! + +# Wait for the master to be ready: its HTTP port coming up implies the actor +# system bound :2552 and DB init finished. +ready=0 +for ((i = 0; i < master_ready_timeout; i++)); do + if port_open; then ready=1; break; fi + if ! alive "$master_pid"; then + echo "::error::smoke-cluster: master exited before listening on :$http_port"; dump_logs; exit 1 + fi + sleep 1 +done +if [[ "$ready" != 1 ]]; then + echo "::error::smoke-cluster: master did not listen on :$http_port within ${master_ready_timeout}s"; dump_logs; exit 1 +fi + +# NB: we deliberately do NOT wait here for the master's own "1 nodes" self-join +# line. The master opens :8085 as soon as its actor system + DB are up, but its +# own MemberUp is gated by cluster.conf's leader-actions-interval (~10s) and can +# land *after* the port is already open -- keying on it here caused a false +# failure. The worker-join assertion below is the real signal; a readiness-log +# drift is reported there instead. +echo "smoke-cluster: master up (listening on :$http_port); launching worker '$worker_bin'" + +# 2) Worker: no --serverAddr => seed localhost:2552, random remote port, no egress. +"$worker_bin" >"$worker_log" 2>&1 & +worker_pid=$! + +# 3) Assert the worker joins: the master observes a 2-member cluster. Poll a +# deliberate readiness line (not a crash-scan); fail fast if either JVM dies. +joined=0 +for ((i = 0; i < join_timeout; i++)); do + if grep -Fq "$PHRASE_JOINED" "$master_log"; then joined=1; break; fi + if ! alive "$master_pid"; then + echo "::error::smoke-cluster: master exited before the worker joined"; dump_logs; exit 1 + fi + if ! alive "$worker_pid"; then + echo "::error::smoke-cluster: worker exited before joining the cluster"; dump_logs; exit 1 + fi + sleep 1 +done + +if [[ "$joined" == 1 ]]; then + echo "smoke-cluster: OK -- worker joined; master observes 2 nodes in the cluster" + grep -F "$PHRASE_JOINED" "$master_log" | tail -n1 + exit 0 +fi +echo "::error::smoke-cluster: worker did not join the cluster within ${join_timeout}s" +if ! grep -Fq "nodes in the cluster" "$master_log"; then + echo "::error::smoke-cluster: the master logged no 'N nodes in the cluster' line at all -- ClusterListener's readiness log may have changed; this test keys on it" +fi +dump_logs +exit 1 diff --git a/.github/scripts/test_smoke_cluster.sh b/.github/scripts/test_smoke_cluster.sh new file mode 100755 index 00000000000..c4d3ee847c4 --- /dev/null +++ b/.github/scripts/test_smoke_cluster.sh @@ -0,0 +1,159 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# End-to-end regression tests for smoke-cluster.sh, driven by fake master/worker +# launchers (no real amber dist). These guard the harness's own decision logic: +# - worker joins (master logs the 2-node readiness line) -> PASS +# - worker exits before joining -> FAIL ("worker exited...") +# - worker never joins within the timeout -> FAIL ("did not join...") +# - master dies during the join wait -> FAIL ("master exited...joined") +# - master logs no "N nodes" line at all (readiness drift) -> FAIL (timeout + hint) +# - master crashes before it listens -> FAIL ("...before listening") +# The failure cases assert the SPECIFIC error message, so a future edit that +# deletes a fast-fail branch (and lets the case pass only via the slow timeout) +# turns the test red instead of silently passing. The happy path ties the join +# signal to the worker actually running, so it can't pass if the worker is never +# launched. The real cluster join is exercised against the packaged dist in +# amber-integration. See https://github.com/apache/texera/issues/6523. + +set -uo pipefail + +command -v python3 >/dev/null || { echo "python3 is required to run these tests" >&2; exit 1; } + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +smoke="$script_dir/smoke-cluster.sh" +work="$(mktemp -d 2>/dev/null || mktemp -d -t smoke-cluster)" +trap 'rm -rf "$work"' EXIT +rc=0 + +PHRASE_SELF='Now we have 1 nodes in the cluster' +PHRASE_JOINED='Now we have 2 nodes in the cluster' + +free_port() { + python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1",0)); print(s.getsockname()[1]); s.close()' +} +pass() { echo "ok: $1"; } +failed() { echo "FAIL: $1"; rc=1; } + +py_listen() { # emit a python one-liner that binds $1 and holds it for $2 seconds + echo "exec python3 -c \"import socket,time; s=socket.socket(); s.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1); s.bind(('127.0.0.1',$1)); s.listen(); time.sleep($2)\"" +} + +# Fake master: binds $port so smoke-cluster's port-wait succeeds. $self=y prints +# the self-join line. $trigger is "n" (never emit the joined line), a number N +# (emit it N seconds after start), or "wait:" (emit it only once +# exists -- used to couple the join to the worker actually running). $hold is how +# long to hold the port (short => master dies mid-run). +# $1=path $2=port $3=self(y/n) $4=trigger $5=hold_secs +make_master() { + local path="$1" port="$2" self="$3" trigger="$4" hold="$5" + { + echo '#!/usr/bin/env bash' + [[ "$self" == y ]] && echo "echo '$PHRASE_SELF'" + case "$trigger" in + n) : ;; + wait:*) echo "( while [ ! -f '${trigger#wait:}' ]; do sleep 0.2; done; echo '$PHRASE_JOINED' ) &" ;; + *) echo "( sleep $trigger; echo '$PHRASE_JOINED' ) &" ;; + esac + py_listen "$port" "$hold" + } >"$path" + chmod +x "$path" +} + +# Fake worker: sleeps (healthy), crashes on boot, or touches a signal file (to +# trigger a coupled master) then sleeps. $1=path $2=sleep|crash|touch: +make_worker() { + case "$2" in + crash) printf '#!/usr/bin/env bash\necho "worker boom" >&2\nexit 1\n' >"$1" ;; + touch:*) printf '#!/usr/bin/env bash\ntouch "%s"\nexec sleep 120\n' "${2#touch:}" >"$1" ;; + *) printf '#!/usr/bin/env bash\nexec sleep 120\n' >"$1" ;; + esac + chmod +x "$1" +} + +# Assert smoke-cluster FAILS and its output contains $needle (locks in the branch +# that produces that message). $1=desc $2=needle, remaining args = smoke-cluster argv. +assert_fail_msg() { + local desc="$1" needle="$2"; shift 2 + local out + if out="$("$smoke" "$@" 2>&1)"; then + failed "$desc: expected FAIL but it passed" + elif printf '%s\n' "$out" | grep -Fq "$needle"; then + pass "$desc" + else + failed "$desc: failed but message missing '$needle'"; printf '%s\n' "$out" | tail -n3 + fi +} + +# --- happy path: the worker triggers the join; master observes 2 nodes -> PASS. +# The joined line fires only after the worker touches the signal, so this cannot +# pass if smoke-cluster never launches the worker. --- +port="$(free_port)"; sig="$work/joined.signal" +make_master "$work/m_ok" "$port" y "wait:$sig" 120 +make_worker "$work/w_ok" "touch:$sig" +if "$smoke" "$work/m_ok" "$work/w_ok" "$port" 30 >/dev/null 2>&1; then + pass "worker joins cluster (master sees 2 nodes) -> OK" +else + failed "worker join should be OK" +fi + +# --- worker exits before joining -> FAIL fast (assert the fast-fail message) --- +port="$(free_port)" +make_master "$work/m_nojoin" "$port" y n 120 +make_worker "$work/w_crash" crash +assert_fail_msg \ + "worker exits before join -> FAIL fast" "worker exited before joining" \ + "$work/m_nojoin" "$work/w_crash" "$port" 15 + +# --- worker never joins within the timeout -> FAIL (assert the timeout message) --- +port="$(free_port)" +make_master "$work/m_nojoin2" "$port" y n 120 +make_worker "$work/w_sleep" sleep +assert_fail_msg \ + "worker never joins -> FAIL (timeout)" "did not join the cluster within" \ + "$work/m_nojoin2" "$work/w_sleep" "$port" 3 + +# --- master dies during the join wait -> FAIL (assert the master-death message) --- +port="$(free_port)" +make_master "$work/m_die" "$port" y n 4 +make_worker "$work/w_sleep_b" sleep +assert_fail_msg \ + "master dies during join wait -> FAIL" "master exited before the worker joined" \ + "$work/m_die" "$work/w_sleep_b" "$port" 20 + +# --- master logs no "N nodes" line at all (readiness log drift) -> FAIL with a hint --- +port="$(free_port)" +make_master "$work/m_drift" "$port" n n 120 +make_worker "$work/w_sleep2" sleep +assert_fail_msg \ + "master readiness log drift -> FAIL (timeout + drift hint)" "readiness log may have changed" \ + "$work/m_drift" "$work/w_sleep2" "$port" 3 + +# --- master crashes before it listens -> FAIL (assert the pre-listen message) --- +port="$(free_port)" +printf '#!/usr/bin/env bash\necho "master boom" >&2\nexit 1\n' >"$work/m_crash"; chmod +x "$work/m_crash" +make_worker "$work/w_sleep3" sleep +SMOKE_MASTER_READY_TIMEOUT=15 assert_fail_msg \ + "master crashes before listening -> FAIL" "master exited before listening" \ + "$work/m_crash" "$work/w_sleep3" "$port" 20 + +if [[ "$rc" -ne 0 ]]; then + echo "smoke-cluster regression tests FAILED" + exit 1 +fi +echo "smoke-cluster regression tests passed" diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1e0092fa4b6..0341af96202 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -639,19 +639,22 @@ jobs: # amber/src/main/resources/web-config.yml (port 8080). if: matrix.os == 'ubuntu-latest' run: .github/scripts/smoke-boot.sh "/tmp/dists/amber-*/bin/texera-web-application" 8080 - - name: Smoke-test computing-unit-master boots - # computing-unit-master boots on postgres only, like texera-web: main - # starts a LOCAL Pekko actor master (clusterMode defaults to false with - # no --cluster arg, so no external seed/discovery and the cluster-only - # checkip.amazonaws.com lookup is never reached), then run() calls - # SqlServer.initConnection against the postgres provisioned above. The - # boot-time result-store cleanup is gated on - # ApplicationConfig.cleanupAllExecutionResults, which defaults to false, - # so no iceberg / S3 access. Config resolves via Utils.amberHomePath to - # amber/src/main/resources/computing-unit-master-config.yml (port 8085). - # Reuses the amber dist built + unzipped above for the texera-web boot. + - name: Smoke-test computing-unit-master boots and the worker joins its cluster + # One check for the whole computing-unit cluster: boot a + # computing-unit-master and a no-arg computing-unit-worker from the + # packaged dist and assert the worker JOINS -- the master's ClusterListener + # logging "2 nodes in the cluster". This supersedes a standalone master + # boot check: smoke-cluster.sh already boots the master, waits for it to + # listen on :8085, and verifies it self-joins its 1-node cluster before the + # worker is launched, so a #6204-class linkage crash on master startup is + # still caught (reported as "master exited before listening"). The master + # runs non-cluster mode (self-seeds localhost:2552) and the worker is + # launched with no --serverAddr (joins :2552, skipping the + # checkip.amazonaws.com lookup), so no external network; it fails fast if + # either JVM dies. Reuses the amber dist built + unzipped above; postgres + # already provisioned. if: matrix.os == 'ubuntu-latest' - run: .github/scripts/smoke-boot.sh "/tmp/dists/amber-*/bin/computing-unit-master" 8085 + run: .github/scripts/smoke-cluster.sh "/tmp/dists/amber-*/bin/computing-unit-master" "/tmp/dists/amber-*/bin/computing-unit-worker" 8085 - name: Run Python integration tests # --junit-xml feeds the Test Analytics upload below. run: |