#!/usr/bin/env bash set -Eeuo pipefail umask 077 BASE="${ROUTER_OPS_BASE:-/opt/router-ops}" ROOT="${BUNDLE_ROOT:-$(cd "$(dirname "$0")/.." && pwd)}" SELFTEST="${ROUTER_EXACT_WORKER_SELFTEST:-false}" CASE="${ROUTER_P23A_SELFTEST_CASE:-pass}" INCOMING="${ROUTER_STEP_INCOMING:-/home/ops/incoming}" STEP=STEP_050M07P23A_R01_VM121_DOMAIN_V2_SCHEMA_DESIGN_AND_MIGRATION_FIXTURES VM121_BASE_COMMIT=d17c0864339bd59a444264ce9305c2a6e560a5bf VM121_BASE_TREE=d1aa54e5774f59bc7eb564bf974ea74e13862723 VM121_BASE_COUNT=42 VM121_EXPECTED_COUNT_AFTER=43 MIGRATION_SHA=56c1fe6fd85d4d8a7a048ccf4efc807c8f3b20f2c98b24d6f21e4230153a9d8d TS="$(date -u +%Y%m%d-%H%M%S)" RUN_ROOT="${ROUTER_P23A_RUN_ROOT:-$BASE/state/step-runs/$STEP}" RUN_DIR="$RUN_ROOT/${TS}_$$" CURRENT="$RUN_DIR/workflow-current.txt" EVENTS="$RUN_DIR/workflow-events.tsv" EVIDENCE="$RUN_DIR/evidence" REPORT="$RUN_DIR/report" PAYLOAD="$RUN_DIR/public-payload" FINAL_ENV="$RUN_DIR/final-publication.env" TARGET_DIR="$RUN_DIR/target" RESULT_ENV="$RUN_DIR/result.env" mkdir -p "$EVIDENCE" "$REPORT" "$PAYLOAD" "$TARGET_DIR" "$INCOMING" chmod 700 "$RUN_DIR" "$EVIDENCE" "$REPORT" "$PAYLOAD" "$TARGET_DIR" source "$BASE/lib/router-step-workflow.sh" source <("$BASE/bin/router-release-contract" env --root "$ROOT") router_step_workflow_init "$CURRENT" "$EVENTS" preflight fixture_and_source_apply machine_close postcheck report_publish archive CORE_CHANGE_COMPLETE=false SOURCE_FILE_INSTALLED=false ROLLBACK_STATUS=not_required STOP_DETAIL=unknown STOP_HANDLED=false STEP_REPORT_URL=unknown LATEST_ARCHIVE=not_created LATEST_ARCHIVE_SHA256=unknown REMOTE_STAGE=unknown REMOTE_PKG=unknown VM121_NEW_COMMIT=unknown VM121_NEW_TREE=unknown VM121_NEW_FILE_COUNT=unknown VM121_NEW_PUBLIC_URL=unknown field(){ local key="$1" file="$2"; awk -F= -v k="$key" '$1==k{print substr($0,index($0,"=")+1); exit}' "$file"; } write_result(){ local result="$1" rc="$2" tmp="${RESULT_ENV}.tmp.$$" { echo "RESULT=$result" echo "WORKER_RC=$rc" echo "CORE_CHANGE_COMPLETE=$CORE_CHANGE_COMPLETE" echo "SOURCE_FILE_INSTALLED=$SOURCE_FILE_INSTALLED" echo "ROLLBACK_STATUS=$ROLLBACK_STATUS" echo "STOP_DETAIL=$STOP_DETAIL" echo "VM121_NEW_COMMIT=$VM121_NEW_COMMIT" echo "VM121_NEW_TREE=$VM121_NEW_TREE" echo "VM121_NEW_FILE_COUNT=$VM121_NEW_FILE_COUNT" echo "VM121_NEW_PUBLIC_URL=$VM121_NEW_PUBLIC_URL" echo "STEP_REPORT_URL=$STEP_REPORT_URL" echo "LATEST_ARCHIVE=$LATEST_ARCHIVE" echo "LATEST_ARCHIVE_SHA256=$LATEST_ARCHIVE_SHA256" echo "RUN_DIR=$RUN_DIR" } > "$tmp" chmod 600 "$tmp"; mv "$tmp" "$RESULT_ENV" } archive_latest(){ local log="$EVIDENCE/archive.log" router_step_archive_latest_python "$BASE" "$INCOMING" "$(date -u +%Y%m%d-%H%M%S)" | tee "$log" LATEST_ARCHIVE="$(field LATEST_ARCHIVE "$log")" LATEST_ARCHIVE_SHA256="$(field LATEST_ARCHIVE_SHA256 "$log")" } cleanup_remote(){ [[ "$SELFTEST" == false && "$REMOTE_STAGE" != unknown ]] || return 0 set +e ssh -T -o BatchMode=yes -o ConnectTimeout=20 vm121 "rm -rf '$REMOTE_STAGE' '$REMOTE_PKG'" >/dev/null 2>&1 set -e } rollback_source(){ if [[ "$SOURCE_FILE_INSTALLED" != true || "$CORE_CHANGE_COMPLETE" == true ]]; then return 0; fi if [[ "$SELFTEST" == true ]]; then echo RESULT=PASS_P23A_VM121_SOURCE_ROLLBACK > "$EVIDENCE/rollback-source.txt" ROLLBACK_STATUS=PASS SOURCE_FILE_INSTALLED=false return 0 fi local rc=0 if ssh -T -o BatchMode=yes -o ConnectTimeout=20 vm121 "/bin/bash '$REMOTE_STAGE/vm121-p23a-domain-v2-prepare.sh' --rollback '$MIGRATION_SHA'" > "$EVIDENCE/rollback-source.txt" 2>&1; then rc=0; else rc=$?; fi if [[ "$rc" -eq 0 ]] && grep -Fqx 'RESULT=PASS_P23A_VM121_SOURCE_ROLLBACK' "$EVIDENCE/rollback-source.txt"; then ROLLBACK_STATUS=PASS; SOURCE_FILE_INSTALLED=false else ROLLBACK_STATUS=FAIL fi } write_stop_report(){ local rc="$1" line="$2" cmd="$3" cat > "$REPORT/report.txt" <> "$REPORT/report.txt" cp "$REPORT/report.txt" "$REPORT/verify.txt" cp "$ROOT/scripts/run-step.sh" "$REPORT/executed-step.sh" cp "$ROOT/release-info.json" "$REPORT/release-info.json" cp "$ROOT/rollback/README.md" "$REPORT/rollback.txt" } stop_handler(){ local rc="$1" line="$2" cmd="$3" trap - ERR [[ "$STOP_HANDLED" == false ]] || return "$rc" STOP_HANDLED=true set +e rollback_source cleanup_remote write_stop_report "$rc" "$line" "$cmd" if [[ "$SELFTEST" == true ]]; then DEST="${ROUTER_PUBLIC_ROOT:?}/latest/p23a-r01-$CASE-selftest"; rm -rf "$DEST"; mkdir -p "$DEST"; cp -a "$REPORT"/. "$DEST"/ STEP_REPORT_URL="selftest://p23a-r01-$CASE/" archive_latest >/dev/null 2>&1 else args=(); while IFS= read -r file; do args+=("$file" "$(basename "$file")"); done < <(find "$REPORT" -maxdepth 1 -type f -print | LC_ALL=C sort) router_step_publish_payload step050m07p23a_r01_vm121_domain_v2_schema_design_and_migration_fixtures_stop "$REPORT" "$PAYLOAD" "$CURRENT" "$EVENTS" "$FINAL_ENV" "${args[@]}" >/dev/null 2>&1 || true [[ -f "$FINAL_ENV" ]] && STEP_REPORT_URL="$(field PUBLIC_URL "$FINAL_ENV")" archive_latest >/dev/null 2>&1 fi set -e write_result STOP_P23A_VM121_DOMAIN_V2_SCHEMA_DESIGN_AND_MIGRATION_FIXTURES "$rc" return "$rc" } trap 'rc=$?; stop_handler "$rc" "$LINENO" "$BASH_COMMAND"; exit "$rc"' ERR write_selftest_source_pre(){ cat < "$EVIDENCE/pre-vm121-source.txt"; else ssh -T -o BatchMode=yes -o ConnectTimeout=20 vm121 /bin/bash -s > "$EVIDENCE/pre-vm121-source.txt" 2>&1 <<'VM121SRC' set -euo pipefail /usr/local/sbin/router-vm121-git-publish.sh --check VM121SRC fi "$ROOT/scripts/source-check-validate.sh" vm121 "$EVIDENCE/pre-vm121-source.txt" "$VM121_BASE_COMMIT" "$VM121_BASE_TREE" "$VM121_BASE_COUNT" > "$EVIDENCE/pre-vm121-source-validation.txt" } write_selftest_prepare(){ case "$CASE" in pass|fail-post) cat <&2; return 2 ;; esac } write_selftest_machine_close(){ if [[ "$CASE" == fail-post ]]; then echo RESULT=STOP_ROUTER_MACHINE_CLOSE_BACKEND_OR_PUBLICATION return 1 fi cat <<'EOF' RESULT=PASS_ROUTER_MACHINE_CLOSE MACHINE=vm121 DESCRIPTION=selftest COMMIT=1111111111111111111111111111111111111111 TREE=2222222222222222222222222222222222222222 GIT_FILE_COUNT=43 VM121_PUBLIC_URL=selftest://vm121-source-p23a/ EOF } write_selftest_post(){ cat <<'EOF' MODE=--check MACHINE_ID=vm121 LOCAL_HEAD=1111111111111111111111111111111111111111 REMOTE_HEAD=1111111111111111111111111111111111111111 STAGED_CHANGE_COUNT=0 SIMULATED_TREE_SHA=2222222222222222222222222222222222222222 SIMULATED_TREE_FILE_COUNT=43 RESULT=PASS_VM121_GIT_PUBLISH_CHECK MACHINE_ID=vm121 GIT_COMMIT=1111111111111111111111111111111111111111 GIT_TREE=2222222222222222222222222222222222222222 GIT_FILE_COUNT=43 COMMIT_CREATED=false PUSH_PERFORMED=false MIGRATION_SHA256=56c1fe6fd85d4d8a7a048ccf4efc807c8f3b20f2c98b24d6f21e4230153a9d8d ALEMBIC_VERSION=0002_live_drift_bridge LIVE_DOMAIN_V2_TABLE_COUNT=0 BACKEND_HEALTH=PASS EOF } router_step_phase_begin preflight collect_source_pre || { STOP_DETAIL=pre_vm121_source; false; } PKG="$ROOT/$ROUTER_RELEASE_TARGET_PACKAGE_PATH" "$ROOT/scripts/target-apply.sh" "$ROUTER_RELEASE_TARGET_PACKAGE_SHA256" "$PKG" "$TARGET_DIR" > "$EVIDENCE/target-extract.txt" router_step_phase_complete preflight exact_vm121_source_and_target_package_verified router_step_phase_begin fixture_and_source_apply apply_rc=0 if [[ "$SELFTEST" == true ]]; then if write_selftest_prepare > "$EVIDENCE/domain-v2-prepare.txt"; then apply_rc=0; else apply_rc=$?; fi else token="$(basename "$RUN_DIR" | tr -cd 'A-Za-z0-9._-')" REMOTE_STAGE="/root/router-step-p23a-$token" REMOTE_PKG="/root/router-step-p23a-$token.tar.gz" if ! ssh -T -o BatchMode=yes -o ConnectTimeout=20 vm121 "umask 077; cat > '$REMOTE_PKG'" < "$PKG"; then STOP_DETAIL=remote_package_transfer; false; fi if ! ssh -T -o BatchMode=yes -o ConnectTimeout=20 vm121 "rm -rf '$REMOTE_STAGE'; install -d -m 0700 '$REMOTE_STAGE'; tar -xzf '$REMOTE_PKG' -C '$REMOTE_STAGE'" > "$EVIDENCE/remote-stage.txt" 2>&1; then STOP_DETAIL=remote_package_extract; false; fi if ssh -T -o BatchMode=yes -o ConnectTimeout=20 vm121 "/bin/bash '$REMOTE_STAGE/vm121-p23a-domain-v2-prepare.sh' --apply '$REMOTE_STAGE/0003_domain_v2_schema.py'" > "$EVIDENCE/domain-v2-prepare.txt" 2>&1; then apply_rc=0; else apply_rc=$?; fi fi [[ "$(field SOURCE_FILE_INSTALLED "$EVIDENCE/domain-v2-prepare.txt")" == true ]] && SOURCE_FILE_INSTALLED=true || SOURCE_FILE_INSTALLED=false if [[ "$apply_rc" -ne 0 ]]; then STOP_DETAIL="fixture_or_apply_$(field FAILED_STAGE "$EVIDENCE/domain-v2-prepare.txt")"; false; fi "$ROOT/scripts/domain-v2-prepare-validate.sh" "$EVIDENCE/domain-v2-prepare.txt" > "$EVIDENCE/domain-v2-prepare-validation.txt" || { STOP_DETAIL=prepare_semantics; false; } CORE_CHANGE_COMPLETE=true router_step_phase_complete fixture_and_source_apply current_and_clean_fixtures_passed_migration_source_installed_live_db_unchanged router_step_phase_begin machine_close close_rc=0 if [[ "$SELFTEST" == true ]]; then if write_selftest_machine_close > "$EVIDENCE/vm121-machine-close.txt"; then close_rc=0; else close_rc=$?; fi else if "$BASE/bin/router-machine-close-safe" vm121 "$STEP: add tested 0003 Domain V2 schema migration; live DB remains 0002" > "$EVIDENCE/vm121-machine-close.txt" 2>&1; then close_rc=0; else close_rc=$?; fi fi if [[ "$close_rc" -ne 0 ]]; then STOP_DETAIL=machine_close_after_core; false; fi "$ROOT/scripts/machine-close-validate.sh" "$EVIDENCE/vm121-machine-close.txt" > "$EVIDENCE/vm121-machine-close-validation.txt" || { STOP_DETAIL=machine_close_semantics; false; } VM121_NEW_COMMIT="$(field VM121_NEW_COMMIT "$EVIDENCE/vm121-machine-close-validation.txt")" VM121_NEW_TREE="$(field VM121_NEW_TREE "$EVIDENCE/vm121-machine-close-validation.txt")" VM121_NEW_FILE_COUNT="$(field VM121_NEW_FILE_COUNT "$EVIDENCE/vm121-machine-close-validation.txt")" VM121_NEW_PUBLIC_URL="$(field VM121_NEW_PUBLIC_URL "$EVIDENCE/vm121-machine-close-validation.txt")" router_step_phase_complete machine_close vm121_machine_git_closed_and_published router_step_phase_begin postcheck if [[ "$SELFTEST" == true ]]; then write_selftest_post > "$EVIDENCE/post-vm121.txt"; else ssh -T -o BatchMode=yes -o ConnectTimeout=20 vm121 /bin/bash -s > "$EVIDENCE/post-vm121.txt" 2>&1 <<'VM121POST' set -euo pipefail /usr/local/sbin/router-vm121-git-publish.sh --check PG=wgaccess-postgres BACKEND=wgaccess-backend PGUSER="$(docker exec "$PG" sh -lc 'printf %s "$POSTGRES_USER"')" PGDB="$(docker exec "$PG" sh -lc 'printf %s "$POSTGRES_DB"')" q(){ docker exec "$PG" psql -v ON_ERROR_STOP=1 -U "$PGUSER" -d "$PGDB" -Atqc "$1"; } echo "MIGRATION_SHA256=$(sha256sum /opt/wg-access/backend/alembic/versions/0003_domain_v2_schema.py | awk '{print $1}')" echo "ALEMBIC_VERSION=$(q 'SELECT version_num FROM alembic_version')" echo "LIVE_DOMAIN_V2_TABLE_COUNT=$(q \"SELECT count(*) FROM information_schema.tables WHERE table_schema='public' AND table_name IN ('plans','access_grants','access_grant_protocol_limits','invites','invite_redemptions','magic_link_tokens','auth_sessions','connection_profiles','peer_credentials','audit_events')\")" if docker exec -i "$BACKEND" python - <<'PY' >/dev/null 2>&1 import urllib.request for url in ('http://127.0.0.1:8000/health','http://127.0.0.1:8000/health/db'): with urllib.request.urlopen(url, timeout=5) as r: assert r.status == 200 PY then echo BACKEND_HEALTH=PASS; else echo BACKEND_HEALTH=FAIL; exit 1; fi VM121POST fi "$ROOT/scripts/source-check-validate.sh" vm121 "$EVIDENCE/post-vm121.txt" "$VM121_NEW_COMMIT" "$VM121_NEW_TREE" "$VM121_NEW_FILE_COUNT" > "$EVIDENCE/post-vm121-source-validation.txt" || { STOP_DETAIL=post_source; false; } [[ "$(field MIGRATION_SHA256 "$EVIDENCE/post-vm121.txt")" == "$MIGRATION_SHA" ]] || { STOP_DETAIL=post_migration_sha; false; } [[ "$(field ALEMBIC_VERSION "$EVIDENCE/post-vm121.txt")" == 0002_live_drift_bridge ]] || { STOP_DETAIL=post_live_alembic_changed; false; } [[ "$(field LIVE_DOMAIN_V2_TABLE_COUNT "$EVIDENCE/post-vm121.txt")" == 0 ]] || { STOP_DETAIL=post_live_domain_tables; false; } [[ "$(field BACKEND_HEALTH "$EVIDENCE/post-vm121.txt")" == PASS ]] || { STOP_DETAIL=post_backend_health; false; } cleanup_remote router_step_phase_complete postcheck source_clean_live_db_0002_backend_healthy cat > "$REPORT/report.txt" <