#!/usr/bin/env bash set -Eeuo pipefail umask 077 [ "$(id -un)" = "ops" ] || { echo "STOP: запускать на router-ops пользователем ops" exit 1 } STEP="STEP_050M07D7_AUDIT_DISK_ACTIVE_IFACE_AND_RUNTIME_DIVERGENCE" PASS_DECISION="PASS_${STEP}" TOKEN="e94a0859747d7b96f29c7fdafc2d0351ba603bb0a7e9e5a4" PUBLIC_BASE="https://helena-background-beam-harry.trycloudflare.com/r/${TOKEN}" ROOT="/opt/router-ops" PUBROOT="${ROOT}/public/r/${TOKEN}" D5_DIR="${PUBROOT}/20260711-212723_step050m07d5_final_cli_discovery_and_pre_night_freeze" D6_DIR="${PUBROOT}/20260712-071025_step050m07d6_post_scheduled_refresh_readonly_comparison" M07D_DIR="${PUBROOT}/20260711-210850_step050m07d_classify_false_success_and_egress1_failure" ARCHITECTURE_PLAN="${PUBLIC_BASE}/20260711-181158_local_architecture_plan_vm101_autonomous_hmn_recovery/" XS_MAP="${PUBLIC_BASE}/20260711-120734_xs_map_audit_repair_publish/" GLOBAL_PROJECT_PLAN="${PUBLIC_BASE}/20260711-123348_global_project_plan_wg_paid/" TS="$(date -u +%Y%m%d-%H%M%S)" SLUG="${TS}_step050m07d7_audit_disk_active_iface_and_runtime_divergence" REPORT_DIR="${PUBROOT}/${SLUG}" TRYCF_REPORT="${PUBLIC_BASE}/${SLUG}/" REPORT_TXT="${TRYCF_REPORT}report.txt" FACTS_JSON="${TRYCF_REPORT}facts.json" mkdir -p "$REPORT_DIR" "$REPORT_DIR/sources" # Сохраняем точный STEP первым. cp -a "$0" "$REPORT_DIR/step.sh" chmod 600 "$REPORT_DIR/step.sh" CURRENT_STAGE="initialization" LAST_SUCCESS="step_saved" VM101_RC="NOT_RUN" stage() { CURRENT_STAGE="$1" echo echo ">>> [$1] $2" date -u '+ utc=%Y-%m-%dT%H:%M:%SZ' } mark_success() { LAST_SUCCESS="$1" } print_links() { echo echo "TRYCF_REPORT=$TRYCF_REPORT" echo "REPORT_TXT=$REPORT_TXT" echo "FACTS_JSON=$FACTS_JSON" echo "ARCHITECTURE_PLAN=$ARCHITECTURE_PLAN" echo "XS_MAP=$XS_MAP" echo "GLOBAL_PROJECT_PLAN=$GLOBAL_PROJECT_PLAN" } create_index() { cat > "$REPORT_DIR/index.html" < ${STEP}

${STEP}

EOF } write_stop() { local reason="$1" local rc="$2" local line="$3" cat > "$REPORT_DIR/report.txt" < "$REPORT_DIR/facts.json" <<'PY' import json import sys ( step, reason, rc, line, stage, last_success, vm101_rc, report, report_txt, facts_json, architecture, xs_map, global_plan, ) = sys.argv[1:] print(json.dumps({ "schema": "router-step-facts-v1", "step": step, "assessment": { "decision": f"STOP_{step}_{reason}", "all_ok": False, "error_rc": int(rc), "error_line": int(line), "failed_stage": stage, "last_success": last_success, "vm101_rc": vm101_rc, }, "safety": { "read_only": True, "production_modified": False, }, "publish": { "trycf_report": report, "report_txt": report_txt, "facts_json": facts_json, "architecture_plan": architecture, "xs_map": xs_map, "global_project_plan": global_plan, }, }, ensure_ascii=False, indent=2)) PY create_index find "$REPORT_DIR" -type f ! -name SHA256SUMS -print0 | sort -z | xargs -0 sha256sum > "$REPORT_DIR/SHA256SUMS" print_links exit "$rc" } fatal() { local reason="$1" local rc="${2:-1}" local line="${3:-$LINENO}" trap - ERR write_stop "$reason" "$rc" "$line" } trap 'rc=$?; fatal "UNEXPECTED_ERROR" "$rc" "$LINENO"' ERR stage "01/05" "Проверяю D5, D6 и M07D" for required in \ "$D5_DIR/report.txt" \ "$D5_DIR/exact-baseline.json" \ "$D6_DIR/report.txt" \ "$D6_DIR/facts.json" \ "$D6_DIR/vm101.txt" \ "$M07D_DIR/source-contract.json" \ "$M07D_DIR/runner-source.txt" \ "$M07D_DIR/rebalance-apply-source.txt" \ "$M07D_DIR/planner-source.txt" do [ -s "$required" ] || { echo "MISSING_REQUIRED=$required" fatal "REQUIRED_ARTIFACT_MISSING" 2 "$LINENO" } done grep -Fq \ "decision=PASS_STEP_050M07D6_POST_SCHEDULED_REFRESH_READONLY_COMPARISON" \ "$D6_DIR/report.txt" || fatal "D6_PASS_NOT_PROVEN" 3 "$LINENO" cp -a "$D5_DIR/exact-baseline.json" \ "$REPORT_DIR/sources/d5-exact-baseline.json" cp -a "$D6_DIR/facts.json" \ "$REPORT_DIR/sources/d6-facts.json" cp -a "$D6_DIR/vm101.txt" \ "$REPORT_DIR/sources/d6-vm101.txt" cp -a "$M07D_DIR/source-contract.json" \ "$REPORT_DIR/sources/m07d-source-contract.json" cp -a "$M07D_DIR/runner-source.txt" \ "$REPORT_DIR/sources/m07d-runner-source.txt" cp -a "$M07D_DIR/rebalance-apply-source.txt" \ "$REPORT_DIR/sources/m07d-rebalance-apply-source.txt" cp -a "$M07D_DIR/planner-source.txt" \ "$REPORT_DIR/sources/m07d-planner-source.txt" mark_success "previous_evidence_verified" stage "02/05" "Публикую точный read-only VM101 extractor" cat > "$REPORT_DIR/vm101.sh" <<'VM101' #!/bin/sh set -u umask 077 AMNEZIAWG="/usr/bin/amneziawg" REFRESH="/root/hmn/hmn-refresh-pool-safe.sh" DOWNLOAD="/root/hmn/hmn-download-all-awg.sh" PLANNER="/usr/local/sbin/router-egress-hmn-plan-top5.sh" RUNNER="/usr/local/sbin/router-egress-emergency-refresh.sh" SCHEDULED_LOG="/root/hmn/logs/refresh-pool-safe-20260712-042000-16148.log" SCHEDULED_BACKUP="/root/hmn/backups/refresh-safe-before-20260712-042000-16148" fact() { printf '__FACT__ %s=%s\n' "$1" "$2" } block() { name="$1" shift echo "__BLOCK_BEGIN__ $name" "$@" 2>&1 || true echo "__BLOCK_END__ $name" } strict_iface() { interface="$1" attempt=1 while [ "$attempt" -le 3 ]; do if ping -I "$interface" -c 1 -W 3 1.1.1.1 \ >/dev/null 2>&1 then return 0 fi attempt=$((attempt + 1)) sleep 1 done return 1 } for required in \ "$AMNEZIAWG" \ "$REFRESH" \ "$DOWNLOAD" \ "$PLANNER" \ "$RUNNER" do [ -f "$required" ] || { echo "__ERROR__ source_missing=$required" exit 21 } done fact read_only true fact snapshot_epoch "$(date +%s)" fact vm101_utc "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" echo "__TRACE__ stage=disk" block filesystem_usage df -Pk block hmn_directory_sizes sh -c ' du -sk \ /root/hmn \ /root/hmn/backups \ /root/hmn/configs \ /root/hmn/cache \ /root/hmn/logs \ /root/hmn/runs \ /var/lib/router-egress-recovery \ /tmp \ 2>/dev/null | sort -n ' block largest_hmn_objects sh -c ' find /root/hmn /var/lib/router-egress-recovery \ -xdev \ -type f \ -exec du -k {} \; \ 2>/dev/null | sort -n | tail -n 120 ' block backup_directories sh -c ' find /root/hmn/backups \ -mindepth 1 \ -maxdepth 1 \ -type d \ -exec du -sk {} \; \ 2>/dev/null | sort -n ' echo "__TRACE__ stage=scheduled_failure" block scheduled_cron_log sh -c ' grep -A300 -B20 \ "2026-07-12T04:20:00+00:00 === hmn-refresh-pool-safe start ===" \ /root/hmn/logs/hmn-refresh-pool-cron.log \ 2>/dev/null || true ' block scheduled_detail_log cat "$SCHEDULED_LOG" block scheduled_backup_listing sh -c " echo 'path=$SCHEDULED_BACKUP' if [ -d '$SCHEDULED_BACKUP' ]; then find '$SCHEDULED_BACKUP' \ -maxdepth 3 \ -printf '%y %s %p\n' \ 2>/dev/null | sort else echo BACKUP_DIRECTORY_NOT_PRESENT fi " echo "__TRACE__ stage=runtime" HEALTHY=0 for interface in vpn1 vpn2 vpn3 vpn4 vpn5; do if strict_iface "$interface"; then strict=true HEALTHY=$((HEALTHY + 1)) else strict=false fi fact "strict.${interface}" "$strict" line="$( "$AMNEZIAWG" show "$interface" dump 2>/dev/null | sed -n '2p' )" endpoint="$( printf '%s\n' "$line" | cut -f3 )" handshake="$( printf '%s\n' "$line" | cut -f5 )" fact "runtime_endpoint.${interface}" "${endpoint:-UNRESOLVED}" fact "runtime_handshake.${interface}" "${handshake:-0}" done fact healthy_slots "$HEALTHY" block amneziawg_all_dump "$AMNEZIAWG" show all dump block planner "$PLANNER" block runner_dry_run "$RUNNER" --dry-run block uci_endpoint_state sh -c ' uci -q show network 2>/dev/null | grep -E \ "^network\.vpn[1-5]=|wireguard_vpn[1-5].*endpoint_(host|port)=" | grep -vE \ "private_key|preshared_key" || true ' block network_config_relevant sh -c ' grep -nE \ "config interface .vpn[1-5].|config wireguard_.vpn[1-5].|option endpoint_(host|port)" \ /etc/config/network \ 2>/dev/null || true ' echo "__TRACE__ stage=source_contract" block active_interface_detection sh -c ' grep -RniE \ "active VPN interface|active_iface|table 200|ip route.*200|default.*vpn|vpn[1-5].*active" \ /root/hmn/*.sh \ /usr/local/sbin/router-egress-*.sh \ 2>/dev/null || true ' block refresh_source_numbered sh -c ' nl -ba /root/hmn/hmn-refresh-pool-safe.sh 2>/dev/null | sed -n "100,260p" ' block download_source_numbered sh -c ' nl -ba /root/hmn/hmn-download-all-awg.sh 2>/dev/null | sed -n "1,280p" ' block planner_source_contract sh -c ' grep -nE \ "current|endpoint|uci|amneziawg|wg show|network|dump" \ /usr/local/sbin/router-egress-hmn-plan-top5.sh \ 2>/dev/null || true ' block apply_source_contract sh -c ' grep -nE \ "slot_apply_failed|commit_failed|apply_ok|adapter|current|endpoint|exit|return" \ /usr/local/sbin/router-egress-hmn-rebalance-top5-apply.sh \ 2>/dev/null || true ' echo "__TRACE__ stage=repair_history" block emergency_log_recent sh -c ' tail -n 1500 \ /var/log/router-egress-emergency-refresh.log \ 2>/dev/null || true ' block repair_log_recent sh -c ' logread 2>/dev/null | grep -Ei \ "vpn[1-5]|health-repair|endpoint|repair|quarantine|rebalance" | tail -n 1500 || true ' block recent_recovery_artifacts sh -c ' find /var/lib/router-egress-recovery \ -maxdepth 5 \ -type f \ -mmin -1440 \ -exec ls -l {} \; \ 2>/dev/null | sort ' fact refresh_ran false fact rebalance_ran false fact network_changed false fact state_changed false fact services_changed false fact plan_changed false fact direct_failopen_changed false echo "__TRACE__ stage=complete" exit 0 VM101 chmod 600 "$REPORT_DIR/vm101.sh" sh -n "$REPORT_DIR/vm101.sh" mark_success "readonly_extractor_published" stage "03/05" "Собираю точные причины на VM101" if ssh pve-mgts \ "ssh \ -o BatchMode=yes \ -o ConnectTimeout=8 \ -o ServerAliveInterval=20 \ -o ServerAliveCountMax=6 \ -o StrictHostKeyChecking=no \ -o UserKnownHostsFile=/dev/null \ -i /root/.ssh/pve_to_openwrt_mgts_ed25519 \ root@10.71.100.2 \ 'sh -s'" \ < "$REPORT_DIR/vm101.sh" \ > >(tee "$REPORT_DIR/vm101.txt") \ 2> >(tee "$REPORT_DIR/vm101.stderr" >&2) then VM101_RC=0 else VM101_RC=$? fi [ "$VM101_RC" -eq 0 ] || fatal "VM101_READONLY_AUDIT_FAILED" "$VM101_RC" "$LINENO" grep -Fq "__TRACE__ stage=complete" "$REPORT_DIR/vm101.txt" || fatal "REMOTE_COMPLETION_MARKER_MISSING" 5 "$LINENO" mark_success "vm101_evidence_collected" stage "04/05" "Классифицирую root causes" python3 - \ "$D5_DIR/exact-baseline.json" \ "$REPORT_DIR/vm101.txt" \ "$REPORT_DIR/vm101.stderr" \ "$REPORT_DIR" <<'PY' import json import re import sys from pathlib import Path baseline_path, runtime_path, stderr_path, report_dir = sys.argv[1:] root = Path(report_dir) before = json.loads( Path(baseline_path).read_text(encoding="utf-8") ) text = Path(runtime_path).read_text( encoding="utf-8", errors="replace", ) stderr = Path(stderr_path).read_text( encoding="utf-8", errors="replace", ) facts = {} for line in text.splitlines(): if not line.startswith("__FACT__ "): continue payload = line[len("__FACT__ "):] if "=" in payload: key, value = payload.split("=", 1) facts[key] = value def block(name: str) -> str: match = re.search( rf"^__BLOCK_BEGIN__ {re.escape(name)}\n" rf"(.*?)" rf"^__BLOCK_END__ {re.escape(name)}$", text, re.MULTILINE | re.DOTALL, ) return match.group(1) if match else "" planner_text = block("planner") planner = json.loads(planner_text) interfaces = ["vpn1", "vpn2", "vpn3", "vpn4", "vpn5"] runtime_endpoints = { iface: facts.get( f"runtime_endpoint.{iface}", "UNRESOLVED", ) for iface in interfaces } planner_endpoints = { item["iface"]: item.get("current") for item in planner.get("plan", []) if item.get("iface") in interfaces } divergence = { iface: { "runtime": runtime_endpoints.get(iface), "planner": planner_endpoints.get(iface), "different": runtime_endpoints.get(iface) != planner_endpoints.get(iface), } for iface in interfaces } divergent_interfaces = [ iface for iface, values in divergence.items() if values["different"] ] scheduled_log = ( block("scheduled_cron_log") + "\n" + block("scheduled_detail_log") ) disk_text = ( block("filesystem_usage") + "\n" + block("hmn_directory_sizes") + "\n" + block("largest_hmn_objects") + "\n" + block("backup_directories") ) backup_listing = block("scheduled_backup_listing") active_contract = block("active_interface_detection") repair_history = ( block("emergency_log_recent") + "\n" + block("repair_log_recent") ) enospc = "No space left on device" in scheduled_log active_detection_failed = ( "не нашёл active VPN interface" in scheduled_log ) old_pool_validation_failed = ( "validate_current_pool_rc=1" in scheduled_log or "old/current pool validation failed" in scheduled_log ) rollback_backup_incomplete = ( "no usable old latest target" in scheduled_log or "configs-latest-readlink.txt" not in backup_listing or "ok-awg1-strict-foreign-latest.tsv" not in backup_listing ) health = { "five_runtime_endpoints": all( value not in {"", "UNRESOLVED", "(none)"} for value in runtime_endpoints.values() ), "five_strict_slots": int(facts.get("healthy_slots", "0")) == 5, "read_only": facts.get("read_only") == "true" and facts.get("refresh_ran") == "false" and facts.get("rebalance_ran") == "false" and facts.get("network_changed") == "false" and facts.get("state_changed") == "false" and facts.get("services_changed") == "false", } pre_night_runtime_known = ( before.get("endpoint_source") not in { "live router-egress-hmn-plan-top5 current", "live planner current fields", } ) classification = { "scheduled_refresh": "FAILED_BEFORE_POOL_PUBLICATION", "disk": "BACKUP_ENOSPC" if enospc else "ENOSPC_NOT_CONFIRMED", "active_interface": "DETECTION_FAILED" if active_detection_failed else "FAILURE_NOT_CONFIRMED", "fallback_validation": "FAILED" if old_pool_validation_failed else "NOT_CONFIRMED", "rollback": "INCOMPLETE_BACKUP_NO_USABLE_LATEST" if rollback_backup_incomplete else "BACKUP_APPEARS_USABLE", "planner_runtime": ( "DIVERGED_" + "_".join(divergent_interfaces) ) if divergent_interfaces else "MATCHED", "overnight_endpoint_delta": ( "NOT_PROVABLE_FROM_D5_" "BECAUSE_D5_USED_PLANNER_NOT_RUNTIME" if not pre_night_runtime_known else "COMPARABLE" ), } assessment = { "all_ok": all(health.values()), "decision": "PASS_STEP_050M07D7_AUDIT_DISK_ACTIVE_IFACE_AND_RUNTIME_DIVERGENCE", "classification": classification, "health": health, "runtime_endpoints": runtime_endpoints, "planner_endpoints": planner_endpoints, "divergence": divergence, "divergent_interfaces": divergent_interfaces, "evidence": { "enospc": enospc, "active_detection_failed": active_detection_failed, "old_pool_validation_failed": old_pool_validation_failed, "rollback_backup_incomplete": rollback_backup_incomplete, "active_contract_found": bool(active_contract.strip()), "repair_history_captured": bool(repair_history.strip()), }, "safety": { "read_only": True, "production_modified": False, }, "required_repairs_before_next_refresh": [ "Add disk-space preflight and non-root/surgical backup.", "Replace table-200 active-interface detection with a proven healthy AmneziaWG bootstrap interface.", "Make planner current endpoints authoritative from amneziawg runtime rather than stale config.", "Make apply return non-zero on commit_failed and make runner validate JSON apply_ok/decision.", "Refuse rollback when backup manifest is incomplete.", ], } (root / "runtime-divergence.json").write_text( json.dumps( { "runtime": runtime_endpoints, "planner": planner_endpoints, "comparison": divergence, "divergent_interfaces": divergent_interfaces, }, ensure_ascii=False, indent=2, ) + "\n", encoding="utf-8", ) (root / "assessment.json").write_text( json.dumps( assessment, ensure_ascii=False, indent=2, ) + "\n", encoding="utf-8", ) (root / "disk-analysis.txt").write_text( disk_text, encoding="utf-8", ) (root / "failure-contract.txt").write_text( "\n".join([ "=== SCHEDULED FAILURE ===", scheduled_log, "", "=== BACKUP LISTING ===", backup_listing, "", "=== ACTIVE INTERFACE CONTRACT ===", active_contract, "", "=== REPAIR HISTORY ===", repair_history, ]), encoding="utf-8", ) if not all(health.values()): raise SystemExit( "current baseline unhealthy: " + ",".join( name for name, value in health.items() if not value ) ) PY mark_success "root_causes_classified" stage "05/05" "Публикую итог аудита" CLASSIFICATION="$( python3 - "$REPORT_DIR/assessment.json" <<'PY' import json import sys data = json.load(open(sys.argv[1], encoding="utf-8")) print(json.dumps(data["classification"], ensure_ascii=False)) PY )" RUNTIME_ENDPOINTS="$( python3 - "$REPORT_DIR/assessment.json" <<'PY' import json import sys data = json.load(open(sys.argv[1], encoding="utf-8")) print(", ".join( f"{name}={value}" for name, value in sorted(data["runtime_endpoints"].items()) )) PY )" DIVERGENT="$( python3 - "$REPORT_DIR/assessment.json" <<'PY' import json import sys data = json.load(open(sys.argv[1], encoding="utf-8")) values = data["divergent_interfaces"] print(",".join(values) if values else "NONE") PY )" cat > "$REPORT_DIR/report.txt" < "$REPORT_DIR/facts.json" <<'PY' import json import sys ( assessment_path, step, timestamp, report, report_txt, facts_json, architecture, xs_map, global_plan, ) = sys.argv[1:] assessment = json.load( open(assessment_path, encoding="utf-8") ) print(json.dumps({ "schema": "router-step-facts-v1", "step": step, "generated_at_utc": timestamp, "assessment": assessment, "safety": assessment["safety"], "plan": { "current_milestone": "M07", "milestone_completed": False, }, "next_step": "M07E_ATOMIC_REPAIR_OF_REFRESH_PLANNER_AND_EXIT_CONTRACTS", "publish": { "trycf_report": report, "report_txt": report_txt, "facts_json": facts_json, "architecture_plan": architecture, "xs_map": xs_map, "global_project_plan": global_plan, }, }, ensure_ascii=False, indent=2)) PY create_index find "$REPORT_DIR" -type f ! -name SHA256SUMS -print0 | sort -z | xargs -0 sha256sum > "$REPORT_DIR/SHA256SUMS" trap - ERR echo "decision=$PASS_DECISION" echo "runtime_endpoints=$RUNTIME_ENDPOINTS" echo "divergent_interfaces=$DIVERGENT" echo "current_milestone=M07" echo "read_only=true" print_links