{ "architecture_plan": "https://helena-background-beam-harry.trycloudflare.com/r/e94a0859747d7b96f29c7fdafc2d0351ba603bb0a7e9e5a4/20260711-095354_step049i_architecture_lock_hmn_egress_recovery/", "assessment": { "all_ok": true, "decision": "PASS_STEP_050H2_SALVAGE_COUNTER_STATE_AFTER_GUARD_TEST", "success_checks": { "all_interfaces_exist": true, "all_tables_have_routes": true, "conf_exists": true, "direct_failopen_disabled": true, "helper_exists": true, "no_emergency_epoch_set": true, "no_emergency_status_set": true, "old_endpoint_quarantined": true, "planner_excludes_old_endpoint": true, "planner_plan_ok": true, "planner_quarantine_enabled": true, "prod_strict_all_ok": true, "production_not_modified": true, "real_daily_counter_is_one": true, "real_quarantine_lines_at_least_two": true, "rebalance_apply_not_run": true, "refresh_not_run": true, "reg_state_dir_unset_before_real_reads": true, "runner_commit_refuses": true, "runner_dryrun_below_threshold": true, "runner_dryrun_daily_count_one": true, "runner_exists": true, "runner_syntax_ok": true, "state_last_egress_matches": true, "state_last_iface_matches": true, "state_last_new_matches": true, "state_last_old_matches": true, "vm100_controller_running": true, "vm100_peer_count_five": true, "vm100_rc_zero": true, "vm101_rc_zero": true, "vm121_enabled_peers_five": true, "vm121_lease_rows_five": true, "vm121_pending_zero": true, "vm121_rc_zero": true, "vpn2_current_is_repaired_endpoint": true, "watcher_all_healthy": true, "watcher_running_enabled": true }, "warnings": [ "050F/050H daily_counter post=0 was a measurement artifact from REG_STATE_DIR temp-state contamination, not production state loss." ] }, "mode": "READ_ONLY_REAL_STATE_SALVAGE_UNSET_REG_STATE_DIR", "planner": { "changes_count": 1, "decision": "plan_ok", "desired_count": 5, "duplicate_excluded_count": 0, "plan": [ { "change": false, "current": "178.215.227.13:44408", "iface": "vpn1", "slot": "egress1", "target": "178.215.227.13:44408", "target_avg_ms": "50.900", "target_rank": "4" }, { "change": true, "current": "151.236.30.139:33998", "iface": "vpn2", "slot": "egress2", "target": "192.121.163.235:42530", "target_avg_ms": "39.008", "target_rank": "1" }, { "change": false, "current": "95.211.68.119:45176", "iface": "vpn3", "slot": "egress3", "target": "95.211.68.119:45176", "target_avg_ms": "44.271", "target_rank": "2" }, { "change": false, "current": "78.31.250.16:41729", "iface": "vpn4", "slot": "egress4", "target": "78.31.250.16:41729", "target_avg_ms": "50.697", "target_rank": "3" }, { "change": false, "current": "157.173.27.69:41797", "iface": "vpn5", "slot": "egress5", "target": "157.173.27.69:41797", "target_avg_ms": "56.803", "target_rank": "5" } ], "pool": "/root/hmn/cache/ok-awg1-strict-foreign-latest.tsv", "pool_mtime_epoch": 1783702742, "quarantine_enabled": true, "quarantine_excluded_count": 1, "schema": "router-egress-hmn-plan-top5-v2" }, "previous_050f_facts_url": "https://helena-background-beam-harry.trycloudflare.com/r/e94a0859747d7b96f29c7fdafc2d0351ba603bb0a7e9e5a4/20260711-105707_step050f_emergency_refresh_skeleton_no_auto_direct/facts.json", "previous_050h_facts_url": "https://helena-background-beam-harry.trycloudflare.com/r/e94a0859747d7b96f29c7fdafc2d0351ba603bb0a7e9e5a4/20260711-112702_step050h_emergency_refresh_threshold_commit_guard_no_refresh/facts.json", "publish": { "architecture_plan": "https://helena-background-beam-harry.trycloudflare.com/r/e94a0859747d7b96f29c7fdafc2d0351ba603bb0a7e9e5a4/20260711-095354_step049i_architecture_lock_hmn_egress_recovery/", "facts_json": "https://helena-background-beam-harry.trycloudflare.com/r/e94a0859747d7b96f29c7fdafc2d0351ba603bb0a7e9e5a4/20260711-113010_step050h2_salvage_counter_state_after_guard_test/facts.json", "report_txt": "https://helena-background-beam-harry.trycloudflare.com/r/e94a0859747d7b96f29c7fdafc2d0351ba603bb0a7e9e5a4/20260711-113010_step050h2_salvage_counter_state_after_guard_test/report.txt", "trycf_report": "https://helena-background-beam-harry.trycloudflare.com/r/e94a0859747d7b96f29c7fdafc2d0351ba603bb0a7e9e5a4/20260711-113010_step050h2_salvage_counter_state_after_guard_test/" }, "runner_commit": { "commit_enabled": false, "cooldown_remaining": 0, "cooldown_seconds": 600, "daily_fail_count": 1, "daily_fail_threshold": 5, "decision": "commit_disabled_in_step050f", "direct_failopen_action": "disabled_not_allowed_in_step050f", "direct_failopen_enabled": false, "force": false, "healthy_vpn_slots": 5, "helper_ok": true, "last_emergency_refresh_epoch": 0, "last_emergency_refresh_status": "NONE", "mode": "commit", "planner_cmd": "/usr/local/sbin/router-egress-hmn-plan-top5.sh", "planner_cmd_exists": true, "reason": "skeleton_installed_but_commit_disabled_until_reviewed_followup", "rebalance_apply_cmd": "/usr/local/sbin/router-egress-hmn-rebalance-top5-apply.sh", "rebalance_apply_cmd_exists": true, "refresh_cmd": "/root/hmn/hmn-refresh-pool-safe.sh", "refresh_cmd_exists": true, "schema": "router-egress-emergency-refresh-v1", "threshold_reached": false, "watcher_cmd": "/usr/local/sbin/router-egress-health-repair-watch.sh", "watcher_cmd_exists": true }, "runner_dryrun": { "commit_enabled": false, "cooldown_remaining": 0, "cooldown_seconds": 600, "daily_fail_count": 1, "daily_fail_threshold": 5, "decision": "below_threshold_noop", "direct_failopen_action": "disabled_not_allowed_in_step050f", "direct_failopen_enabled": false, "force": false, "healthy_vpn_slots": 5, "helper_ok": true, "last_emergency_refresh_epoch": 0, "last_emergency_refresh_status": "NONE", "mode": "dry-run", "planner_cmd": "/usr/local/sbin/router-egress-hmn-plan-top5.sh", "planner_cmd_exists": true, "reason": "daily_fail_count_below_threshold", "rebalance_apply_cmd": "/usr/local/sbin/router-egress-hmn-rebalance-top5-apply.sh", "rebalance_apply_cmd_exists": true, "refresh_cmd": "/root/hmn/hmn-refresh-pool-safe.sh", "refresh_cmd_exists": true, "schema": "router-egress-emergency-refresh-v1", "threshold_reached": false, "watcher_cmd": "/usr/local/sbin/router-egress-health-repair-watch.sh", "watcher_cmd_exists": true }, "schema": "router-step-facts-v1", "step": "STEP_050H2_SALVAGE_COUNTER_STATE_AFTER_GUARD_TEST", "summary": { "classification": "050F_AND_050H_STOP_WERE_FALSE_BLOCKERS_DUE_TO_REG_STATE_DIR_TEMP_SELFTEST_CONTAMINATION", "direct_failopen_enabled": false, "ifaces": { "vpn1": true, "vpn2": true, "vpn3": true, "vpn4": true, "vpn5": true }, "latest_quarantine_ts_old": "1783766954", "new_endpoint": "151.236.30.139:33998", "next_step": "STEP_050I_ENABLE_EMERGENCY_COMMIT_AND_THRESHOLD_DRYRUN_ONLY", "old_endpoint": "109.248.198.4:37129", "planner_quarantine_excluded_count": 1, "planner_targets": [ "178.215.227.13:44408", "192.121.163.235:42530", "95.211.68.119:45176", "78.31.250.16:41729", "157.173.27.69:41797" ], "real_daily_counter": "1", "real_quarantine_lines": "2", "routes": { "201": true, "202": true, "203": true, "204": true, "205": true }, "runner_commit_decision": "commit_disabled_in_step050f", "runner_dryrun_daily_fail_count": 1, "runner_dryrun_decision": "below_threshold_noop", "strict": { "vpn1": true, "vpn2": true, "vpn3": true, "vpn4": true, "vpn5": true }, "vm100_peer_count": "5", "vm121_db": { "enabled_peers": 5, "lease_rows": 5, "pending_jobs": 0, "postgres_container": "wgaccess-postgres" }, "vpn2_current": "151.236.30.139:33998", "watcher_all_healthy": true }, "vm121_db": { "enabled_peers": 5, "lease_rows": 5, "pending_jobs": 0, "postgres_container": "wgaccess-postgres" }, "watcher": { "any_action": false, "mode": "--dry-run", "run_mode": "once", "schema": "router-egress-health-repair-watch-v2", "slots": [ { "action": "none", "decision": "healthy", "fail_count": 0, "iface": "vpn1", "required_confirm": "", "slot": "egress1", "status_ok": true }, { "action": "none", "decision": "healthy", "fail_count": 0, "iface": "vpn2", "required_confirm": "", "slot": "egress2", "status_ok": true }, { "action": "none", "decision": "healthy", "fail_count": 0, "iface": "vpn3", "required_confirm": "", "slot": "egress3", "status_ok": true }, { "action": "none", "decision": "healthy", "fail_count": 0, "iface": "vpn4", "required_confirm": "", "slot": "egress4", "status_ok": true }, { "action": "none", "decision": "healthy", "fail_count": 0, "iface": "vpn5", "required_confirm": "", "slot": "egress5", "status_ok": true } ] } }