{ "schema": "router-step-facts-v1", "step": "STEP_050M07A_READONLY_REVIEW_FULL_HMN_REFRESH_CONTRACT", "assessment": { "decision": "STOP_STEP_050M07A_READONLY_REVIEW_FULL_HMN_REFRESH_CONTRACT_UNEXPECTED_ERROR", "all_ok": false, "error_rc": 1, "error_line": 1374, "failed_stage": "06/09", "last_success": "frozen_toolchain_verified" }, "inline_diagnostic": { "schema": "router-step-inline-diagnostic-v1", "step": "STEP_050M07A_READONLY_REVIEW_FULL_HMN_REFRESH_CONTRACT", "failure": { "reason": "UNEXPECTED_ERROR", "rc": 1, "line": 1374, "stage": "06/09", "last_success": "frozen_toolchain_verified" }, "remote_commands": { "hash_check_rc": "0", "vm101_audit_rc": "NOT_RUN" }, "stderr_files": { "vm101-tool-hash-check.stderr": { "size_bytes": 80, "tail": "Warning: Permanently added '10.71.100.2' (ED25519) to the list of known hosts.\n" }, "vm101.stderr": { "size_bytes": 116, "tail": "Warning: Permanently added '10.71.100.2' (ED25519) to the list of known hosts.\nawk: cmd. line:18: Unexpected token\n" } }, "stdout_files": { "source-contract-excerpts.txt": { "size_bytes": 10159, "tail": "=== runner ===\nreferences_refresh=true\nreferences_apply=true\nreferences_planner=true\nreferences_state_helper=true\n-- dry_run_refs --\n21: mode=\"dry-run\"\n35: --dry-run|--dryrun)\n36: mode=\"dry-run\"\n39: mode=\"dry-run\"\n215: if [ \"$mode\" = \"dry-run\" ]; then\n-- commit_refs --\n9: EMERGENCY_COMMIT_ENABLED=\"${EMERGENCY_COMMIT_ENABLED:-false}\"\n41: --commit|--apply)\n168: echo \" \\\"commit_enabled\\\": $(json_bool \"$EMERGENCY_COMMIT_ENABLED\"),\"\n229: if [ \"$EMERGENCY_COMMIT_ENABLED\" != \"true\" ]; then\n273: \"$EMERGENCY_REBALANCE_APPLY_CMD\" --commit --confirm REBALANCE_TOP5_DAILY >> \"$EMERGENCY_LOG\" 2>&1\n-- refresh_invocation_refs --\n13: EMERGENCY_REFRESH_CMD=\"${EMERGENCY_REFRESH_CMD:-/root/hmn/hmn-refresh-pool-safe.sh}\"\n172: echo \" \\\"refresh_cmd\\\": \\\"$(json_s \"$EMERGENCY_REFRESH_CMD\")\\\",\"\n-- apply_invocation_refs --\n15: EMERGENCY_REBALANCE_APPLY_CMD=\"${EMERGENCY_REBALANCE_APPLY_CMD:-/usr/local/sbin/router-egress-hmn-rebalance-top5-apply.sh}\"\n-- counter_reset_refs --\n105: reg_daily_repair_get 2>/dev/null || echo 0\n-- success_guard_refs --\n114: if ip link show dev \"$iface\" >/dev/null 2>&1 && ping -I \"$iface\" -c 1 -W 3 1.1.1.1 >/tmp/emergency-refresh-ping-${iface}.$$ 2>/dev/null; then\n260: if [ ! -x \"$EMERGENCY_REFRESH_CMD\" ]; then\n270: if [ \"$refresh_rc\" = \"0\" ]; then\n272: if [ -x \"$EMERGENCY_REBALANCE_APPLY_CMD\" ]; then\n280: if [ \"$rebalance_rc\" = \"0\" ]; then\n-- lock_refs --\n18: EMERGENCY_LOCK_DIR=\"${EMERGENCY_LOCK_DIR:-/var/lock/router-egress-emergency-refresh.lock}\"\n-- direct_failopen_refs --\n10: EMERGENCY_DIRECT_FAILOPEN_ENABLED=\"${EMERGENCY_DIRECT_FAILOPEN_ENABLED:-false}\"\n169: echo \" \\\"direct_failopen_enabled\\\": $(json_bool \"$EMERGENCY_DIRECT_FAILOPEN_ENABLED\"),\"\n170: echo \" \\\"direct_failopen_action\\\": \\\"disabled_not_allowed_in_step050f\\\",\"\n258: printf '%s action=emergency_refresh_start count=%s threshold=%s direct_failopen=false\\n' \"$(date -u '+%Y-%m-%dT%H:%M:%SZ' 2>/dev/null || date)\" \"$cnt\" \"$EMERGENCY_DAILY_FAIL_THRESHOLD\" >> \"$EMERGENCY_LOG\" 2>/dev/null || true\n290: emit_json \"refresh_ok_rebalance_failed\" \"refresh_succeeded_but_rebalance_failed_no_direct_failopen\"\n298: emit_json \"refresh_failed_degraded_no_direct\" \"refresh_failed_but_direct_failopen_disabled_in_state_machine\"\n\n=== refresh ===\n-- backup_refs --\n60: say \"rollback: no backup dir available\"\n64: say \"rollback: restoring published pointers/tables from $BACK\"\n73: say \"rollback: latest symlink restored to readlink target: $OLD_LINK\"\n77: say \"rollback: latest symlink restored to realpath: $OLD_REAL\"\n79: say \"rollback WARN: no usable old latest target\"\n89: say \"rollback: restored cache/$BN\"\n149: echo \"=== backup current published pointers/tables ===\"\n164: say \"backup=$BACK\"\n216: echo \"rollback-after-fresh-validation-failed\" > \"$LAST_REFRESH_POOL_SOURCE\"\n237: echo \"rollback-after-old-validation-failed\" > \"$LAST_REFRESH_POOL_SOURCE\"\n-- atomic_write_refs --\n14: LOCK=\"/tmp/hmn-refresh-pool-safe.lock\"\n22: RCFILE=\"/tmp/hmn-refresh-pool-safe.rc.$$\"\n-- failure_guard_refs --\n60: say \"rollback: no backup dir available\"\n64: say \"rollback: restoring published pointers/tables from $BACK\"\n73: say \"rollback: latest symlink restored to readlink target: $OLD_LINK\"\n77: say \"rollback: latest symlink restored to realpath: $OLD_REAL\"\n79: say \"rollback WARN: no usable old latest target\"\n89: say \"rollback: restored cache/$BN\"\n140: trap 'rmdir \"$LOCK\" 2>/dev/null || true' EXIT\n216: echo \"rollback-after-fresh-validation-failed\" > \"$LAST_REFRESH_POOL_SOURCE\"\n237: echo \"rollback-after-old-validation-failed\" > \"$LAST_REFRESH_POOL_SOURCE\"\n-- pool_refs --\n11: VALIDATE=\"$BASE/hmn-validate-current-pool.sh\"\n14: LOCK=\"/tmp/hmn-refresh-pool-safe.lock\"\n21: LOG=\"$LOGDIR/refresh-pool-safe-$TS-$$.log\"\n22: RCFILE=\"/tmp/hmn-refresh-pool-safe.rc.$$\"\n30: LAST_REFRESH_POOL_SOURCE=\"$STATE/last-refresh-pool-source\"\n83: ok-awg1-strict-all-latest.tsv \\\n84: ok-awg1-strict-foreign-latest.tsv \\\n85: selected-awg1-latest.tsv\n116: say \"=== hmn-refresh-pool-safe start ===\"\n157: \"$BASE/cache/ok-awg1-strict-all-latest.tsv\" \\\n158: \"$BASE/cache/ok-awg1-strict-foreign-latest.tsv\" \\\n159: \"$BASE/cache/selected-awg1-latest.tsv\"\n205: say \"validate fresh/current latest pool through vpn_test\"\n213: say \"fresh pool validation failed; rolling back to previous published pool\"\n222: say \"fallback: keep old configs/latest and validate current local pool through vpn_test\"\n234: say \"old/current pool validation failed; rolling back published tables\"\n282: head -n 12 \"$BASE/cache/ok-awg1-strict-foreign-latest.tsv\" 2>/dev/null || echo \"missing\"\n286: cat \"$BASE/cache/selected-awg1-latest.tsv\" 2>/dev/null || echo \"missing\"\n293: say \"=== hmn-refresh-pool-safe done ===\"\n-- provider_fetch_refs --\n5: BASE=\"/root/hmn\"\n10: DOWNLOADER=\"$BASE/hmn-download-all-awg.sh\"\n11: VALIDATE=\"$BASE/hmn-validate-current-pool.sh\"\n14: LOCK=\"/tmp/hmn-refresh-pool-safe.lock\"\n22: RCFILE=\"/tmp/hmn-refresh-pool-safe.rc.$$\"\n116: say \"=== hmn-refresh-pool-safe start ===\"\n173: say \"trying fresh HMN download via: $DOWNLOADER\"\n188: say \"simulating HMN fresh download failure\"\n293: say \"=== hmn-refresh-pool-safe done ===\"\n\n=== apply ===\n-- backup_refs --\n151: cat > \"$backup_dir/rollback-${slot}.sh\" </dev/null 2>&1 || true\n211: echo ' \"safety\": {\"commit_requires_confirm\": true, \"per_slot_strict_check\": true, \"per_slot_rollback\": true}'\n-- rollback_refs --\n64: trap 'rm -f \"$tmp_top\" \"$tmp_current\" \"$tmp_missing\" \"$tmp_plan\"' EXIT\n151: cat > \"$backup_dir/rollback-${slot}.sh\" </dev/null 2>&1 || true\n211: echo ' \"safety\": {\"commit_requires_confirm\": true, \"per_slot_strict_check\": true, \"per_slot_rollback\": true}'\n-- validation_refs --\n9: PREFERRED_POOL_FILES=\"${PREFERRED_POOL_FILES:-ok-awg1-strict-foreign-latest.tsv ok-awg1-strict-all-latest.tsv}\"\n27: strict_ping() {\n31: ping -I \"$iface\" -c 3 -W 2 \"$target\" >/tmp/rebalance-ping.out 2>/tmp/rebalance-ping.err\n33: recv=\"$(grep -Eo '[0-9]+ packets received' /tmp/rebalance-ping.out 2>/dev/null | awk '{print $1}' | tail -1)\"\n37: rm -f /tmp/rebalance-ping.out /tmp/rebalance-ping.err\n150: ip route show table \"$table\" > \"$backup_dir/table.before\" 2>/dev/null || true\n171: if [ \"$rc\" != \"0\" ] || ! ip route show table \"$table\" 2>/dev/null | grep -q . || ! strict_ping \"$iface\"; then\n211: echo ' \"safety\": {\"commit_requires_confirm\": true, \"per_slot_strict_check\": true, \"per_slot_rollback\": true}'\n-- slot_refs --\n-- planner_refs --\n\n=== planner ===\n-- quarantine_refs --\n78: QUARANTINE_ENABLED=true\n80: QUARANTINE_ENABLED=false\n96: quarantine_excluded=0\n110: if [ \"$QUARANTINE_ENABLED\" = \"true\" ] && reg_endpoint_quarantined_for_pool \"$endpoint\" \"$POOL\"; then\n111: quarantine_excluded=$((quarantine_excluded + 1))\n172: echo \" \\\"quarantine_enabled\\\": $QUARANTINE_ENABLED,\"\n173: echo \" \\\"quarantine_excluded_count\\\": $quarantine_excluded,\"\n-- five_slot_refs --\n7: WORK=\"/tmp/router-egress-hmn-plan-top5.$$\"\n8: SLOTS=\"egress1:vpn1 egress2:vpn2 egress3:vpn3 egress4:vpn4 egress5:vpn5\"\n71: echo \"{\\\"schema\\\":\\\"router-egress-hmn-plan-top5-v2\\\",\\\"decision\\\":\\\"missing_pool\\\",\\\"pool\\\":\\\"$(json_s \"$POOL\")\\\",\\\"changes_count\\\":0,\\\"plan\\\":[]}\"\n168: echo \" \\\"schema\\\": \\\"router-egress-hmn-plan-top5-v2\\\",\"\n-- json_refs --\n13: json_s() {\n71: echo \"{\\\"schema\\\":\\\"router-egress-hmn-plan-top5-v2\\\",\\\"decision\\\":\\\"missing_pool\\\",\\\"pool\\\":\\\"$(json_s \"$POOL\")\\\",\\\"changes_count\\\":0,\\\"plan\\\":[]}\"\n115: printf '%s\\t%s\\t%s\\n' \"$endpoint\" \"${rank:-0}\" \"${avg_ms:-}\" >> \"$CAND\"\n138: rank=\"$(printf '%s\\n' \"$row\" | awk -F '\\t' '{print $2}')\"\n139: avg=\"$(printf '%s\\n' \"$row\" | awk -F '\\t' '{print $3}')\"\n143: target=\"$(printf '%s\\n' \"$row\" | awk -F '\\t' '{print $1}')\"\n144: rank=\"$(printf '%s\\n' \"$row\" | awk -F '\\t' '{print $2}')\"\n145: avg=\"$(printf '%s\\n' \"$row\" | awk -F '\\t' '{print $3}')\"\n169: echo \" \\\"decision\\\": \\\"$(json_s \"$decision\")\\\",\"\n170: echo \" \\\"pool\\\": \\\"$(json_s \"$POOL\")\\\",\"\n181: printf ' {\"slot\":\"%s\",\"iface\":\"%s\",\"current\":\"%s\",\"target\":\"%s\",\"change\":%s,\"target_rank\":\"%s\",\"target_avg_ms\":\"%s\"}\\n' \\\n182: \"$(json_s \"$slot\")\" \\\n183: \"$(json_s \"$iface\")\" \\\n184: \"$(json_s \"$cur\")\" \\\n185: \"$(json_s \"$target\")\" \\\n187: \"$(json_s \"$rank\")\" \\\n188: \"$(json_s \"$avg\")\"\n\n=== helper ===\n-- counter_get_refs --\n177: reg_daily_repair_get() {\n-- counter_reset_refs --\n-- state_dir_refs --\n3: REG_STATE_DIR=\"${REG_STATE_DIR:-/var/lib/router-egress-recovery}\"\n4: REG_QUARANTINE_TSV=\"${REG_QUARANTINE_TSV:-$REG_STATE_DIR/quarantine.tsv}\"\n5: REG_COUNTER_DIR=\"${REG_COUNTER_DIR:-$REG_STATE_DIR/fail-counter}\"\n6: REG_STATE_KV=\"${REG_STATE_KV:-$REG_STATE_DIR/state.kv}\"\n7: REG_LOCK_DIR=\"${REG_LOCK_DIR:-$REG_STATE_DIR/locks}\"\n30: mkdir -p \"$REG_STATE_DIR\" \"$REG_COUNTER_DIR\" \"$REG_LOCK_DIR\" 2>/dev/null || return 1\n186: old_state_dir=\"$REG_STATE_DIR\"\n193: REG_STATE_DIR=\"$test_root/state\"\n194: REG_QUARANTINE_TSV=\"$REG_STATE_DIR/quarantine.tsv\"\n195: REG_COUNTER_DIR=\"$REG_STATE_DIR/fail-counter\"\n196: REG_STATE_KV=\"$REG_STATE_DIR/state.kv\"\n197: REG_LOCK_DIR=\"$REG_STATE_DIR/locks\"\n255: REG_STATE_DIR=\"$old_state_dir\"\n\n=== hook ===\n-- runner_refs --\n8: RUNNER=\"${EMERGENCY_DECISION_RUNNER:-/usr/local/sbin/router-egress-emergency-refresh.sh}\"\n47: echo \" \\\"runner\\\": \\\"$(json_s \"$RUNNER\")\\\"\"\n60: elif [ ! -x \"$RUNNER\" ]; then\n64: \"$RUNNER\" --dry-run > \"$tmp\" 2> \"$err\"\n-- dry_run_refs --\n28: --dry-run)\n58: emit_fallback \"unsafe_config_refused\" \"dry_run_hook_refuses_commit_or_direct_configuration\" > \"$tmp\"\n64: \"$RUNNER\" --dry-run > \"$tmp\" 2> \"$err\"\n-- commit_refs --\n\n=== sequence ===\n{\n \"refresh_position\": 539,\n \"apply_position\": 737,\n \"counter_reset_position\": 2459,\n \"refresh_before_apply\": true,\n \"counter_reset_after_apply\": true\n}\n" }, "vm101-tool-hash-check.txt": { "size_bytes": 2347, "tail": "__HASH__ match=true missing=false expected=d48c41014ff94cbecd6dd8a2f543b5a08c797d226c3e745756a2787340d57151 actual=d48c41014ff94cbecd6dd8a2f543b5a08c797d226c3e745756a2787340d57151 path=/usr/local/sbin/router-egress-health-repair-watch.sh\n__HASH__ match=true missing=false expected=258ef9daf0b3f8aaf2960025e6d4d7f0c1ae30cede781badfbd6b3919ea25afc actual=258ef9daf0b3f8aaf2960025e6d4d7f0c1ae30cede781badfbd6b3919ea25afc path=/usr/local/sbin/router-egress-emergency-decision-hook.sh\n__HASH__ match=true missing=false expected=bea82c260ba6006b524ef01c614848097bd231fa6d49a725baded99046fe5d11 actual=bea82c260ba6006b524ef01c614848097bd231fa6d49a725baded99046fe5d11 path=/usr/local/sbin/router-egress-emergency-refresh.sh\n__HASH__ match=true missing=false expected=6ec99e157235d811d6f0f3ab92de7f3371dc813f2478a654358982cda7ce34ce actual=6ec99e157235d811d6f0f3ab92de7f3371dc813f2478a654358982cda7ce34ce path=/usr/local/sbin/router-egress-hmn-plan-top5.sh\n__HASH__ match=true missing=false expected=2fa6b725c5e123b7fdc3dd14747058a02415d025231054c7537d72b8edc2346f actual=2fa6b725c5e123b7fdc3dd14747058a02415d025231054c7537d72b8edc2346f path=/usr/local/sbin/router-egress-hmn-rebalance-top5-apply.sh\n__HASH__ match=true missing=false expected=9a546c2958c9fd44c927af3559966d1aa0ceb7a5e20f2373c7a95a31728dd0c0 actual=9a546c2958c9fd44c927af3559966d1aa0ceb7a5e20f2373c7a95a31728dd0c0 path=/usr/local/lib/router-egress-recovery-state.sh\n__HASH__ match=true missing=false expected=bb047e820196c4c5cb263499ed890c782190293257a2033568015c6b19230682 actual=bb047e820196c4c5cb263499ed890c782190293257a2033568015c6b19230682 path=/root/hmn/hmn-refresh-pool-safe.sh\n__HASH__ match=true missing=false expected=195a8450fa333a50ae209be25f823873d475989bdbce1a824a86268d36c9e942 actual=195a8450fa333a50ae209be25f823873d475989bdbce1a824a86268d36c9e942 path=/etc/init.d/router-egress-health-repair\n__HASH__ match=true missing=false expected=152fa0d45cb57ac60d34529ef0b29d32e65a2ad03845129fd7c2b41eaf787934 actual=152fa0d45cb57ac60d34529ef0b29d32e65a2ad03845129fd7c2b41eaf787934 path=/etc/init.d/router-egress-emergency-decision\n__HASH__ match=true missing=false expected=42ba3c0b10d8535d4100cc4e35e02bc2c0f3fe104bdedf8cf91700832d9051a8 actual=42ba3c0b10d8535d4100cc4e35e02bc2c0f3fe104bdedf8cf91700832d9051a8 path=/etc/init.d/router-egress-slots\n__SUMMARY__ checked=10 errors=0\n" }, "vm101.txt": { "size_bytes": 236, "tail": "__TRACE__ stage=required_files\n__TRACE__ stage=config_and_services\n__FACT__ commit_enabled=true\n__FACT__ repair_counter=7\n__FACT__ hook_running=true\n__FACT__ hook_enabled=true\n__FACT__ watcher_running=true\n__FACT__ watcher_enabled=true\n" } }, "artifacts_present": [ "diagnostic.json", "progress.log", "source-contract-excerpts.txt", "source-contract.json", "sources/hmn-refresh-pool-safe.sh", "sources/router-egress-emergency-decision-hook.sh", "sources/router-egress-emergency-decision.init", "sources/router-egress-emergency-refresh.sh", "sources/router-egress-hmn-plan-top5.sh", "sources/router-egress-hmn-rebalance-top5-apply.sh", "sources/router-egress-recovery-state.sh", "sources/step050m06e-assessment.json", "sources/step050m06e-step.sh", "step.sh", "vm101-tool-hash-check.sh", "vm101-tool-hash-check.stderr", "vm101-tool-hash-check.txt", "vm101.sh", "vm101.stderr", "vm101.txt" ], "automatic_classification": "LOCAL_VALIDATION_OR_ARTIFACT_FAILURE", "recommended_next_step": "Использовать diagnostic.json, stderr и failed stage из этого же отчёта; отдельный диагностический STEP не требуется.", "safety": { "production_modified": false, "vm101_modified": false, "refresh_ran": false, "rebalance_apply_ran": false, "counter_changed": false, "direct_failopen_changed": false }, "publish": { "trycf_report": "https://helena-background-beam-harry.trycloudflare.com/r/e94a0859747d7b96f29c7fdafc2d0351ba603bb0a7e9e5a4/20260711-181714_step050m07a_readonly_review_full_hmn_refresh_contract/", "report_txt": "https://helena-background-beam-harry.trycloudflare.com/r/e94a0859747d7b96f29c7fdafc2d0351ba603bb0a7e9e5a4/20260711-181714_step050m07a_readonly_review_full_hmn_refresh_contract/report.txt", "facts_json": "https://helena-background-beam-harry.trycloudflare.com/r/e94a0859747d7b96f29c7fdafc2d0351ba603bb0a7e9e5a4/20260711-181714_step050m07a_readonly_review_full_hmn_refresh_contract/facts.json", "architecture_plan": "https://helena-background-beam-harry.trycloudflare.com/r/e94a0859747d7b96f29c7fdafc2d0351ba603bb0a7e9e5a4/20260711-181158_local_architecture_plan_vm101_autonomous_hmn_recovery/", "xs_map": "https://helena-background-beam-harry.trycloudflare.com/r/e94a0859747d7b96f29c7fdafc2d0351ba603bb0a7e9e5a4/20260711-120734_xs_map_audit_repair_publish/", "global_project_plan": "https://helena-background-beam-harry.trycloudflare.com/r/e94a0859747d7b96f29c7fdafc2d0351ba603bb0a7e9e5a4/20260711-123348_global_project_plan_wg_paid/" } }, "mode": "READ_ONLY_M07_CONTRACT_AUDIT", "safety": { "production_modified": false, "vm101_modified": false, "refresh_ran": false, "rebalance_apply_ran": false, "counter_changed": false, "direct_failopen_changed": false }, "plan": { "current_milestone": "M07", "milestone_changed": false }, "publish": { "trycf_report": "https://helena-background-beam-harry.trycloudflare.com/r/e94a0859747d7b96f29c7fdafc2d0351ba603bb0a7e9e5a4/20260711-181714_step050m07a_readonly_review_full_hmn_refresh_contract/", "report_txt": "https://helena-background-beam-harry.trycloudflare.com/r/e94a0859747d7b96f29c7fdafc2d0351ba603bb0a7e9e5a4/20260711-181714_step050m07a_readonly_review_full_hmn_refresh_contract/report.txt", "facts_json": "https://helena-background-beam-harry.trycloudflare.com/r/e94a0859747d7b96f29c7fdafc2d0351ba603bb0a7e9e5a4/20260711-181714_step050m07a_readonly_review_full_hmn_refresh_contract/facts.json", "architecture_plan": "https://helena-background-beam-harry.trycloudflare.com/r/e94a0859747d7b96f29c7fdafc2d0351ba603bb0a7e9e5a4/20260711-181158_local_architecture_plan_vm101_autonomous_hmn_recovery/", "xs_map": "https://helena-background-beam-harry.trycloudflare.com/r/e94a0859747d7b96f29c7fdafc2d0351ba603bb0a7e9e5a4/20260711-120734_xs_map_audit_repair_publish/", "global_project_plan": "https://helena-background-beam-harry.trycloudflare.com/r/e94a0859747d7b96f29c7fdafc2d0351ba603bb0a7e9e5a4/20260711-123348_global_project_plan_wg_paid/" } }