{ "assessment": { "all_ok": false, "success_checks": { "manual_direct_agent_service_start_not_used": true, "precheck_all_ok": true, "precheck_vm100_peer_count_five": true, "precheck_vm100_selector_count_five": true, "precheck_vm100_uci_empty": true, "precheck_vm101_table201_vpn1_ok": true, "precheck_vm101_table202_vpn2_ok": true, "precheck_vm101_table203_vpn3_ok": true, "precheck_vm101_table204_vpn4_ok": true, "precheck_vm101_table205_vpn5_ok": true, "precheck_vm121_ok": true, "private_material_not_published": true, "reconcile_service_started": true, "reconcile_service_used_for_recovery": true, "runtime_restored_to_five": true, "runtime_selector_count_five": true, "runtime_uci_still_empty": true, "vm100_after_reboot_observed": true, "vm100_pre_reboot_snapshot_created": true, "vm100_reboot_command_issued": false, "vm100_returned_after_reboot": true, "vm100_selector_service_enabled": true, "vm100_selector_service_status_ok": true, "vm101_legacy_table200_vpn1_ok_final": true, "vm101_table201_vpn1_ok_final": true, "vm101_table202_vpn2_ok_final": true, "vm101_table203_vpn3_ok_final": true, "vm101_table204_vpn4_ok_final": true, "vm101_table205_vpn5_ok_final": true, "vm121_backend_health_ok_final": true, "vm121_enabled_peer_count_five_final": true, "vm121_pending_global_zero_final": true, "vm121_reconcile_timer_active_final": true, "vm121_watcher_active_final": true } }, "changes": { "manual_direct_agent_service_start_used": "NO", "private_material_published": "NO", "reconcile_service_started_for_recovery": "YES", "vm100_pre_reboot_snapshot_created": "YES", "vm100_rebooted": "UNKNOWN" }, "mode": "REBOOT_VM100_AND_RESTORE_RUNTIME_PEERS_VIA_RECONCILE_SERVICE", "observed": { "vm100_after_reboot_peer_count": 5, "vm100_final_peer_count": 5, "vm100_final_selector_count": 5, "vm100_final_uci_count": 0 }, "publish": { "local_report": "http://127.0.0.1:8099/r/e94a0859747d7b96f29c7fdafc2d0351ba603bb0a7e9e5a4/20260709-105350_step042f_vm100_reboot_recovery_reconcile_test/", "trycf_report": "https://helena-background-beam-harry.trycloudflare.com/r/e94a0859747d7b96f29c7fdafc2d0351ba603bb0a7e9e5a4/20260709-105350_step042f_vm100_reboot_recovery_reconcile_test/" }, "pve_snapshot_vm100_before_raw": "`-> MGTS_RESTORED_BEFORE_FIRST_BOOT 2026-07-06 12:28:36 pve-wg-paid: VM100 restored from golden, before first boot\n `-> MGTS_INTERNAL_LAN_RENUMBERED_OK 2026-07-06 12:52:29 pve-wg-paid: VM100 LAN renumbered to 10.71.100.1, wg_remote 10.250.100.1\n `-> MGTS_VM100_WG_REMOTE_ROTATED_OK 2026-07-06 13:04:15 pve-wg-paid: VM100 wg_remote keys rotated for 10.250.100.0/24; client configs stored on VM100\n `-> MGTS_VM100_ROUTER_OPS_WG_PEER_OK 2026-07-06 14:08:27 pve-wg-paid: VM100 wg_remote has router-ops-mgts peer 10.250.100.4/32; management path via WG\n `-> MGTS_VM100_BEFORE_WG_PAID_CREATE 2026-07-06 17:37:41 Before creating wg_paid on MGTS VM100 for WG Paid migration\n `-> MGTS_VM100_WG_PAID_BASE_OK 2026-07-06 17:47:21 MGTS VM100: wg_paid created; 10.253.1.1/16; UDP 51830; firewall wg_paid; live wg set dummy peer add/remove OK\n `-> STEP_004B_WG_PAID_CLIENT_PATH_OK 2026-07-06 18:33:19 MGTS VM100: wg_paid live client smoke OK; client can reach 10.253.1.1 and internet via VM101\n `-> STEP_004C_ACCESS_MAP_VM121_DIRECT_OK 2026-07-06 18:43:30 MGTS VM100: VM121 direct wg-mgts admin access documented in access-map\n `-> STEP_005_MGTS_EGRESS_PANEL_OK 2026-07-06 20:52:45 MGTS/WG Paid: VM100 PBR fixed; VM101 egress OK; vpn_user manual mode smoke OK; VM105 control panel identified; deferred UI/cleanup tasks recorded\n `-> STEP_028A_MGTS_WG_PAID_E2E_OK 2026-07-06 22:01:22 MGTS VM100: wg_paid target E2E smoke OK; peer lifecycle via agent OK; PBR direct/vpn route decision OK; final test peer removed\n `-> S034K_WGPAY_DSCP_CANARY_PROVEN 2026-07-08 15:23:36 WGPay canary peer 10.253.1.10 DSCP/fwmark path proven to vpn3 after STEP_034J\n `-> S034O_WGPAY_PERSISTED_CANARY_OK 2026-07-08 15:31:52 WGPay persisted per-peer DSCP canary proven by real traffic after STEP_034N\n `-> S035D_WGPAY_SELECTOR_CANARY_OK 2026-07-08 15:41:26 WGPay selector-file canary proven by real traffic after STEP_035C\n `-> S035H_WGPAY_MULTICLASS_MAPPER_OK 2026-07-08 15:48:57 WGPay multiclass mapper proven by real canary traffic after STEP_035G\n `-> S036H_WGPAY_5SLOT_POLICY_PERSISTED 2026-07-08 16:02:14 WGPay VM101 persistent 5-slot policy routes/rules OK after STEP_036G\n `-> S036K_WGPAY_5CLASS_MAPPER_OK 2026-07-08 16:30:01 WGPay 5-class mapper proven by canary traffic after STEP_036J\n `-> S037D_WGPAY_CS2_PROVEN_OK 2026-07-08 17:00:09 WG Paid STEP_037D cs2 proven checkpoint: cs2 peer 10.253.1.11 -> fwmark 0x204 -> vpn4\n `-> S038D_WGPAY_CS3_PROVEN_OK 2026-07-08 17:13:59 WG Paid STEP_038D cs3 proven checkpoint: 10.253.1.12 -> cs3 -> fwmark 0x205 -> table205/vpn5\n `-> S039E_WGPAY_CS4_PROVEN_OK 2026-07-08 18:18:57 WG Paid checkpoint: cs1/cs2/cs3/cs4 proven; cs4 10.253.1.13 -> cs4 -> 0x201 -> vpn1\n `-> current You are here!\nsnapshotting 'drive-sata1' (local-lvm:vm-100-disk-0)\n Logical volume \"snap_vm-100-disk-0_STEP_042F_BEFORE_VM100_REBOOT_TEST\" created.\nsnapshot_create_rc=0\n`-> MGTS_RESTORED_BEFORE_FIRST_BOOT 2026-07-06 12:28:36 pve-wg-paid: VM100 restored from golden, before first boot\n `-> MGTS_INTERNAL_LAN_RENUMBERED_OK 2026-07-06 12:52:29 pve-wg-paid: VM100 LAN renumbered to 10.71.100.1, wg_remote 10.250.100.1\n `-> MGTS_VM100_WG_REMOTE_ROTATED_OK 2026-07-06 13:04:15 pve-wg-paid: VM100 wg_remote keys rotated for 10.250.100.0/24; client configs stored on VM100\n `-> MGTS_VM100_ROUTER_OPS_WG_PEER_OK 2026-07-06 14:08:27 pve-wg-paid: VM100 wg_remote has router-ops-mgts peer 10.250.100.4/32; management path via WG\n `-> MGTS_VM100_BEFORE_WG_PAID_CREATE 2026-07-06 17:37:41 Before creating wg_paid on MGTS VM100 for WG Paid migration\n `-> MGTS_VM100_WG_PAID_BASE_OK 2026-07-06 17:47:21 MGTS VM100: wg_paid created; 10.253.1.1/16; UDP 51830; firewall wg_paid; live wg set dummy peer add/remove OK\n `-> STEP_004B_WG_PAID_CLIENT_PATH_OK 2026-07-06 18:33:19 MGTS VM100: wg_paid live client smoke OK; client can reach 10.253.1.1 and internet via VM101\n `-> STEP_004C_ACCESS_MAP_VM121_DIRECT_OK 2026-07-06 18:43:30 MGTS VM100: VM121 direct wg-mgts admin access documented in access-map\n `-> STEP_005_MGTS_EGRESS_PANEL_OK 2026-07-06 20:52:45 MGTS/WG Paid: VM100 PBR fixed; VM101 egress OK; vpn_user manual mode smoke OK; VM105 control panel identified; deferred UI/cleanup tasks recorded\n `-> STEP_028A_MGTS_WG_PAID_E2E_OK 2026-07-06 22:01:22 MGTS VM100: wg_paid target E2E smoke OK; peer lifecycle via agent OK; PBR direct/vpn route decision OK; final test peer removed\n `-> S034K_WGPAY_DSCP_CANARY_PROVEN 2026-07-08 15:23:36 WGPay canary peer 10.253.1.10 DSCP/fwmark path proven to vpn3 after STEP_034J\n `-> S034O_WGPAY_PERSISTED_CANARY_OK 2026-07-08 15:31:52 WGPay persisted per-peer DSCP canary proven by real traffic after STEP_034N\n `-> S035D_WGPAY_SELECTOR_CANARY_OK 2026-07-08 15:41:26 WGPay selector-file canary proven by real traffic after STEP_035C\n `-> S035H_WGPAY_MULTICLASS_MAPPER_OK 2026-07-08 15:48:57 WGPay multiclass mapper proven by real canary traffic after STEP_035G\n `-> S036H_WGPAY_5SLOT_POLICY_PERSISTED 2026-07-08 16:02:14 WGPay VM101 persistent 5-slot policy routes/rules OK after STEP_036G\n `-> S036K_WGPAY_5CLASS_MAPPER_OK 2026-07-08 16:30:01 WGPay 5-class mapper proven by canary traffic after STEP_036J\n `-> S037D_WGPAY_CS2_PROVEN_OK 2026-07-08 17:00:09 WG Paid STEP_037D cs2 proven checkpoint: cs2 peer 10.253.1.11 -> fwmark 0x204 -> vpn4\n `-> S038D_WGPAY_CS3_PROVEN_OK 2026-07-08 17:13:59 WG Paid STEP_038D cs3 proven checkpoint: 10.253.1.12 -> cs3 -> fwmark 0x205 -> table205/vpn5\n `-> S039E_WGPAY_CS4_PROVEN_OK 2026-07-08 18:18:57 WG Paid checkpoint: cs1/cs2/cs3/cs4 proven; cs4 10.253.1.13 -> cs4 -> 0x201 -> vpn1\n `-> STEP_042F_BEFORE_VM100_REBOOT_TEST 2026-07-09 13:53:53 before STEP_042F VM100 reboot recovery test\n `-> current You are here!\n", "pve_vm100_reboot_cmd_raw": "before_status=status: running\n", "schema": "router-step-facts-v1", "step": "STEP_042F_VM100_REBOOT_RECOVERY_RECONCILE_TEST", "vm100_after_reboot_before_reconcile_raw": "vm100_uptime=300.16\nvm100_peer_count=5\nvm100_allowed_ips=10.253.1.10/32,10.253.1.11/32,10.253.1.12/32,10.253.1.13/32,10.253.1.14/32\nvm100_selector_count=5\nvm100_uci_count=0\nvm100_selector_service_enabled_rc=0\nvm100_selector_service_status_rc=0\n", "vm100_before_raw": "vm100_uptime=152198.98\nvm100_peer_count=5\nvm100_allowed_ips=10.253.1.10/32,10.253.1.11/32,10.253.1.12/32,10.253.1.13/32,10.253.1.14/32\nvm100_selector_count=5\nvm100_uci_count=0\nvm100_selector_service_enabled_rc=0\nvm100_selector_service_status_rc=0\n", "vm100_final_raw": "vm100_uptime=302.45\nvm100_peer_count=5\nvm100_allowed_ips=10.253.1.10/32,10.253.1.11/32,10.253.1.12/32,10.253.1.13/32,10.253.1.14/32\nvm100_selector_count=5\nvm100_uci_count=0\nvm100_selector_service_enabled_rc=0\nvm100_selector_service_status_rc=0\n", "vm101_before_raw": "vm101_table201_vpn1_rc=0\nvm101_table202_vpn2_rc=0\nvm101_table203_vpn3_rc=0\nvm101_table204_vpn4_rc=0\nvm101_table205_vpn5_rc=0\nvm101_legacy_table200_vpn1_rc=0\nvm101_slots_service_enabled_rc=0\nvm101_mapper_service_enabled_rc=0\n", "vm101_final_raw": "vm101_table201_vpn1_rc=0\nvm101_table202_vpn2_rc=0\nvm101_table203_vpn3_rc=0\nvm101_table204_vpn4_rc=0\nvm101_table205_vpn5_rc=0\nvm101_legacy_table200_vpn1_rc=0\nvm101_slots_service_enabled_rc=0\nvm101_mapper_service_enabled_rc=0\n", "vm121_final": { "db": { "enabled_ips": { "rc": 0, "stderr": "", "stdout": "10.253.1.10\n10.253.1.11\n10.253.1.12\n10.253.1.13\n10.253.1.14" }, "enabled_peer_count": { "rc": 0, "stderr": "", "stdout": "5" }, "pending_global": { "rc": 0, "stderr": "", "stdout": "0" }, "pg_container": "wgaccess-postgres" }, "errors": [], "journals": { "agent_tail": "Starting wg-access-agent.service - WG Access Agent real poller...\nJul 09 10:53:07 wg-access-dev wg_access_agent.py[152090]: [2026-07-09T10:53:07.087766+00:00] sync desired enabled peers: 5\nJul 09 10:53:07 wg-access-dev wg_access_agent.py[152090]: [2026-07-09T10:53:07.246231+00:00] remote stdout: fmcqSUy1fErBV5RrcoVoex2VAgaxz2i79GzK8qd88UI= 10.253.1.10/32\nJul 09 10:53:07 wg-access-dev wg_access_agent.py[152090]: zwzkWJkpX7q5PfcT9pG2v52Es4yM0VRP3w1drj7LJmw= 10.253.1.11/32\nJul 09 10:53:07 wg-access-dev wg_access_agent.py[152090]: HknQ+VpqOIvsd3RxkrgaUqiR7zkS9nMfDRzDAvdqUVw= 10.253.1.12/32\nJul 09 10:53:07 wg-access-dev wg_access_agent.py[152090]: JQXornv9vzH7nn9E1laZehYLnwMXuitP+eP35Gde0D4= 10.253.1.13/32\nJul 09 10:53:07 wg-access-dev wg_access_agent.py[152090]: Kl5Ox5HbG4yvDUxkRuoyZmAF3Wx5HW0lHbVS9B0vFXM= 10.253.1.14/32\nJul 09 10:53:07 wg-access-dev wg_access_agent.py[152090]: [2026-07-09T10:53:07.246339+00:00] sync current runtime peers: 5\nJul 09 10:53:07 wg-access-dev wg_access_agent.py[152090]: [2026-07-09T10:53:07.246406+00:00] sync keep peer tunnel_ip=10.253.1.10\nJul 09 10:53:07 wg-access-dev wg_access_agent.py[152090]: [2026-07-09T10:53:07.246460+00:00] sync keep peer tunnel_ip=10.253.1.11\nJul 09 10:53:07 wg-access-dev wg_access_agent.py[152090]: [2026-07-09T10:53:07.246511+00:00] sync keep peer tunnel_ip=10.253.1.12\nJul 09 10:53:07 wg-access-dev wg_access_agent.py[152090]: [2026-07-09T10:53:07.246559+00:00] sync keep peer tunnel_ip=10.253.1.13\nJul 09 10:53:07 wg-access-dev wg_access_agent.py[152090]: [2026-07-09T10:53:07.246613+00:00] sync keep peer tunnel_ip=10.253.1.14\nJul 09 10:53:07 wg-access-dev wg_access_agent.py[152090]: [2026-07-09T10:53:07.411027+00:00] remote stdout: interface: wg_paid\nJul 09 10:53:07 wg-access-dev wg_access_agent.py[152090]: public key: FD1J+KVZ965pCjgtO0YWxJsKANY9uz6v1P54JLkiZzY=\nJul 09 10:53:07 wg-access-dev wg_access_agent.py[152090]: private key: (hidden)\nJul 09 10:53:07 wg-access-dev wg_access_agent.py[152090]: listening port: 51830\nJul 09 10:53:07 wg-access-dev wg_access_agent.py[152090]: peer: Kl5Ox5HbG4yvDUxkRuoyZmAF3Wx5HW0lHbVS9B0vFXM=\nJul 09 10:53:07 wg-access-dev wg_access_agent.py[152090]: preshared key: (hidden)\nJul 09 10:53:07 wg-access-dev wg_access_agent.py[152090]: endpoint: 10.71.100.2:56264\nJul 09 10:53:07 wg-access-dev wg_access_agent.py[152090]: allowed ips: 10.253.1.14/32\nJul 09 10:53:07 wg-access-dev wg_access_agent.py[152090]: latest handshake: 1 hour, 12 minutes, 58 seconds ago\nJul 09 10:53:07 wg-access-dev wg_access_agent.py[152090]: transfer: 820 B received, 3.61 KiB sent\nJul 09 10:53:07 wg-access-dev wg_access_agent.py[152090]: peer: JQXornv9vzH7nn9E1laZehYLnwMXuitP+eP35Gde0D4=\nJul 09 10:53:07 wg-access-dev wg_access_agent.py[152090]: preshared key: (hidden)\nJul 09 10:53:07 wg-access-dev wg_access_agent.py[152090]: endpoint: 10.71.100.2:39020\nJul 09 10:53:07 wg-access-dev wg_access_agent.py[152090]: allowed ips: 10.253.1.13/32\nJul 09 10:53:07 wg-access-dev wg_access_agent.py[152090]: latest handshake: 19 hours, 40 minutes, 34 seconds ago\nJul 09 10:53:07 wg-access-dev wg_access_agent.py[152090]: transfer: 1.35 KiB received, 6.96 KiB sent\nJul 09 10:53:07 wg-access-dev wg_access_agent.py[152090]: peer: HknQ+VpqOIvsd3RxkrgaUqiR7zkS9nMfDRzDAvdqUVw=\nJul 09 10:53:07 wg-access-dev wg_access_agent.py[152090]: preshared key: (hidden)\nJul 09 10:53:07 wg-access-dev wg_access_agent.py[152090]: endpoint: 10.71.100.2:41351\nJul 09 10:53:07 wg-access-dev wg_access_agent.py[152090]: allowed ips: 10.253.1.12/32\nJul 09 10:53:07 wg-access-dev wg_access_agent.py[152090]: latest handshake: 20 hours, 42 minutes, 41 seconds ago\nJul 09 10:53:07 wg-access-dev wg_access_agent.py[152090]: transfer: 692 B received, 3.48 KiB sent\nJul 09 10:53:07 wg-access-dev wg_access_agent.py[152090]: peer: zwzkWJkpX7q5PfcT9pG2v52Es4yM0VRP3w1drj7LJmw=\nJul 09 10:53:07 wg-access-dev wg_access_agent.py[152090]: preshared key: (hidden)\nJul 09 10:53:07 wg-access-dev wg_access_agent.py[152090]: endpoint: 10.71.100.2:35150\nJul 09 10:53:07 wg-access-dev wg_access_agent.py[152090]: allowed ips: 10.253.1.11/32\nJul 09 10:53:07 wg-access-dev wg_access_agent.py[152090]: latest handshake: 20 hours, 56 minutes, 18 seconds ago\nJul 09 10:53:07 wg-access-dev wg_access_agent.py[152090]: transfer: 1.35 KiB received, 6.96 KiB sent\nJul 09 10:53:07 wg-access-dev wg_access_agent.py[152090]: peer: fmcqSUy1fErBV5RrcoVoex2VAgaxz2i79GzK8qd88UI=\nJul 09 10:53:07 wg-access-dev wg_access_agent.py[152090]: preshared key: (hidden)\nJul 09 10:53:07 wg-access-dev wg_access_agent.py[152090]: endpoint: 10.71.100.2:40520\nJul 09 10:53:07 wg-access-dev wg_access_agent.py[152090]: allowed ips: 10.253.1.10/32\nJul 09 10:53:07 wg-access-dev wg_access_agent.py[152090]: latest handshake: 21 hours, 47 minutes, 37 seconds ago\nJul 09 10:53:07 wg-access-dev wg_access_agent.py[152090]: transfer: 3.38 KiB received, 17.40 KiB sent\nJul 09 10:53:07 wg-access-dev wg_access_agent.py[152090]: [2026-07-09T10:53:07.415416+00:00] no pending jobs\nJul 09 10:53:07 wg-access-dev systemd[1]: wg-access-agent.service: Deactivated successfully.\nJul 09 10:53:07 wg-access-dev systemd[1]: Finished wg-access-agent.service - WG Access Agent real poller.\nJul 09 10:55:29 wg-access-dev systemd[1]: Starting wg-access-agent.service - WG Access Agent real poller...\nJul 09 10:55:31 wg-access-dev wg_access_agent.py[152500]: [2026-07-09T10:55:29.528979+00:00] sync desired enabled peers: 5\nJul 09 10:55:31 wg-access-dev wg_access_agent.py[152500]: [2026-07-09T10:55:29.730517+00:00] sync current runtime peers: 0\nJul 09 10:55:31 wg-access-dev wg_access_agent.py[152500]: [2026-07-09T10:55:29.943641+00:00] sync set peer tunnel_ip=10.253.1.10\nJul 09 10:55:31 wg-access-dev wg_access_agent.py[152500]: [2026-07-09T10:55:30.153250+00:00] sync set peer tunnel_ip=10.253.1.11\nJul 09 10:55:31 wg-access-dev wg_access_agent.py[152500]: [2026-07-09T10:55:30.470292+00:00] sync set peer tunnel_ip=10.253.1.12\nJul 09 10:55:31 wg-access-dev wg_access_agent.py[152500]: [2026-07-09T10:55:30.796331+00:00] sync set peer tunnel_ip=10.253.1.13\nJul 09 10:55:31 wg-access-dev wg_access_agent.py[152500]: [2026-07-09T10:55:30.970583+00:00] sync set peer tunnel_ip=10.253.1.14\nJul 09 10:55:31 wg-access-dev wg_access_agent.py[152500]: [2026-07-09T10:55:31.135429+00:00] remote stdout: interface: wg_paid\nJul 09 10:55:31 wg-access-dev wg_access_agent.py[152500]: public key: FD1J+KVZ965pCjgtO0YWxJsKANY9uz6v1P54JLkiZzY=\nJul 09 10:55:31 wg-access-dev wg_access_agent.py[152500]: private key: (hidden)\nJul 09 10:55:31 wg-access-dev wg_access_agent.py[152500]: listening port: 51830\nJul 09 10:55:31 wg-access-dev wg_access_agent.py[152500]: peer: fmcqSUy1fErBV5RrcoVoex2VAgaxz2i79GzK8qd88UI=\nJul 09 10:55:31 wg-access-dev wg_access_agent.py[152500]: preshared key: (hidden)\nJul 09 10:55:31 wg-access-dev wg_access_agent.py[152500]: allowed ips: 10.253.1.10/32\nJul 09 10:55:31 wg-access-dev wg_access_agent.py[152500]: peer: zwzkWJkpX7q5PfcT9pG2v52Es4yM0VRP3w1drj7LJmw=\nJul 09 10:55:31 wg-access-dev wg_access_agent.py[152500]: preshared key: (hidden)\nJul 09 10:55:31 wg-access-dev wg_access_agent.py[152500]: allowed ips: 10.253.1.11/32\nJul 09 10:55:31 wg-access-dev wg_access_agent.py[152500]: peer: HknQ+VpqOIvsd3RxkrgaUqiR7zkS9nMfDRzDAvdqUVw=\nJul 09 10:55:31 wg-access-dev wg_access_agent.py[152500]: preshared key: (hidden)\nJul 09 10:55:31 wg-access-dev wg_access_agent.py[152500]: allowed ips: 10.253.1.12/32\nJul 09 10:55:31 wg-access-dev wg_access_agent.py[152500]: peer: JQXornv9vzH7nn9E1laZehYLnwMXuitP+eP35Gde0D4=\nJul 09 10:55:31 wg-access-dev wg_access_agent.py[152500]: preshared key: (hidden)\nJul 09 10:55:31 wg-access-dev wg_access_agent.py[152500]: allowed ips: 10.253.1.13/32\nJul 09 10:55:31 wg-access-dev wg_access_agent.py[152500]: peer: Kl5Ox5HbG4yvDUxkRuoyZmAF3Wx5HW0lHbVS9B0vFXM=\nJul 09 10:55:31 wg-access-dev wg_access_agent.py[152500]: preshared key: (hidden)\nJul 09 10:55:31 wg-access-dev wg_access_agent.py[152500]: allowed ips: 10.253.1.14/32\nJul 09 10:55:31 wg-access-dev wg_access_agent.py[152500]: [2026-07-09T10:55:31.140750+00:00] no pending jobs\nJul 09 10:55:31 wg-access-dev systemd[1]: wg-access-agent.service: Deactivated successfully.\nJul 09 10:55:31 wg-access-dev systemd[1]: Finished wg-access-agent.service - WG Access Agent real poller.\nJul 09 10:58:29 wg-access-dev systemd[1]: Starting wg-access-agent.service - WG Access Agent real poller...\nJul 09 10:58:30 wg-access-dev wg_access_agent.py[152863]: [2026-07-09T10:58:29.611731+00:00] sync desired enabled peers: 5\nJul 09 10:58:30 wg-access-dev wg_access_agent.py[152863]: [2026-07-09T10:58:29.879786+00:00] remote stdout: fmcqSUy1fErBV5RrcoVoex2VAgaxz2i79GzK8qd88UI= 10.253.1.10/32\nJul 09 10:58:30 wg-access-dev wg_access_agent.py[152863]: zwzkWJkpX7q5PfcT9pG2v52Es4yM0VRP3w1drj7LJmw= 10.253.1.11/32\nJul 09 10:58:30 wg-access-dev wg_access_agent.py[152863]: HknQ+VpqOIvsd3RxkrgaUqiR7zkS9nMfDRzDAvdqUVw= 10.253.1.12/32\nJul 09 10:58:30 wg-access-dev wg_access_agent.py[152863]: JQXornv9vzH7nn9E1laZehYLnwMXuitP+eP35Gde0D4= 10.253.1.13/32\nJul 09 10:58:30 wg-access-dev wg_access_agent.py[152863]: Kl5Ox5HbG4yvDUxkRuoyZmAF3Wx5HW0lHbVS9B0vFXM= 10.253.1.14/32\nJul 09 10:58:30 wg-access-dev wg_access_agent.py[152863]: [2026-07-09T10:58:29.879882+00:00] sync current runtime peers: 5\nJul 09 10:58:30 wg-access-dev wg_access_agent.py[152863]: [2026-07-09T10:58:29.879946+00:00] sync keep peer tunnel_ip=10.253.1.10\nJul 09 10:58:30 wg-access-dev wg_access_agent.py[152863]: [2026-07-09T10:58:29.879998+00:00] sync keep peer tunnel_ip=10.253.1.11\nJul 09 10:58:30 wg-access-dev wg_access_agent.py[152863]: [2026-07-09T10:58:29.880051+00:00] sync keep peer tunnel_ip=10.253.1.12\nJul 09 10:58:30 wg-access-dev wg_access_agent.py[152863]: [2026-07-09T10:58:29.880100+00:00] sync keep peer tunnel_ip=10.253.1.13\nJul 09 10:58:30 wg-access-dev wg_access_agent.py[152863]: [2026-07-09T10:58:29.880153+00:00] sync keep peer tunnel_ip=10.253.1.14\nJul 09 10:58:30 wg-access-dev wg_access_agent.py[152863]: [2026-07-09T10:58:30.186923+00:00] remote stdout: interface: wg_paid\nJul 09 10:58:30 wg-access-dev wg_access_agent.py[152863]: public key: FD1J+KVZ965pCjgtO0YWxJsKANY9uz6v1P54JLkiZzY=\nJul 09 10:58:30 wg-access-dev wg_access_agent.py[152863]: private key: (hidden)\nJul 09 10:58:30 wg-access-dev wg_access_agent.py[152863]: listening port: 51830\nJul 09 10:58:30 wg-access-dev wg_access_agent.py[152863]: peer: fmcqSUy1fErBV5RrcoVoex2VAgaxz2i79GzK8qd88UI=\nJul 09 10:58:30 wg-access-dev wg_access_agent.py[152863]: preshared key: (hidden)\nJul 09 10:58:30 wg-access-dev wg_access_agent.py[152863]: allowed ips: 10.253.1.10/32\nJul 09 10:58:30 wg-access-dev wg_access_agent.py[152863]: peer: zwzkWJkpX7q5PfcT9pG2v52Es4yM0VRP3w1drj7LJmw=\nJul 09 10:58:30 wg-access-dev wg_access_agent.py[152863]: preshared key: (hidden)\nJul 09 10:58:30 wg-access-dev wg_access_agent.py[152863]: allowed ips: 10.253.1.11/32\nJul 09 10:58:30 wg-access-dev wg_access_agent.py[152863]: peer: HknQ+VpqOIvsd3RxkrgaUqiR7zkS9nMfDRzDAvdqUVw=\nJul 09 10:58:30 wg-access-dev wg_access_agent.py[152863]: preshared key: (hidden)\nJul 09 10:58:30 wg-access-dev wg_access_agent.py[152863]: allowed ips: 10.253.1.12/32\nJul 09 10:58:30 wg-access-dev wg_access_agent.py[152863]: peer: JQXornv9vzH7nn9E1laZehYLnwMXuitP+eP35Gde0D4=\nJul 09 10:58:30 wg-access-dev wg_access_agent.py[152863]: preshared key: (hidden)\nJul 09 10:58:30 wg-access-dev wg_access_agent.py[152863]: allowed ips: 10.253.1.13/32\nJul 09 10:58:30 wg-access-dev wg_access_agent.py[152863]: peer: Kl5Ox5HbG4yvDUxkRuoyZmAF3Wx5HW0lHbVS9B0vFXM=\nJul 09 10:58:30 wg-access-dev wg_access_agent.py[152863]: preshared key: (hidden)\nJul 09 10:58:30 wg-access-dev wg_access_agent.py[152863]: allowed ips: 10.253.1.14/32\nJul 09 10:58:30 wg-access-dev wg_access_agent.py[152863]: [2026-07-09T10:58:30.191392+00:00] no pending jobs\nJul 09 10:58:30 wg-access-dev systemd[1]: wg-access-agent.service: Deactivated successfully.\nJul 09 10:58:30 wg-access-dev systemd[1]: Finished wg-access-agent.service - WG Access Agent real poller.\nJul 09 10:59:15 wg-access-dev systemd[1]: Starting wg-access-agent.service - WG Access Agent real poller...\nJul 09 10:59:16 wg-access-dev wg_access_agent.py[152965]: [2026-07-09T10:59:15.604911+00:00] sync desired enabled peers: 5\nJul 09 10:59:16 wg-access-dev wg_access_agent.py[152965]: [2026-07-09T10:59:16.076659+00:00] remote stdout: fmcqSUy1fErBV5RrcoVoex2VAgaxz2i79GzK8qd88UI= 10.253.1.10/32\nJul 09 10:59:16 wg-access-dev wg_access_agent.py[152965]: zwzkWJkpX7q5PfcT9pG2v52Es4yM0VRP3w1drj7LJmw= 10.253.1.11/32\nJul 09 10:59:16 wg-access-dev wg_access_agent.py[152965]: HknQ+VpqOIvsd3RxkrgaUqiR7zkS9nMfDRzDAvdqUVw= 10.253.1.12/32\nJul 09 10:59:16 wg-access-dev wg_access_agent.py[152965]: JQXornv9vzH7nn9E1laZehYLnwMXuitP+eP35Gde0D4= 10.253.1.13/32\nJul 09 10:59:16 wg-access-dev wg_access_agent.py[152965]: Kl5Ox5HbG4yvDUxkRuoyZmAF3Wx5HW0lHbVS9B0vFXM= 10.253.1.14/32\nJul 09 10:59:16 wg-access-dev wg_access_agent.py[152965]: [2026-07-09T10:59:16.076757+00:00] sync current runtime peers: 5\nJul 09 10:59:16 wg-access-dev wg_access_agent.py[152965]: [2026-07-09T10:59:16.076821+00:00] sync keep peer tunnel_ip=10.253.1.10\nJul 09 10:59:16 wg-access-dev wg_access_agent.py[152965]: [2026-07-09T10:59:16.076874+00:00] sync keep peer tunnel_ip=10.253.1.11\nJul 09 10:59:16 wg-access-dev wg_access_agent.py[152965]: [2026-07-09T10:59:16.076927+00:00] sync keep peer tunnel_ip=10.253.1.12\nJul 09 10:59:16 wg-access-dev wg_access_agent.py[152965]: [2026-07-09T10:59:16.076976+00:00] sync keep peer tunnel_ip=10.253.1.13\nJul 09 10:59:16 wg-access-dev wg_access_agent.py[152965]: [2026-07-09T10:59:16.077029+00:00] sync keep peer tunnel_ip=10.253.1.14\nJul 09 10:59:16 wg-access-dev wg_access_agent.py[152965]: [2026-07-09T10:59:16.284088+00:00] remote stdout: interface: wg_paid\nJul 09 10:59:16 wg-access-dev wg_access_agent.py[152965]: public key: FD1J+KVZ965pCjgtO0YWxJsKANY9uz6v1P54JLkiZzY=\nJul 09 10:59:16 wg-access-dev wg_access_agent.py[152965]: private key: (hidden)\nJul 09 10:59:16 wg-access-dev wg_access_agent.py[152965]: listening port: 51830\nJul 09 10:59:16 wg-access-dev wg_access_agent.py[152965]: peer: fmcqSUy1fErBV5RrcoVoex2VAgaxz2i79GzK8qd88UI=\nJul 09 10:59:16 wg-access-dev wg_access_agent.py[152965]: preshared key: (hidden)\nJul 09 10:59:16 wg-access-dev wg_access_agent.py[152965]: allowed ips: 10.253.1.10/32\nJul 09 10:59:16 wg-access-dev wg_access_agent.py[152965]: peer: zwzkWJkpX7q5PfcT9pG2v52Es4yM0VRP3w1drj7LJmw=\nJul 09 10:59:16 wg-access-dev wg_access_agent.py[152965]: preshared key: (hidden)\nJul 09 10:59:16 wg-access-dev wg_access_agent.py[152965]: allowed ips: 10.253.1.11/32\nJul 09 10:59:16 wg-access-dev wg_access_agent.py[152965]: peer: HknQ+VpqOIvsd3RxkrgaUqiR7zkS9nMfDRzDAvdqUVw=\nJul 09 10:59:16 wg-access-dev wg_access_agent.py[152965]: preshared key: (hidden)\nJul 09 10:59:16 wg-access-dev wg_access_agent.py[152965]: allowed ips: 10.253.1.12/32\nJul 09 10:59:16 wg-access-dev wg_access_agent.py[152965]: peer: JQXornv9vzH7nn9E1laZehYLnwMXuitP+eP35Gde0D4=\nJul 09 10:59:16 wg-access-dev wg_access_agent.py[152965]: preshared key: (hidden)\nJul 09 10:59:16 wg-access-dev wg_access_agent.py[152965]: allowed ips: 10.253.1.13/32\nJul 09 10:59:16 wg-access-dev wg_access_agent.py[152965]: peer: Kl5Ox5HbG4yvDUxkRuoyZmAF3Wx5HW0lHbVS9B0vFXM=\nJul 09 10:59:16 wg-access-dev wg_access_agent.py[152965]: preshared key: (hidden)\nJul 09 10:59:16 wg-access-dev wg_access_agent.py[152965]: allowed ips: 10.253.1.14/32\nJul 09 10:59:16 wg-access-dev wg_access_agent.py[152965]: [2026-07-09T10:59:16.289153+00:00] no pending jobs\nJul 09 10:59:16 wg-access-dev systemd[1]: wg-access-agent.service: Deactivated successfully.\nJul 09 10:59:16 wg-access-dev systemd[1]: Finished wg-access-agent.service - WG Access Agent real poller.", "reconcile_tail": "Jul 09 10:44:59 wg-access-dev systemd[1]: Starting wg-access-reconcile.service - WG Access reconcile VM100 runtime peers from VM121 DB...\nJul 09 10:44:59 wg-access-dev wg-access-reconcile-now.sh[149760]: [wg-access-reconcile] start 2026-07-09T10:44:59+00:00\nJul 09 10:44:59 wg-access-dev wg-access-reconcile-now.sh[149760]: [wg-access-reconcile] source=systemd-reconcile-service\nJul 09 10:44:59 wg-access-dev wg-access-reconcile-now.sh[149768]: Result=success\nJul 09 10:44:59 wg-access-dev wg-access-reconcile-now.sh[149768]: ExecMainStartTimestamp=Thu 2026-07-09 10:44:59 UTC\nJul 09 10:44:59 wg-access-dev wg-access-reconcile-now.sh[149768]: ExecMainExitTimestamp=Thu 2026-07-09 10:44:59 UTC\nJul 09 10:44:59 wg-access-dev wg-access-reconcile-now.sh[149768]: ExecMainStatus=0\nJul 09 10:44:59 wg-access-dev wg-access-reconcile-now.sh[149760]: [wg-access-reconcile] done 2026-07-09T10:44:59+00:00\nJul 09 10:44:59 wg-access-dev systemd[1]: wg-access-reconcile.service: Deactivated successfully.\nJul 09 10:44:59 wg-access-dev systemd[1]: Finished wg-access-reconcile.service - WG Access reconcile VM100 runtime peers from VM121 DB.\nJul 09 10:50:06 wg-access-dev systemd[1]: Starting wg-access-reconcile.service - WG Access reconcile VM100 runtime peers from VM121 DB...\nJul 09 10:50:06 wg-access-dev wg-access-reconcile-now.sh[151339]: [wg-access-reconcile] start 2026-07-09T10:50:06+00:00\nJul 09 10:50:06 wg-access-dev wg-access-reconcile-now.sh[151339]: [wg-access-reconcile] source=systemd-reconcile-service\nJul 09 10:50:06 wg-access-dev wg-access-reconcile-now.sh[151348]: Result=success\nJul 09 10:50:06 wg-access-dev wg-access-reconcile-now.sh[151348]: ExecMainStartTimestamp=Thu 2026-07-09 10:50:06 UTC\nJul 09 10:50:06 wg-access-dev wg-access-reconcile-now.sh[151348]: ExecMainExitTimestamp=Thu 2026-07-09 10:50:06 UTC\nJul 09 10:50:06 wg-access-dev wg-access-reconcile-now.sh[151348]: ExecMainStatus=0\nJul 09 10:50:06 wg-access-dev wg-access-reconcile-now.sh[151339]: [wg-access-reconcile] done 2026-07-09T10:50:06+00:00\nJul 09 10:50:06 wg-access-dev systemd[1]: wg-access-reconcile.service: Deactivated successfully.\nJul 09 10:50:06 wg-access-dev systemd[1]: Finished wg-access-reconcile.service - WG Access reconcile VM100 runtime peers from VM121 DB.\nJul 09 10:55:29 wg-access-dev systemd[1]: Starting wg-access-reconcile.service - WG Access reconcile VM100 runtime peers from VM121 DB...\nJul 09 10:55:29 wg-access-dev wg-access-reconcile-now.sh[152495]: [wg-access-reconcile] start 2026-07-09T10:55:29+00:00\nJul 09 10:55:29 wg-access-dev wg-access-reconcile-now.sh[152495]: [wg-access-reconcile] source=systemd-reconcile-service\nJul 09 10:55:31 wg-access-dev wg-access-reconcile-now.sh[152508]: Result=success\nJul 09 10:55:31 wg-access-dev wg-access-reconcile-now.sh[152508]: ExecMainStartTimestamp=Thu 2026-07-09 10:55:29 UTC\nJul 09 10:55:31 wg-access-dev wg-access-reconcile-now.sh[152508]: ExecMainExitTimestamp=Thu 2026-07-09 10:55:31 UTC\nJul 09 10:55:31 wg-access-dev wg-access-reconcile-now.sh[152508]: ExecMainStatus=0\nJul 09 10:55:31 wg-access-dev wg-access-reconcile-now.sh[152495]: [wg-access-reconcile] done 2026-07-09T10:55:31+00:00\nJul 09 10:55:31 wg-access-dev systemd[1]: wg-access-reconcile.service: Deactivated successfully.\nJul 09 10:55:31 wg-access-dev systemd[1]: Finished wg-access-reconcile.service - WG Access reconcile VM100 runtime peers from VM121 DB.\nJul 09 10:59:15 wg-access-dev systemd[1]: Starting wg-access-reconcile.service - WG Access reconcile VM100 runtime peers from VM121 DB...\nJul 09 10:59:15 wg-access-dev wg-access-reconcile-now.sh[152960]: [wg-access-reconcile] start 2026-07-09T10:59:15+00:00\nJul 09 10:59:15 wg-access-dev wg-access-reconcile-now.sh[152960]: [wg-access-reconcile] source=systemd-reconcile-service\nJul 09 10:59:16 wg-access-dev wg-access-reconcile-now.sh[152968]: Result=success\nJul 09 10:59:16 wg-access-dev wg-access-reconcile-now.sh[152968]: ExecMainStartTimestamp=Thu 2026-07-09 10:59:15 UTC\nJul 09 10:59:16 wg-access-dev wg-access-reconcile-now.sh[152968]: ExecMainExitTimestamp=Thu 2026-07-09 10:59:16 UTC\nJul 09 10:59:16 wg-access-dev wg-access-reconcile-now.sh[152968]: ExecMainStatus=0\nJul 09 10:59:16 wg-access-dev wg-access-reconcile-now.sh[152960]: [wg-access-reconcile] done 2026-07-09T10:59:16+00:00\nJul 09 10:59:16 wg-access-dev systemd[1]: wg-access-reconcile.service: Deactivated successfully.\nJul 09 10:59:16 wg-access-dev systemd[1]: Finished wg-access-reconcile.service - WG Access reconcile VM100 runtime peers from VM121 DB." }, "services": { "agent_service_state": { "rc": 0, "stderr": "", "stdout": "Result=success\nExecMainStatus=0\nActiveState=inactive\nSubState=dead" }, "backend_health": { "rc": 0, "stderr": "", "stdout": "{\"status\":\"ok\",\"service\":\"wg-access-backend\",\"environment\":\"dev\"}" }, "reconcile_timer_active": { "rc": 0, "stderr": "", "stdout": "active" }, "reconcile_timer_enabled": { "rc": 0, "stderr": "", "stdout": "enabled" }, "watcher_active": { "rc": 0, "stderr": "", "stdout": "active" }, "watcher_enabled": { "rc": 0, "stderr": "", "stdout": "enabled" } } }, "vm121_precheck": { "assessment": { "all_ok": true, "success_checks": { "backend_health_ok": true, "enabled_peer_count_five": true, "old_path_inactive": true, "pending_global_zero": true, "private_material_not_published": true, "reconcile_timer_active": true, "reconcile_timer_enabled": true, "watcher_active": true, "watcher_enabled": true } }, "changes": { "manual_agent_service_start_used": "NO", "private_material_published": "NO", "vm121_changed": "NO" }, "db": { "enabled_ips": { "rc": 0, "stderr": "", "stdout": "10.253.1.10\n10.253.1.11\n10.253.1.12\n10.253.1.13\n10.253.1.14" }, "enabled_peer_count": { "rc": 0, "stderr": "", "stdout": "5" }, "pending_global": { "rc": 0, "stderr": "", "stdout": "0" }, "pg_container": "wgaccess-postgres" }, "errors": [], "mode": "VM121_PRECHECK_BEFORE_VM100_REBOOT", "schema": "router-step-facts-v1", "services": { "backend_health": { "rc": 0, "stderr": "", "stdout": "{\"status\":\"ok\",\"service\":\"wg-access-backend\",\"environment\":\"dev\"}" }, "old_path_active": { "rc": 3, "stderr": "", "stdout": "inactive" }, "reconcile_timer_active": { "rc": 0, "stderr": "", "stdout": "active" }, "reconcile_timer_enabled": { "rc": 0, "stderr": "", "stdout": "enabled" }, "watcher_active": { "rc": 0, "stderr": "", "stdout": "active" }, "watcher_enabled": { "rc": 0, "stderr": "", "stdout": "enabled" } }, "step": "STEP_042F_VM100_REBOOT_RECOVERY_RECONCILE_TEST" }, "vm121_reconcile_after_reboot_raw": "--- start reconcile ---\nreconcile_start_rc=0\n--- status ---\nā—‹ wg-access-reconcile.service - WG Access reconcile VM100 runtime peers from VM121 DB\n Loaded: loaded (/etc/systemd/system/wg-access-reconcile.service; static)\n Active: inactive (dead) since Thu 2026-07-09 10:59:16 UTC; 9ms ago\nTriggeredBy: ā— wg-access-reconcile.timer\n Process: 152960 ExecStart=/usr/local/sbin/wg-access-reconcile-now.sh (code=exited, status=0/SUCCESS)\n Main PID: 152960 (code=exited, status=0/SUCCESS)\n CPU: 17ms\n\nJul 09 10:59:15 wg-access-dev systemd[1]: Starting wg-access-reconcile.service - WG Access reconcile VM100 runtime peers from VM121 DB...\nJul 09 10:59:15 wg-access-dev wg-access-reconcile-now.sh[152960]: [wg-access-reconcile] start 2026-07-09T10:59:15+00:00\nJul 09 10:59:15 wg-access-dev wg-access-reconcile-now.sh[152960]: [wg-access-reconcile] source=systemd-reconcile-service\nJul 09 10:59:16 wg-access-dev wg-access-reconcile-now.sh[152968]: Result=success\nJul 09 10:59:16 wg-access-dev wg-access-reconcile-now.sh[152968]: ExecMainStartTimestamp=Thu 2026-07-09 10:59:15 UTC\nJul 09 10:59:16 wg-access-dev wg-access-reconcile-now.sh[152968]: ExecMainExitTimestamp=Thu 2026-07-09 10:59:16 UTC\nJul 09 10:59:16 wg-access-dev wg-access-reconcile-now.sh[152968]: ExecMainStatus=0\nJul 09 10:59:16 wg-access-dev wg-access-reconcile-now.sh[152960]: [wg-access-reconcile] done 2026-07-09T10:59:16+00:00\nJul 09 10:59:16 wg-access-dev systemd[1]: wg-access-reconcile.service: Deactivated successfully.\nJul 09 10:59:16 wg-access-dev systemd[1]: Finished wg-access-reconcile.service - WG Access reconcile VM100 runtime peers from VM121 DB.\n", "wait_reconcile_restore_tsv": "0\tpeer_count=5\tselector_count=5\tuci_count=0\tallowed_ips=10.253.1.10/32,10.253.1.11/32,10.253.1.12/32,10.253.1.13/32,10.253.1.14/32\n", "wait_vm100_return_tsv": "0\tpve_rc=0\tvm100_rc=0\tpeer_count=5\tselector_count=5\tuci_count=0\n" }