#!/usr/bin/env bash
set -Eeuo pipefail
STEP_ID="${STEP_ID:?}"
STEP_DIR="${STEP_DIR:?}"
STEP_SHA256="${STEP_SHA256:?}"
STEP_FILE="$STEP_DIR/step-code.sh"
TRANSFER_VERIFY="$STEP_DIR/transfer-verify.txt"
REPORT="$STEP_DIR/report.txt"
FACTS="$STEP_DIR/facts.json"
VERIFY="$STEP_DIR/verify.txt"
ROLLBACK="$STEP_DIR/rollback.sh"
LOG="$STEP_DIR/main.log"
SYSTEM_MODEL="$STEP_DIR/affected-system-model.txt"
METHODS="$STEP_DIR/methods-and-limitations.txt"
ACCESS_ROUTES="$STEP_DIR/access-routes.txt"
PUBLISH_OUTPUT="$STEP_DIR/publish-output.txt"
BEFORE_STEP_REPORTS="$STEP_DIR/before-step-reports.txt"
ACTIVE_HASH_BEFORE="$STEP_DIR/active-publisher-files.before.sha256"
ACTIVE_HASH_AFTER="$STEP_DIR/active-publisher-files.after.sha256"
LOCAL_MANIFEST="$STEP_DIR/published-files.sha256"
PUBLICATION_PLAN="$STEP_DIR/publication-plan.txt"
DOMAIN="reports.secret-studio.ru"
VM130_IP="10.71.100.84"
BASE="/opt/router-ops"
PUBLIC="$BASE/public"
TOKEN_FILE="$PUBLIC/.router-public-token"
PUBLISHER="$BASE/bin/router-publish-report"
REINDEX="$BASE/bin/router-public-reindex"
LOCK_FILE="$BASE/state/locks/public-publish.lock"
LABEL="step_direct_https_r04d_evidence_publish"
exec > >(tee -a "$LOG") 2>&1
progress() {
printf '>>> [%s] %s\n' "$(date '+%F %T')" "$*"
}
stop_before_change() {
local result="$1"
echo "RESULT=$result"
echo "STEP_WAS_NOT_STARTED=true"
echo "COPY_VERIFY=PASS"
echo "STEP_SHA256=$STEP_SHA256"
echo "SYSTEM_CHANGE=false"
echo "RUNTIME_IMPACT=false"
exit 20
}
resolve_one() {
local description="$1"
shift
local -a matches=()
mapfile -t matches < <(find "$LATEST" -mindepth 1 -maxdepth 1 -type d "$@" -printf '%f\n' | LC_ALL=C sort)
if [[ "${#matches[@]}" -ne 1 ]]; then
printf '%s_COUNT=%s\n' "$description" "${#matches[@]}" >&2
printf '%s_MATCH=%s\n' "$description" "${matches[*]:-NONE}" >&2
return 1
fi
printf '%s\n' "${matches[0]}"
}
build_folder_index() {
local dir="$1"
local folder
folder="$(basename "$dir")"
{
echo 'router report'
printf '%s
\n' "$folder"
echo ''
find "$dir" -maxdepth 1 -type f ! -name 'index.html' -printf '%f\n' | LC_ALL=C sort |
while IFS= read -r name; do
[[ -n "$name" ]] || continue
printf '- %s
\n' "$name" "$name"
done
echo '
'
echo 'Back to latest
'
echo ''
} > "$dir/index.html.tmp"
chmod 644 "$dir/index.html.tmp"
mv -f "$dir/index.html.tmp" "$dir/index.html"
}
http_code() {
local url="$1"
curl -sS \
--connect-timeout 5 \
--max-time 20 \
--resolve "$DOMAIN:443:$VM130_IP" \
-o /dev/null \
-w '%{http_code}' \
"$url" 2>/dev/null || printf '000'
}
progress "preflight"
for command_name in bash sh python3 sha256sum gzip tar curl openssl find awk sed sort cp chmod stat flock grep wc cmp xargs systemctl; do
command -v "$command_name" >/dev/null || stop_before_change "STOP_REQUIRED_COMMAND_MISSING_${command_name^^}"
done
[[ -s "$TOKEN_FILE" ]] || stop_before_change "STOP_PUBLIC_TOKEN_MISSING"
TOKEN="$(cat "$TOKEN_FILE")"
[[ "$TOKEN" =~ ^[a-f0-9]{48}$ ]] || stop_before_change "STOP_PUBLIC_TOKEN_INVALID"
ROOT="$PUBLIC/r/$TOKEN"
LATEST="$ROOT/latest"
OLD="$ROOT/old"
[[ -d "$LATEST" && -d "$OLD" ]] || stop_before_change "STOP_PUBLIC_LAYOUT_MISSING"
[[ -x "$PUBLISHER" ]] || stop_before_change "STOP_PUBLISHER_NOT_EXECUTABLE"
[[ -x "$REINDEX" ]] || stop_before_change "STOP_REINDEX_NOT_EXECUTABLE"
sh -n "$PUBLISHER" || stop_before_change "STOP_PUBLISHER_SYNTAX_INVALID"
sh -n "$REINDEX" || stop_before_change "STOP_REINDEX_SYNTAX_INVALID"
[[ -s "$STEP_FILE" ]] || stop_before_change "STOP_STEP_CODE_MISSING"
[[ -s "$TRANSFER_VERIFY" ]] || stop_before_change "STOP_TRANSFER_VERIFY_MISSING"
grep -qx 'COPY_VERIFY=PASS' "$TRANSFER_VERIFY" || stop_before_change "STOP_COPY_VERIFY_NOT_CONFIRMED"
grep -qx "STEP_SHA256=$STEP_SHA256" "$TRANSFER_VERIFY" || stop_before_change "STOP_TRANSFER_SHA_NOT_CONFIRMED"
[[ "$(sha256sum "$STEP_FILE" | awk '{print $1}')" == "$STEP_SHA256" ]] || stop_before_change "STOP_EXECUTED_STEP_SHA_MISMATCH"
progress "locating confirmed R04D evidence"
R04D_DIR="$(
find "$BASE/steps" \
-mindepth 1 \
-maxdepth 1 \
-type d \
-name '*_PRECHECK_VM130_ROUTER_PUBLISH_REPORT_DIFF_R04D' \
-printf '%T@ %p\n' 2>/dev/null |
sort -nr |
awk 'NR==1 {$1=""; sub(/^ /,""); print}'
)"
[[ -n "$R04D_DIR" && -d "$R04D_DIR" ]] || stop_before_change "STOP_R04D_DIRECTORY_NOT_FOUND"
[[ -s "$R04D_DIR/report.txt" ]] || stop_before_change "STOP_R04D_REPORT_MISSING"
grep -qx 'RESULT=PASS_ROUTER_PUBLISH_REPORT_EXACT_DIFF' "$R04D_DIR/report.txt" || stop_before_change "STOP_R04D_NOT_CONFIRMED_PASS"
[[ -s "$R04D_DIR/facts.json" ]] || stop_before_change "STOP_R04D_FACTS_MISSING"
python3 -m json.tool "$R04D_DIR/facts.json" >/dev/null || stop_before_change "STOP_R04D_FACTS_INVALID"
[[ -s "$R04D_DIR/router-publish-report.exact.diff" ]] || stop_before_change "STOP_R04D_EXACT_DIFF_MISSING"
[[ -s "$R04D_DIR/router-publish-report.live" ]] || stop_before_change "STOP_R04D_LIVE_COPY_MISSING"
[[ -s "$R04D_DIR/router-publish-report.published" ]] || stop_before_change "STOP_R04D_PUBLISHED_COPY_MISSING"
R04D_STEP_SHA="$(sha256sum "$R04D_DIR/step-code.sh" | awk '{print $1}')"
R04D_RECORDED_SHA="$(awk -F= '/^STEP_SHA256=/{print $2}' "$R04D_DIR/transfer-verify.txt" | tail -n 1)"
[[ -n "$R04D_RECORDED_SHA" && "$R04D_STEP_SHA" == "$R04D_RECORDED_SHA" ]] || stop_before_change "STOP_R04D_STEP_SHA_MISMATCH"
progress "resolving unchanged canonical URLs"
LOCAL_M07_FOLDER="$(resolve_one LOCAL_M07 -name '*_local_architecture_plan_vm101_autonomous_hmn_recovery')" || stop_before_change "STOP_LOCAL_M07_COUNT_INVALID"
GLOBAL_PLAN_FOLDER="$(resolve_one GLOBAL_PROJECT_PLAN -name '*_global_project_plan_wg_paid')" || stop_before_change "STOP_GLOBAL_PLAN_COUNT_INVALID"
ACCESS_MAP_FOLDER="$(resolve_one ACCESS_MAP \( -name '*_access_map_*' -o -name '*_xs_map_*' \))" || stop_before_change "STOP_ACCESS_MAP_COUNT_INVALID"
VM100_FOLDER="$(resolve_one VM100 -name '*_vm100_git_source_*')" || stop_before_change "STOP_VM100_COUNT_INVALID"
VM101_FOLDER="$(resolve_one VM101 -name '*_vm101_git_source_*')" || stop_before_change "STOP_VM101_COUNT_INVALID"
VM121_FOLDER="$(resolve_one VM121 -name '*_vm121_git_source_*')" || stop_before_change "STOP_VM121_COUNT_INVALID"
VM101_MODEL_FOLDER="$(resolve_one VM101_MODEL -name '*_vm101_model_*')" || stop_before_change "STOP_VM101_MODEL_COUNT_INVALID"
VM101_METHODS_FOLDER="$(resolve_one VM101_METHODS -name '*_vm101_methods_*')" || stop_before_change "STOP_VM101_METHODS_COUNT_INVALID"
VM130_SOURCE_FOLDER="$(resolve_one VM130_SOURCE -name '*_vm130_router_ops_source_snapshot_*')" || stop_before_change "STOP_VM130_SOURCE_COUNT_INVALID"
LOCAL_M07_PLAN_URL="https://$DOMAIN/latest/$LOCAL_M07_FOLDER/"
GLOBAL_PROJECT_PLAN_URL="https://$DOMAIN/latest/$GLOBAL_PLAN_FOLDER/"
ACCESS_MAP_URL="https://$DOMAIN/latest/$ACCESS_MAP_FOLDER/"
VM100_PUBLIC_URL="https://$DOMAIN/latest/$VM100_FOLDER/"
VM101_PUBLIC_URL="https://$DOMAIN/latest/$VM101_FOLDER/"
VM121_PUBLIC_URL="https://$DOMAIN/latest/$VM121_FOLDER/"
VM101_MODEL_URL="https://$DOMAIN/latest/$VM101_MODEL_FOLDER/"
VM101_METHODS_URL="https://$DOMAIN/latest/$VM101_METHODS_FOLDER/"
VM130_SOURCE_URL="https://$DOMAIN/latest/$VM130_SOURCE_FOLDER/"
LATEST_ROOT_URL="https://$DOMAIN/latest/"
OLD_ROOT_URL="https://$DOMAIN/old/"
progress "capturing publication state and active source hashes"
find "$LATEST" -mindepth 1 -maxdepth 1 -type d -name '*_step*' -printf '%f\n' | LC_ALL=C sort > "$BEFORE_STEP_REPORTS"
BEFORE_STEP_REPORT_COUNT="$(wc -l < "$BEFORE_STEP_REPORTS" | tr -d ' ')"
ACTIVE_FILES=(
"$BASE/bin/router-public-url"
"$BASE/bin/router-publish-report"
"$BASE/bin/router-vm100-git-source-publish"
"$BASE/bin/router-vm101-git-source-publish"
"$BASE/bin/router-vm121-git-source-publish"
"$BASE/bin/router-machine-close"
)
sha256sum "${ACTIVE_FILES[@]}" > "$ACTIVE_HASH_BEFORE"
progress "copying exact R04D evidence into current STEP"
copy_evidence() {
local source="$1"
local destination="$2"
[[ -f "$source" ]] || stop_before_change "STOP_R04D_EVIDENCE_FILE_MISSING"
cp -a "$source" "$STEP_DIR/$destination"
chmod 644 "$STEP_DIR/$destination"
}
copy_evidence "$R04D_DIR/step-code.sh" "r04d-step-code.sh"
copy_evidence "$R04D_DIR/transfer-verify.txt" "r04d-transfer-verify.txt"
copy_evidence "$R04D_DIR/report.txt" "r04d-report.txt"
copy_evidence "$R04D_DIR/facts.json" "r04d-facts.json"
copy_evidence "$R04D_DIR/verify.txt" "r04d-verify.txt"
copy_evidence "$R04D_DIR/rollback.txt" "r04d-rollback.txt"
copy_evidence "$R04D_DIR/router-publish-report.live" "r04d-router-publish-report.live"
copy_evidence "$R04D_DIR/router-publish-report.published" "r04d-router-publish-report.published"
copy_evidence "$R04D_DIR/router-publish-report.exact.diff" "r04d-router-publish-report.exact.diff"
copy_evidence "$R04D_DIR/router-publish-report.r04c.diff" "r04d-router-publish-report.r04c.diff"
copy_evidence "$R04D_DIR/main.log" "r04d-main.log"
cat > "$SYSTEM_MODEL" < "$METHODS" < "$ACCESS_ROUTES" < "$PUBLICATION_PLAN" < "$PUBLISH_OUTPUT" 2>&1
PUBLISH_RC=$?
set -e
cat "$PUBLISH_OUTPUT"
if [[ "$PUBLISH_RC" -ne 0 ]]; then
cat > "$REPORT" < "$ROLLBACK" <"\$LOCK_FILE"
flock -n 9
if [[ -d "\$PUBLISHED_DIR" ]]; then
mv "\$PUBLISHED_DIR" "\$QUARANTINE/\$PUBLISHED_FOLDER"
fi
while IFS= read -r name; do
[[ -n "\$name" ]] || continue
if [[ -d "\$OLD/\$name" && ! -e "\$LATEST/\$name" ]]; then
mv "\$OLD/\$name" "\$LATEST/\$name"
fi
done < "\$BEFORE_STEP_REPORTS"
"\$REINDEX" >/dev/null
echo "RESULT=PASS_R04D_EVIDENCE_PUBLICATION_ROLLBACK"
EOF
chmod 700 "$ROLLBACK"
PROBE_VALUE="R04D_EVIDENCE_${STEP_SHA256}_$(date +%s)"
printf '%s\n' "$PROBE_VALUE" > "$PUBLISHED_DIR/publication-probe.txt"
chmod 644 "$PUBLISHED_DIR/publication-probe.txt"
build_folder_index "$PUBLISHED_DIR"
"$REINDEX" >/dev/null
progress "independent post-check"
sha256sum "${ACTIVE_FILES[@]}" > "$ACTIVE_HASH_AFTER"
ACTIVE_SOURCES_UNCHANGED=false
cmp -s "$ACTIVE_HASH_BEFORE" "$ACTIVE_HASH_AFTER" && ACTIVE_SOURCES_UNCHANGED=true
MOVED_STATE_PASS=true
while IFS= read -r name; do
[[ -n "$name" ]] || continue
if [[ -e "$LATEST/$name" || ! -d "$OLD/$name" ]]; then
MOVED_STATE_PASS=false
fi
done < "$BEFORE_STEP_REPORTS"
CURRENT_STEP_REPORT_COUNT="$(find "$LATEST" -mindepth 1 -maxdepth 1 -type d -name '*_step*' -printf '.' | wc -c | tr -d ' ')"
PUBLISHED_FILE_COUNT="$(find "$PUBLISHED_DIR" -maxdepth 1 -type f -printf '.' | wc -c | tr -d ' ')"
INDEX_CODE="$(http_code "$STEP_REPORT_URL")"
PROBE_CODE="$(http_code "${STEP_REPORT_URL}publication-probe.txt")"
DIFF_CODE="$(http_code "${STEP_REPORT_URL}r04d-router-publish-report.exact.diff")"
LATEST_CODE="$(http_code "$LATEST_ROOT_URL")"
OLD_CODE="$(http_code "$OLD_ROOT_URL")"
PROBE_FETCH="$STEP_DIR/probe-fetch.txt"
curl -sS \
--connect-timeout 5 \
--max-time 20 \
--resolve "$DOMAIN:443:$VM130_IP" \
"${STEP_REPORT_URL}publication-probe.txt" > "$PROBE_FETCH" 2>/dev/null || true
PROBE_CONTENT_PASS=false
grep -qxF "$PROBE_VALUE" "$PROBE_FETCH" && PROBE_CONTENT_PASS=true
CERT_ISSUER="$(echo | openssl s_client -connect "$VM130_IP:443" -servername "$DOMAIN" 2>/dev/null | openssl x509 -noout -issuer 2>/dev/null | sed 's/^issuer=//' || true)"
CADDY_ACTIVE="$(systemctl is-active caddy 2>/dev/null || true)"
CADDY_ENABLED="$(systemctl is-enabled caddy 2>/dev/null || true)"
CORE_POSTCHECK=FAIL
if [[ "$PREVIOUS_MOVED_COUNT" -eq "$BEFORE_STEP_REPORT_COUNT" ]] &&
[[ "$MOVED_STATE_PASS" == true ]] &&
[[ "$CURRENT_STEP_REPORT_COUNT" -eq 1 ]] &&
[[ "$ACTIVE_SOURCES_UNCHANGED" == true ]] &&
[[ "$INDEX_CODE" == "200" ]] &&
[[ "$PROBE_CODE" == "200" ]] &&
[[ "$DIFF_CODE" == "200" ]] &&
[[ "$LATEST_CODE" == "200" ]] &&
[[ "$OLD_CODE" == "200" ]] &&
[[ "$PROBE_CONTENT_PASS" == true ]] &&
[[ "$CADDY_ACTIVE" == "active" ]] &&
[[ "$CADDY_ENABLED" == "enabled" ]] &&
[[ "$CERT_ISSUER" == *"Let's Encrypt"* ]]; then
CORE_POSTCHECK=PASS
fi
RESULT="PASS_R04D_EVIDENCE_PUBLISHED_LOCAL"
RUN_RC=0
ROLLBACK_REQUIRED=false
if [[ "$CORE_POSTCHECK" != "PASS" ]]; then
RESULT="STOP_R04D_EVIDENCE_PUBLICATION_POSTCHECK_FAILED"
RUN_RC=40
ROLLBACK_REQUIRED=true
fi
cat > "$VERIFY" </dev/null
cat > "$REPORT" < "$PUBLISHED_DIR/published-files.sha256"
chmod 644 "$PUBLISHED_DIR/published-files.sha256"
cp -a "$PUBLISHED_DIR/published-files.sha256" "$LOCAL_MANIFEST"
chmod 644 "$LOCAL_MANIFEST"
build_folder_index "$PUBLISHED_DIR"
"$REINDEX" >/dev/null
FINAL_REPORT_CODE="$(http_code "${STEP_REPORT_URL}report.txt")"
FINAL_FACTS_CODE="$(http_code "$FACTS_URL")"
FINAL_VERIFY_CODE="$(http_code "$VERIFY_URL")"
FINAL_ROLLBACK_CODE="$(http_code "$ROLLBACK_URL")"
FINAL_DIFF_CODE="$(http_code "${STEP_REPORT_URL}r04d-router-publish-report.exact.diff")"
FINAL_POSTCHECK=FAIL
if [[ "$CORE_POSTCHECK" == "PASS" ]] &&
[[ "$FINAL_REPORT_CODE" == "200" ]] &&
[[ "$FINAL_FACTS_CODE" == "200" ]] &&
[[ "$FINAL_VERIFY_CODE" == "200" ]] &&
[[ "$FINAL_ROLLBACK_CODE" == "200" ]] &&
[[ "$FINAL_DIFF_CODE" == "200" ]] &&
python3 -m json.tool "$PUBLISHED_DIR/facts.json" >/dev/null 2>&1 &&
cmp -s "$REPORT" "$PUBLISHED_DIR/report.txt" &&
cmp -s "$FACTS" "$PUBLISHED_DIR/facts.json" &&
cmp -s "$VERIFY" "$PUBLISHED_DIR/verify.txt"; then
FINAL_POSTCHECK=PASS
fi
printf '\nFINAL_REPORT_CODE=%s\nFINAL_FACTS_CODE=%s\nFINAL_VERIFY_CODE=%s\nFINAL_ROLLBACK_CODE=%s\nFINAL_DIFF_CODE=%s\nFINAL_POSTCHECK=%s\n' \
"$FINAL_REPORT_CODE" "$FINAL_FACTS_CODE" "$FINAL_VERIFY_CODE" "$FINAL_ROLLBACK_CODE" "$FINAL_DIFF_CODE" "$FINAL_POSTCHECK" >> "$VERIFY"
cp -a "$VERIFY" "$PUBLISHED_DIR/verify.txt"
chmod 644 "$PUBLISHED_DIR/verify.txt"
if [[ "$FINAL_POSTCHECK" != "PASS" ]]; then
RESULT="STOP_R04D_EVIDENCE_PUBLICATION_FINALIZATION_FAILED"
{
echo "RESULT=$RESULT"
echo "STEP_ID=$STEP_ID"
echo "STEP_REPORT_URL=$STEP_REPORT_URL"
echo "COPY_VERIFY=PASS"
echo "STEP_SHA256=$STEP_SHA256"
echo "CORE_POSTCHECK=$CORE_POSTCHECK"
echo "FINAL_POSTCHECK=$FINAL_POSTCHECK"
echo "RUNTIME_IMPACT=false"
echo "ROLLBACK_REQUIRED=true"
} > "$REPORT"
"$ROLLBACK" || true
cat "$REPORT"
exit 41
fi
printf 'FINAL_POSTCHECK=PASS\n' >> "$REPORT"
python3 - "$FACTS" <<'PYFINAL'
import json
import sys
path = sys.argv[1]
with open(path, encoding="utf-8") as fh:
data = json.load(fh)
data["final_postcheck"] = "PASS"
with open(path, "w", encoding="utf-8") as fh:
json.dump(data, fh, indent=2, ensure_ascii=False)
fh.write("\n")
PYFINAL
cp -a "$REPORT" "$FACTS" "$VERIFY" "$PUBLISHED_DIR/"
chmod 644 "$PUBLISHED_DIR/report.txt" "$PUBLISHED_DIR/facts.json" "$PUBLISHED_DIR/verify.txt"
POSTFINAL_REPORT_CODE="$(http_code "${STEP_REPORT_URL}report.txt")"
POSTFINAL_FACTS_CODE="$(http_code "$FACTS_URL")"
POSTFINAL_VERIFY_CODE="$(http_code "$VERIFY_URL")"
if [[ "$POSTFINAL_REPORT_CODE" != "200" ||
"$POSTFINAL_FACTS_CODE" != "200" ||
"$POSTFINAL_VERIFY_CODE" != "200" ]] ||
! cmp -s "$REPORT" "$PUBLISHED_DIR/report.txt" ||
! cmp -s "$FACTS" "$PUBLISHED_DIR/facts.json" ||
! cmp -s "$VERIFY" "$PUBLISHED_DIR/verify.txt"; then
RESULT="STOP_R04D_EVIDENCE_POSTFINAL_SYNC_FAILED"
"$ROLLBACK" || true
echo "RESULT=$RESULT"
echo "RUNTIME_IMPACT=false"
echo "ROLLBACK_ATTEMPTED=true"
exit 42
fi
cat "$REPORT"
cp -a "$LOG" "$PUBLISHED_DIR/main.log"
chmod 644 "$PUBLISHED_DIR/main.log"
find "$PUBLISHED_DIR" -maxdepth 1 -type f ! -name 'index.html' ! -name 'published-files.sha256' -print0 |
LC_ALL=C sort -z |
xargs -0 sha256sum > "$PUBLISHED_DIR/published-files.sha256"
chmod 644 "$PUBLISHED_DIR/published-files.sha256"
cp -a "$PUBLISHED_DIR/published-files.sha256" "$LOCAL_MANIFEST"
build_folder_index "$PUBLISHED_DIR"
"$REINDEX" >/dev/null
exit 0