test: prove native reconnect recovery
This commit is contained in:
@@ -78,6 +78,15 @@ compose() {
|
||||
docker compose -p "$project" -f "$repo_root/test/linux-server/compose.yaml" "$@"
|
||||
}
|
||||
|
||||
remove_project_networks() {
|
||||
# The project label is Docker Compose's exact resource-ownership key. This
|
||||
# also reclaims legacy certgen default networks left by fixtures created
|
||||
# before certgen joined the labeled native network.
|
||||
for network_id in $(docker network ls --filter "label=com.docker.compose.project=$project" --quiet); do
|
||||
docker network rm "$network_id" >/dev/null || fail "could not remove owned Docker network $network_id"
|
||||
done
|
||||
}
|
||||
|
||||
rvc() {
|
||||
compose exec -T server /opt/rvbox/rvc --socket /run/rvbox/server.sock "$@"
|
||||
}
|
||||
@@ -205,6 +214,43 @@ assert_signal_term() {
|
||||
fail "TERM command did not reach terminal state"
|
||||
}
|
||||
|
||||
assert_server_restart_reconnect() {
|
||||
issued=$(rvc run --background --shell cmd "$client_id" 'ping -t 127.0.0.1 >NUL' 2>&1) || fail "restart command admission failed: $issued"
|
||||
issue=$(printf '%s\n' "$issued" | awk 'NR == 1 { print $1 }')
|
||||
case $issue in ????????-????-7???-????-????????????) ;; *) fail "restart command returned invalid issue UUID: $issued" ;; esac
|
||||
attempt=0
|
||||
while [ "$attempt" -lt 30 ]; do
|
||||
result=$(rvc stat "$client_id" "$issue" 2>/dev/null || true)
|
||||
if printf '%s\n' "$result" | grep -q 'lifecycle=COMMAND_RUNNING'; then break; fi
|
||||
attempt=$((attempt + 1)); sleep 1
|
||||
done
|
||||
[ "$attempt" -lt 30 ] || fail "restart command did not reach running state"
|
||||
# This preserves the server state volume while replacing the actual server
|
||||
# process behind nginx. A later TERM terminal result can only arrive after
|
||||
# the Windows daemon has established a fresh reconciled WSS session.
|
||||
compose restart server
|
||||
attempt=0
|
||||
while [ "$attempt" -lt 45 ]; do
|
||||
if rvc stat "$client_id" >/dev/null 2>&1; then break; fi
|
||||
attempt=$((attempt + 1)); sleep 1
|
||||
done
|
||||
[ "$attempt" -lt 45 ] || fail "server control socket did not recover after restart"
|
||||
wait_client
|
||||
rvc kill TERM "$client_id" "$issue" >/dev/null || fail "TERM after server restart failed"
|
||||
attempt=0
|
||||
while [ "$attempt" -lt 45 ]; do
|
||||
result=$(rvc stat "$client_id" "$issue" 2>/dev/null || true)
|
||||
if printf '%s\n' "$result" | grep -q 'lifecycle=COMMAND_TERMINATED'; then
|
||||
printf '%s\n' "$result" >"$fixture_dir/server-restart-reconnect.stat"
|
||||
printf 'passed server-restart-reconnect issue=%s\n' "$issue"
|
||||
return 0
|
||||
fi
|
||||
case $result in *'lifecycle=COMMAND_FAILED'*|*'lifecycle=COMMAND_REJECTED'*|*'lifecycle=COMMAND_SUCCEEDED'*) fail "restart TERM command reached wrong terminal state: $result" ;; esac
|
||||
attempt=$((attempt + 1)); sleep 1
|
||||
done
|
||||
fail "restart TERM command did not reach terminal state"
|
||||
}
|
||||
|
||||
collect() {
|
||||
if [ "$vm_prepared" = yes ]; then
|
||||
"$repo_root/scripts/windows/test-host" collect --run-id "$run_id" || true
|
||||
@@ -216,6 +262,7 @@ collect() {
|
||||
|
||||
clean() {
|
||||
compose down --volumes --remove-orphans || true
|
||||
remove_project_networks
|
||||
# A reset is the isolation boundary for the next run. Do not conceal a
|
||||
# failed shutdown/snapshot restore behind a successful-looking `clean`:
|
||||
# callers must repair or explicitly inspect the retained VM lease first.
|
||||
@@ -251,6 +298,7 @@ case $action in
|
||||
wait_client
|
||||
assert_stdin_close
|
||||
assert_signal_term
|
||||
assert_server_restart_reconnect
|
||||
assert_context active-user no active-user
|
||||
assert_context active-user-elevated yes active-user-elevated
|
||||
"$repo_root/scripts/windows/test-host" run --run-id "$run_id" --fail-contexts ACTIVE_USER_ELEVATED
|
||||
|
||||
@@ -23,6 +23,7 @@ Actions:
|
||||
logoff log off the sole active fixture user; use only after service installation
|
||||
collect copy bounded guest artifacts to the local test-run directory
|
||||
inspect read-only RVBox SCM state and bounded client log from a prepared run
|
||||
logs read-only service-startup and client logs from a prepared run
|
||||
stop stop RVBox through SCM and request a graceful guest shutdown
|
||||
reset stop the guest if necessary, restore the declared baseline, and leave it off
|
||||
recover read-only fixture/run-state check for a stopped-resumable run
|
||||
@@ -87,7 +88,7 @@ while [ "$#" -gt 0 ]; do
|
||||
esac
|
||||
done
|
||||
|
||||
case $action in status|prepare|stage|install|run|logoff|collect|inspect|stop|reset|recover) ;; *) usage >&2; fail "unknown action $action" ;; esac
|
||||
case $action in status|prepare|stage|install|run|logoff|collect|inspect|logs|stop|reset|recover) ;; *) usage >&2; fail "unknown action $action" ;; esac
|
||||
if [ "$action" != status ]; then
|
||||
[ -n "$run_id" ] || fail "$action requires --run-id"
|
||||
safe_id "$run_id"
|
||||
@@ -169,7 +170,7 @@ remote() {
|
||||
# retry. Lifecycle transitions remain single-attempt: their caller must
|
||||
# inspect/recover rather than risk a duplicate reset, shutdown, or logoff.
|
||||
case $remote_action in
|
||||
status|recover|prepare-stage|stage|stage-create-root|stage-copy-exe|stage-copy-config|stage-copy-ca|collect|inspect|install|run)
|
||||
status|recover|prepare-stage|stage|stage-create-root|stage-copy-exe|stage-copy-config|stage-copy-ca|collect|inspect|logs|install|run)
|
||||
retry_limit=4
|
||||
;;
|
||||
esac
|
||||
@@ -488,9 +489,9 @@ case "$action" in
|
||||
require_lease
|
||||
install -d -m 700 "$host_stage/artifacts"
|
||||
if [ "$(state)" = running ]; then
|
||||
guest_run --exe 'C:\Windows\System32\cmd.exe' --wait-stdout --wait-stderr --unquoted-args -- \
|
||||
/d /s /c "sc.exe queryex RVBoxClient > \"$guest_root\\service-status.txt\" 2>&1 & echo RVBOX_GUEST_OK" >/dev/null || true
|
||||
VBoxManage guestcontrol "$vm" --username "$guest_user" --passwordfile "$password_file" \
|
||||
provisioner_run --exe 'C:\Windows\System32\cmd.exe' --wait-stdout --wait-stderr --unquoted-args -- \
|
||||
/d /s /c "sc.exe queryex RVBoxClient > \"$guest_root\\service-status.txt\" 2>&1 & if exist \"C:/ProgramData/RVBox/service-startup.log\" copy /y \"C:/ProgramData/RVBox/service-startup.log\" \"$guest_root\\service-startup.log\" >NUL & if exist \"C:/ProgramData/RVBox/test-logs/rvbox.log\" copy /y \"C:/ProgramData/RVBox/test-logs/rvbox.log\" \"$guest_root\\rvbox.log\" >NUL & echo RVBOX_GUEST_OK" >/dev/null || true
|
||||
VBoxManage guestcontrol "$vm" --username "$provisioner_user" --passwordfile "$provisioner_password_file" \
|
||||
copyfrom "$guest_root" "$host_stage/artifacts" --recursive </dev/null >/dev/null 2>&1 || true
|
||||
fi
|
||||
printf 'collected host_stage=%s/artifacts\n' "$host_stage"
|
||||
@@ -500,7 +501,14 @@ case "$action" in
|
||||
require_lease
|
||||
[ "$(state)" = running ] || fail "inspect requires a running prepared VM"
|
||||
guest_run --exe 'C:\Windows\System32\cmd.exe' --wait-stdout --wait-stderr --unquoted-args -- \
|
||||
/d /s /c "sc.exe queryex RVBoxClient & sc.exe qc RVBoxClient & reg.exe query \"HKLM\\SYSTEM\\CurrentControlSet\\Services\\RVBoxClient\" /v ImagePath & reg.exe query \"HKLM\\SYSTEM\\CurrentControlSet\\Services\\RVBoxClient\" /v ObjectName & dir \"C:/ProgramData/RVBox\" & icacls \"C:/ProgramData/RVBox\" & certutil -hashfile \"$guest_root\\rvbox.exe\" SHA256 & \"$guest_root\\rvbox.exe\" --check-config --config \"$guest_root\\client.toml\" > \"$guest_root\\check-config.txt\" 2>&1 & type \"$guest_root\\check-config.txt\" & \"$guest_root\\rvbox.exe\" --service --config \"$guest_root\\client.toml\" > \"$guest_root\\direct-service-probe.txt\" 2>&1 & type \"$guest_root\\direct-service-probe.txt\" & if exist \"C:/ProgramData/RVBox/service-startup.log\" type \"C:/ProgramData/RVBox/service-startup.log\" & wevtutil qe System /q:\"*[System[(EventID=7000 or EventID=7009 or EventID=7031 or EventID=7034)]]\" /c:3 /rd:true /f:text & if exist \"C:/ProgramData/RVBox/test-logs/rvbox.log\" type \"C:/ProgramData/RVBox/test-logs/rvbox.log\" & echo RVBOX_GUEST_OK"
|
||||
/d /s /c "sc.exe queryex RVBoxClient & sc.exe qc RVBoxClient & reg.exe query \"HKLM\\SYSTEM\\CurrentControlSet\\Services\\RVBoxClient\" /v ImagePath & reg.exe query \"HKLM\\SYSTEM\\CurrentControlSet\\Services\\RVBoxClient\" /v ObjectName & dir \"C:/ProgramData/RVBox\" & icacls \"C:/ProgramData/RVBox\" & certutil -hashfile \"$guest_root\\rvbox.exe\" SHA256 & \"$guest_root\\rvbox.exe\" --check-config --config \"$guest_root\\client.toml\" > \"$guest_root\\check-config.txt\" 2>&1 & type \"$guest_root\\check-config.txt\" & if exist \"C:/ProgramData/RVBox/service-startup.log\" type \"C:/ProgramData/RVBox/service-startup.log\" & wevtutil qe System /q:\"*[System[(EventID=7000 or EventID=7009 or EventID=7031 or EventID=7034)]]\" /c:3 /rd:true /f:text & if exist \"C:/ProgramData/RVBox/test-logs/rvbox.log\" type \"C:/ProgramData/RVBox/test-logs/rvbox.log\" & echo RVBOX_GUEST_OK"
|
||||
;;
|
||||
logs)
|
||||
assert_identity
|
||||
require_lease
|
||||
[ "$(state)" = running ] || fail "logs requires a running prepared VM"
|
||||
provisioner_run --exe 'C:\Windows\System32\cmd.exe' --wait-stdout --wait-stderr --unquoted-args -- \
|
||||
/d /s /c "dir \"C:\\ProgramData\\RVBox\" & dir \"C:\\ProgramData\\RVBox\\test-logs\" & type \"C:\\ProgramData\\RVBox\\service-startup.log\" & type \"C:\\ProgramData\\RVBox\\test-logs\\rvbox.log\" & echo RVBOX_GUEST_OK"
|
||||
;;
|
||||
stop)
|
||||
assert_identity
|
||||
|
||||
Reference in New Issue
Block a user