Repository navigation
Expand file tree
/
Copy pathcloud.sh
More file actions
executable file
·1178 lines (1100 loc) · 51.4 KB
/
Copy pathcloud.sh
File metadata and controls
executable file
·1178 lines (1100 loc) · 51.4 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#!/bin/bash
#
# Unified fleet management for SSB-operated StatBus boxes
#
# This is an OPERATOR tool, not a product feature.
# ./sb manages a single installation. This script manages the fleet.
#
# Usage:
# ./cloud.sh status Show version, channel, name, and group for all boxes
# ./cloud.sh notify [target] Tell target boxes to check for updates (non-disruptive)
# ./cloud.sh upgrade [target] Force target boxes to apply latest now (via upgrader)
# ./cloud.sh install <target> Smart install: tries upgrader first; full bootstrap if unreachable
# ./cloud.sh install <target> <version> Pin to specific version — always full bootstrap
# ./cloud.sh install all Install ALL servers (smart, in sequence)
# ./cloud.sh tail <target> Follow upgrade log; auto-disconnects on completion
# ./cloud.sh rescue <server> Alias for install (backwards compat)
# ./cloud.sh wipe <server> DESTRUCTIVE: delete DB and recreate from scratch
#
# Escalation levels:
# notify — gentle. Servers discover new version. Admin chooses when to upgrade.
# upgrade — firm. All servers apply latest NOW via upgrader (non-disruptive binary).
# install — smart. Tries upgrader first (fast path); falls back to full bootstrap
# (stop service, replace binary, re-run install) only if service is unreachable.
# Pinning a version always takes the full bootstrap path.
# tail — observe. Streams upgrader journal; exits automatically on completion.
# create — provision. Creates a deployment slot at a named release.
# inspect — read-only. Shows credentials/URLs for all deployment slots.
# wipe — destructive. Deletes database and recreates. Data is lost.
#
set -euo pipefail
# DEBUG=1 ./cloud.sh <command> traces every command to stderr via `set -x`.
# Matches the convention in dev.sh.
if [ "${DEBUG:-}" = "true" ] || [ "${DEBUG:-}" = "1" ]; then
set -x
fi
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# Unified SSB-operated fleet registry (STATBUS-337 history: standalone.sh was
# merged here and deleted). Display name and channel are deliberately absent:
# they are read live from each box via ./sb config show so this registry never
# becomes configuration truth.
# Auth is fleet-policy DATA: token means the box may be configured with the
# optional GITHUB_TOKEN; anon means it deliberately exercises customer HTTPS.
# Fields: code|group|ssh target|public domain|auth.
FLEET_REGISTRY=(
"dev|cloud|statbus_dev@niue.statbus.org|dev.statbus.org|token"
"demo|cloud|statbus_demo@niue.statbus.org|demo.statbus.org|anon"
"et|cloud|statbus_et@niue.statbus.org|et.statbus.org|anon"
"jo|cloud|statbus_jo@niue.statbus.org|jo.statbus.org|anon"
"ma|cloud|statbus_ma@niue.statbus.org|ma.statbus.org|anon"
"mw|cloud|statbus_mw@niue.statbus.org|mw.statbus.org|anon"
"ug|cloud|statbus_ug@niue.statbus.org|ug.statbus.org|anon"
"ua|cloud|statbus_ua@niue.statbus.org|ua.statbus.org|anon"
"gh|cloud|statbus_gh@niue.statbus.org|gh.statbus.org|anon"
"no|standalone|statbus@rune.statbus.org|no.statbus.org|anon"
)
INSTALL_URL="https://statbus.org/install.sh"
# Unified trust-key setting, passed as --trust-github-user to ./sb install.
# No default. Example: FLEET_TRUST_KEY_USER=jhf ./cloud.sh install all
if [ -n "${STANDALONE_TRUST_KEY_USER+x}" ]; then
echo "Error: STANDALONE_TRUST_KEY_USER was retired; set FLEET_TRUST_KEY_USER instead." >&2
if [ "${BASH_SOURCE[0]}" != "$0" ]; then return 2; fi
exit 2
fi
FLEET_TRUST_KEY_USER="${FLEET_TRUST_KEY_USER:-}"
finish_cloud_log() {
local exit_code=$?
if [ "$exit_code" -eq 0 ]; then
echo "Log saved to $LOG"
else
echo "Command failed (exit $exit_code). The log has the full output: $LOG" >&2
fi
}
start_cloud_log() {
local verb="$1" target="$2" timestamp
timestamp=$(date -u +%Y%m%dT%H%M%SZ)
mkdir -p ./tmp
LOG="./tmp/cloud-${verb}-${target}-${timestamp}.log"
echo "Logging to $LOG"
trap finish_cloud_log EXIT
exec > >(tee -a "$LOG") 2>&1
}
usage() {
echo "Usage: $0 <command> [args]"
echo "Targets: all | <code> | stable | prerelease | cloud | standalone"
echo "Commands: status, health, notify, upgrade, install, rescue, tail, create, wipe, inspect, import, ssh"
echo "Group-only verbs accepting 'all': create, wipe, import (run eligible entries; print one skip per ineligible entry; wipe all covers cloud boxes only)."
echo "Group-only verbs refusing 'all': inspect, ssh (these operations require a single scope or box)."
echo "wipe <code>: deletes that box's database and reinstalls it empty (cloud or standalone; asks you to type the code)."
exit 1
}
upgrade_usage() {
echo "Usage: $0 upgrade [target] [--yes|--force|-y]"
echo "Force target boxes to apply latest. Prompts for confirmation unless a confirmation flag is supplied."
echo "Targets: all | <code> | stable | prerelease | cloud | standalone"
}
registry_entry() {
local code="$1" entry
for entry in "${FLEET_REGISTRY[@]}"; do
[ "${entry%%|*}" = "$code" ] && { echo "$entry"; return 0; }
done
return 1
}
all_codes() {
local entry
for entry in "${FLEET_REGISTRY[@]}"; do echo "${entry%%|*}"; done
}
registry_entries_for_group() {
local wanted="$1" entry code group rest
for entry in "${FLEET_REGISTRY[@]}"; do
IFS='|' read -r code group rest <<< "$entry"
[ "$group" = "$wanted" ] && echo "$entry"
done
}
entry_field() {
local code="$1" field="$2" entry
entry=$(registry_entry "$code") || return 1
echo "$entry" | cut -d'|' -f"$field"
}
ssh_transport() {
local target="$1"; shift
ssh -o ConnectTimeout=10 -o ServerAliveInterval=10 -o ServerAliveCountMax=3 "$target" "$@"
}
# The one transport boundary. Tests stub ssh_transport, while every verb calls
# ssh_entry with a registry code and never constructs an SSH address itself.
ssh_entry() {
local code="$1" target; shift
target=$(entry_field "$code" 3) || { echo "Error: unknown box '$code'" >&2; return 1; }
ssh_transport "$target" "$@"
}
entry_group() { entry_field "$1" 2; }
entry_domain() { entry_field "$1" 4; }
entry_ssh_target() { entry_field "$1" 3; }
entry_auth() { entry_field "$1" 5; }
service_instance() { entry_ssh_target "$1" | cut -d@ -f1; }
validate_code() {
registry_entry "$1" >/dev/null || { echo "Error: unknown box '$1'" >&2; return 1; }
}
# Metadata is operational truth from the box. `./sb config show` supplies the
# channel and display name. The version comes from the running box binary.
read_server_metadata() {
local code="$1" metadata version channel name
# shellcheck disable=SC2016 # This script is evaluated on the remote box.
metadata=$(ssh_entry "$code" '
cd statbus 2>/dev/null || exit 1
ver=$(./sb --version 2>/dev/null | head -1)
config=$(./sb config show 2>/dev/null)
channel=$(printf "%s\n" "$config" | sed -n "s/^UPGRADE_CHANNEL=//p" | head -1)
name=$(printf "%s\n" "$config" | sed -n "s/^DEPLOYMENT_SLOT_NAME=//p" | head -1)
[ -n "$ver" ] && [ -n "$channel" ] && [ -n "$name" ] || exit 1
printf "%s|%s|%s\n" "$ver" "$channel" "$name"
') || return 1
# The remote side promised exactly three nonempty fields. A successful exit
# with anything else (a MOTD, a shell warning, an old script) is unreadable,
# not a non-member: every channel-targeted verb would otherwise silently
# drop this box. Validate here so every consumer inherits the same contract.
# Exactly one line, exactly two delimiters, no control characters (a
# trailing CR from a Windows-configured shell must not ride into a field).
# Control characters are rejected byte-wise so the check does not depend
# on the caller's locale: DEPLOYMENT_SLOT_NAME is human-facing and may
# legitimately be UTF-8 ("Norge Ø"), which [:print:] under LC_ALL=C rejects.
[ "$(printf '%s\n' "$metadata" | wc -l)" -eq 1 ] || return 1
[ "$(tr -cd '|' <<< "$metadata" | wc -c)" -eq 2 ] || return 1
[ "$(printf '%s' "$metadata" | LC_ALL=C tr -d '\000-\037\177' | wc -c)" -eq "$(printf '%s' "$metadata" | wc -c)" ] || return 1
IFS='|' read -r version channel name <<< "$metadata"
[ -n "$version" ] && is_channel "$channel" && [ -n "$name" ] || return 1
printf '%s\n' "$metadata"
}
is_channel() { [ "$1" = stable ] || [ "$1" = prerelease ]; }
is_group() { [ "$1" = cloud ] || [ "$1" = standalone ]; }
resolve_target_codes() {
local target="$1" code metadata channel matches="" unreadable=""
if [ "$target" = all ]; then all_codes; return; fi
if is_group "$target"; then
registry_entries_for_group "$target" | cut -d'|' -f1
return
fi
if ! is_channel "$target"; then validate_code "$target"; echo "$target"; return; fi
for code in $(all_codes); do
if ! metadata=$(read_server_metadata "$code" 2>/dev/null); then
unreadable="${unreadable:+$unreadable }$code"
continue
fi
IFS='|' read -r _ channel _ <<< "$metadata"
[ "$channel" = "$target" ] && matches="${matches:+$matches }$code"
done
if [ -n "$unreadable" ]; then
echo "Error: cannot resolve channel '$target'; metadata unreadable for: $unreadable" >&2
return 1
fi
[ -n "$matches" ] || { echo "Error: no readable boxes found on channel '$target'" >&2; return 1; }
tr ' ' '\n' <<< "$matches"
}
resolve_target_servers() { resolve_target_codes "$@"; }
resolve_single_target() {
local target="$1" targets; targets=$(resolve_target_codes "$target")
targets="${targets//$'\n'/ }"; local -a resolved=(); read -r -a resolved <<< "$targets"
[ "${#resolved[@]}" -eq 1 ] || { echo "Error: target '$target' resolves to ${#resolved[@]} boxes; specify a box code" >&2; return 1; }
echo "${resolved[0]}"
}
verb_group() {
case "$1" in
create|inspect) echo cloud ;;
import|ssh) echo standalone ;;
*) echo all ;;
esac
}
assert_verb_target_group() {
local verb="$1" target="$2" eligible actual code
eligible=$(verb_group "$verb"); [ "$eligible" = all ] && return 0
[ "$target" = all ] && return 0
# A group selector is eligible only when it names the declared group. Other
# selectors are expanded and every registry entry decides independently.
if is_group "$target"; then
[ "$target" = "$eligible" ] && return 0
echo "Error: $verb is only available for $eligible targets; '$target' is $target." >&2
return 2
fi
for code in $(resolve_target_codes "$target"); do
actual=$(entry_group "$code")
if [ "$actual" != "$eligible" ]; then
echo "Error: $verb is only available for $eligible targets; '$code' is $actual." >&2
return 2
fi
done
}
# NOTE: there is deliberately NO stop_upgrade_service here (STATBUS-041,
# sibling of STATBUS-040 / the deploy-stop footgun). `systemctl --user
# stop` sends SIGTERM, and an in-flight upgrade process catches TERM,
# cancels its context, and ROLLS BACK — on a wedged slot that restores a
# stale snapshot over the live DB. Both of the old rationales for a
# pre-install stop are false:
# - mutex: `./sb install` (post-STATBUS-039) REFUSES a genuinely-
# progressing upgrade (deploy reports failure, operator retries) and
# TAKES OVER a crash-looping unit (NRestarts >= 3) with a SIGKILL-class
# quiesce — no handler runs, no rollback fires.
# - "text file busy": every binary-replacement path is an atomic rename —
# install.sh places ./sb via curl-to-sb.tmp + mv, and the build-from-
# source path does `mv sb-linux-amd64 sb`. rename(2) swaps the directory
# entry while the running process keeps its old inode; ETXTBSY only
# fires on write-in-place, which no path does.
# The per-slot unit instance is statbus-upgrader@<server>.service (the
# suffix is the deployment user; see cli/cmd/install.go:serviceInstance).
# STATBUS-527: boxes installed before the rename still run
# statbus-upgrade@<server>.service until their post-upgrade install migrates
# them, so remote lookups resolve the base name on the box itself.
# Ensure the user-level upgrader is running on exit. Idempotent —
# `systemctl start` on a running unit is a no-op. Used at the end of
# `cmd_install_one` (and on its error paths) so that any cloud.sh exit
# leaves the server in a normal "service running" state, not "stopped
# pending operator intervention".
ensure_service_started() {
local server="$1"
local instance
instance=$(service_instance "$server")
ssh_entry "$server" "systemctl --user start statbus-upgrader@${instance}.service 2>/dev/null || systemctl --user start statbus-upgrade@${instance}.service" 2>&1 || true
}
cmd_status() {
local target="${1:-all}" targets failed=0
targets=$(resolve_target_codes "$target")
echo "StatBus Fleet Status"
echo "===================="
printf " %-8s %-12s %-7s %-31s %-12s %s\n" "CODE" "GROUP" "AUTH" "VERSION" "CHANNEL" "NAME"
for server in $targets; do
local metadata program channel name group auth serving version note
group=$(entry_group "$server")
auth=$(entry_auth "$server")
if ! metadata=$(read_server_metadata "$server" 2>/dev/null); then
printf " %-8s %-12s %-7s METADATA READ FAILED\n" "$server" "$group" "$auth"
failed=1
continue
fi
IFS='|' read -r program channel name <<< "$metadata"
program=$(sed -n 's/.*(commit \([0-9a-f]\{8\}\)).*/\1/p' <<<"$program")
serving=$(read_serving_commits "$server" 2>/dev/null) || serving=UNREADABLE
IFS='|' read -r version note <<< "$(status_version_and_note "$program" "$serving")"
printf " %-8s %-12s %-7s %-31s %-12s %s%s\n" "$server" "$group" "$auth" "$version" "$channel" "$name" \
"${note:+ ← $note}"
done
[ "$failed" -eq 0 ]
}
# read_serving_commits prints the short commit each running StatBus container
# was started from, one distinct commit per line. It reads the container's own
# record of its requested image (Config.Image, e.g. statbus-app:c65d1802),
# which a moved or overwritten tag cannot change: the same identity the
# upgrade service trusts (STATBUS-436). The marker comment names the probe.
read_serving_commits() {
# shellcheck disable=SC2016 # This script is evaluated on the remote box.
ssh_entry "$1" '# statbus-serving-commits
cd statbus 2>/dev/null || exit 1
for id in $(docker compose ps -q app worker proxy db 2>/dev/null); do
docker inspect -f "{{.Config.Image}}" "$id" 2>/dev/null
done | sed -n "s#^.*/statbus-[a-z]*:\([0-9a-f]\{8\}\)\$#\1#p" | sort -u'
}
# release_label names a commit by the release it is, from this checkout's tags:
# the stable tag when there is one (v2026.11.0 over its v2026.11.0-rc.01),
# else the newest candidate tag, else "untagged". Always with the commit.
release_label() {
local commit="$1" tags tag
tags=$(git -C "$SCRIPT_DIR" tag --points-at "$commit" 2>/dev/null | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+(-rc\.[0-9]+)?$' || true)
tag=$(grep -v -- '-rc\.' <<<"$tags" | sort -V | tail -1)
[ -n "$tag" ] || tag=$(sort -V <<<"$tags" | tail -1)
printf '%s (%s)' "${tag:-untagged}" "$commit"
}
# status_version_and_note is "VERSION|NOTE" for one box. VERSION is what the
# box serves: the release its running containers were started from, which is
# what users get. The sb program on disk can be ahead of it: an install that
# stopped after swapping the program left ug serving v2026.09.0 while sb said
# v2026.11.0 (2026-11-03). NOTE is empty on a healthy box and otherwise says
# what is wrong, naming releases by tag.
status_version_and_note() {
local program="$1" serving="$2" count labels commit
if [ "$serving" = UNREADABLE ]; then
printf '%s|%s' "$(release_label "$program")" "could not read the running containers; the version shown is the sb program"
return
fi
count=$(printf '%s' "$serving" | grep -c . || true)
if [ "$count" -eq 0 ]; then
printf '%s|%s' "$(release_label "$program")" "no StatBus containers are running; the version shown is the sb program"
return
fi
if [ "$count" -gt 1 ]; then
labels=$(while IFS= read -r commit; do release_label "$commit"; echo; done <<<"$serving" \
| sort -rV | paste -sd, - | sed 's/,/, /g')
printf '%s|%s' "$(release_label "$program")" "containers run different releases: $labels"
return
fi
if [ -n "$program" ] && [ "$program" != "$serving" ]; then
printf '%s|%s' "$(release_label "$serving")" "sb program is $(release_label "$program"), the containers run $(release_label "$serving")"
return
fi
printf '%s|' "$(release_label "$serving")"
}
# cmd_health_one gathers upgrade-subsystem health for one server in a single
# SSH call. Outputs one formatted line. Designed to run in parallel.
cmd_health_one() {
local server="$1" instance result
instance=$(service_instance "$server")
result=$(ssh_entry "$server" "
cd statbus 2>/dev/null || { printf 'NO_DIR|||'; exit; }
ver=\$(./sb --version 2>/dev/null | head -1 || echo 'UNKNOWN')
svc=\$(systemctl --user is-active 'statbus-upgrader@${instance}.service' 2>/dev/null || systemctl --user is-active 'statbus-upgrade@${instance}.service' 2>/dev/null || echo 'unknown')
hb='tmp/upgrade-heartbeat'
if [ -f \"\$hb\" ]; then
hb_ts=\$(cat \"\$hb\" | tr -d '[:space:]')
now=\$(date +%s); age=\$((now - hb_ts))
if [ \"\$age\" -lt 60 ]; then progress=\"\${age}s ago\"
elif [ \"\$age\" -lt 3600 ]; then progress=\"\$((age/60))m ago\"
else progress=\"stale \$((age/3600))h\"; fi
else
last=\$(journalctl --user -u 'statbus-upgrader@${instance}.service' -u 'statbus-upgrade@${instance}.service' \
-n 1 -o short-unix --no-pager 2>/dev/null | awk '{print int(\$1)}')
if [ -n \"\$last\" ] && [ \"\$last\" -gt 0 ] 2>/dev/null; then
now=\$(date +%s); age=\$((now - last))
if [ \"\$age\" -lt 60 ]; then progress=\"\${age}s ago\"
elif [ \"\$age\" -lt 3600 ]; then progress=\"\$((age/60))m ago\"
else progress=\"stale \$((age/3600))h\"; fi
else
progress='no data'
fi
fi
state=\$(./sb upgrade list 2>/dev/null \
| grep -oE 'completed|failed|in progress|in_progress|rolled_back|pending' | head -1)
[ -z \"\$state\" ] && state='none'
printf '%s|%s|%s|%s' \"\$ver\" \"\$svc\" \"\$progress\" \"\$state\"
" 2>/dev/null) || result="SSH FAILED|||"
if [ -z "$result" ] || [ "$result" = "SSH FAILED|||" ]; then
printf " %-22s SSH FAILED\n" "$server"
return
fi
local ver svc progress state flag
IFS='|' read -r ver svc progress state <<< "$result"
flag=""
[ "${svc:-unknown}" != "active" ] && flag=" ← service ${svc:-unknown}"
echo "${state:-}" | grep -qE 'in[_ ]progress' && flag="${flag} ← WEDGED?"
printf " %-22s %-12s service=%-10s last=%-18s upgrade=%s%s\n" \
"$server" "${ver:-UNKNOWN}" "${svc:-unknown}" "${progress:-?}" "${state:-none}" "$flag"
}
# cmd_health shows upgrade health for one or all servers, in parallel.
cmd_health() {
local target="${1:-all}"
local targets
targets=$(resolve_target_servers "$target")
echo "StatBus Fleet Health"
echo "===================="
if [ "$target" = "all" ] || is_channel "$target" || is_group "$target"; then
local tmpdir pids=()
tmpdir=$(mktemp -d)
for server in $targets; do
cmd_health_one "$server" > "$tmpdir/$server" &
pids+=($!)
done
wait "${pids[@]}"
for server in $targets; do
cat "$tmpdir/$server"
done
rm -rf "$tmpdir"
else
cmd_health_one "$targets"
fi
}
cmd_notify() {
local target="${1:-all}" server targets
targets=$(resolve_target_codes "$target")
echo "Notifying $target boxes to check for updates..."
for server in $targets; do
printf " %-16s " "$server:"
ssh_entry "$server" "cd statbus && ./sb upgrade discover" 2>/dev/null \
&& echo "notified" || echo "FAILED"
done
}
cmd_upgrade() {
local target="${1:-all}" confirmed="${2:-0}" server targets confirm=""
targets=$(resolve_target_codes "$target")
if [ "$confirmed" != 1 ]; then
echo "WARNING: This will force the selected boxes to apply the latest release immediately."
echo "Boxes to upgrade: $(tr '\n' ' ' <<< "$targets" | xargs)"
echo "Use './cloud.sh status' for a read-only version overview first."
if ! read -r -p "Type 'upgrade' to confirm (or Ctrl-C to abort): " confirm </dev/tty 2>/dev/null \
|| [ "$confirm" != upgrade ]; then
echo "Aborted."
return 1
fi
fi
echo "Forcing $target boxes to apply latest..."
for server in $targets; do
printf " %-16s " "$server:"
ssh_entry "$server" "cd statbus && ./sb upgrade apply-latest" 2>/dev/null \
&& echo "scheduled" || echo "FAILED"
done
}
# refresh_local_sb brings this checkout's ./sb to HEAD with `./dev.sh build-sb`
# when it is stale: missing, older than an edited cli/ source, or built from an
# older commit with cli/ changes since (./sb committed-drift).
# An sb outside a source checkout (no dev.sh beside it: the release binary the
# gate's operator-install proof stages) is the binary under test and is used
# exactly as it is.
refresh_local_sb() {
[ -x "$SCRIPT_DIR/dev.sh" ] || return 0
if [ -x "$SCRIPT_DIR/sb" ] && (cd "$SCRIPT_DIR" && ./sb committed-drift) >/dev/null 2>&1 \
&& [ -z "$(cd "$SCRIPT_DIR" && find cli -name '*.go' -newer ./sb -print -quit 2>/dev/null)" ]; then
return 0
fi
echo "Refreshing the local ./sb to this checkout's HEAD..."
(cd "$SCRIPT_DIR" && ./dev.sh build-sb) || { echo "Error: could not refresh ./sb; run ./dev.sh build-sb" >&2; return 1; }
}
cmd_install() {
local target="$1"
local version="${2:-}"
local targets
targets=$(resolve_target_servers "$target")
# install runs this checkout's ./sb (release checks, version comparison).
# Bring it to HEAD first, the same way every dev.sh command does, so a
# binary built before the last pull never runs here or warns "stale".
refresh_local_sb || return 1
if [ "$target" = "all" ] || is_channel "$target" || is_group "$target"; then
echo "Installing $target servers${version:+ (pinned to $version)}"
echo "======================"
for server in $targets; do
echo ""
echo "--- $server ---"
cmd_install_one "$server" "$version"
done
else
cmd_install_one "$targets" "$version"
fi
}
# trust_flag returns the --trust-github-user flag for ./sb install if configured.
trust_flag() {
local user="${1:-}"
if [ -n "$user" ]; then
echo "--trust-github-user $user"
fi
}
# Migration immutability is a RELEASE-CUT concern, enforced by `./sb release
# prerelease` / `release stable` preflight (checkMigrationImmutability in
# cli/cmd/release.go). The install-time check that used to live here was
# wrong-layer: it diffed git history between two HEADs on the slot, which
# bears no relationship to what's recorded in db.migration. A migrate-down
# (which clears the db.migration row) couldn't satisfy a git-history-based
# check, producing an infinite loop on operators recovering from a known-
# corrected migration. Removed 2026-05-22 after the loop bit a dev recovery.
# Release-cut gate remains authoritative; install just applies forward.
# cmd_migrate_down is RETIRED — see the refusal below for why.
# Takes a migration number — rolls back until that migration is gone.
# This is a manual, explicit, operator-invoked command — the upgrade
# service NEVER runs down migrations.
cmd_migrate_down() {
local server="$1"
local migration="$2"
# EDGE IS RETIRED (King, 2026-08-19), and this command went with it.
#
# It was only ever permitted on an edge box, and that was not an arbitrary
# restriction: rolling a migration BACKWARDS is safe only where migrations
# are not immutable, and edge was the one channel that applied ungated master
# commits. Every remaining box follows released tags, where migrations ARE
# immutable — a released migration that wrote wrong data is corrected by a
# FORWARD repair migration, never by rolling the released one back.
#
# The rollback body is deleted rather than left behind a guard: dead code
# under a refusal reads as "this could still be right", and this one would be
# reached for at exactly the wrong moment, on a production box, under
# pressure.
echo "Error: cloud.sh migrate-down is retired together with the edge channel."
echo " Requested: rollback of migration ${migration:-<none>} on ${server:-<none>}."
echo " Released migrations are IMMUTABLE. To correct one, ship a forward repair"
echo " migration and release it: ./sb migrate new --description \"fix_...\""
echo " See AGENTS.md (STATBUS-172) for how to test a repair against data written"
echo " under the corruption, not just against seed state."
exit 1
}
# cmd_migrate_up is RETIRED — see the refusal below for why.
# Symmetric counterpart to migrate-down. Edge-only.
cmd_migrate_up() {
local server="$1"
# Retired with the edge channel (King, 2026-08-19) — the symmetric
# counterpart to migrate-down above, and edge-only for the same reason.
#
# Applying pending migrations is not a thing an operator does to a release
# box by hand: the upgrade pipeline runs migrations as part of the upgrade it
# is performing, inside the backup/health-check/rollback envelope. Running
# them outside that envelope is how a box ends up half-migrated with no
# snapshot to go back to.
echo "Error: cloud.sh migrate-up is retired together with the edge channel."
echo " Requested on: ${server:-<none>}."
echo " Migrations are applied by the upgrade itself, inside its backup and"
echo " rollback envelope. To move a box forward, schedule a candidate:"
echo " ./sb upgrade register <version> && ./sb upgrade schedule <version>"
exit 1
}
# cmd_tail_one tails the upgrader journal for one server and
# auto-disconnects when a terminal state is logged. Prints the final
# upgrade status afterwards.
# Optional $2 = target_version: narrows the exit pattern to that specific
# upgrade so stale recovery log lines for previous versions don't cause
# a premature exit.
cmd_tail_one() {
local server="$1" instance
local target_version="${2:-}"
instance=$(service_instance "$server")
# Build the awk exit pattern locally before SSH so we avoid nested-quote
# hell. The pattern is embedded in the remote awk /.../ regex via double-
# quote expansion of the outer SSH string. Version strings (sha-*, v*.*)
# contain no single quotes or shell metacharacters, so expansion is safe.
local awk_pattern
if [ -n "$target_version" ]; then
awk_pattern="Upgrade to ${target_version} .*(completed|failed)|FAILED:"
else
awk_pattern="Upgrade to .*(completed|failed)|FAILED:"
fi
echo "--- Tailing upgrade log for $server (auto-disconnect on completion) ---"
ssh_entry "$server" \
"journalctl --user -u 'statbus-upgrader@${instance}.service' -u 'statbus-upgrade@${instance}.service' -o cat -f -n 50 2>&1 | \
awk '/${awk_pattern}/{print; fflush(); exit} {print; fflush()}'" \
|| true
echo "--- Log tail disconnected for $server ---"
echo "Final upgrade status on $server:"
# Poll until the DB reflects the terminal state (service commits the
# in_progress→completed transition after logging "Installation complete!").
# Bounded at 8 tries × 2 s = 16 s max; exits early once state clears.
# shellcheck disable=SC2016 # This polling script is evaluated remotely.
ssh_entry "$server" \
'cd statbus && i=0; while [ $i -lt 8 ]; do
out=$(./sb upgrade list 2>&1)
echo "$out" | head -5 | grep -qE "in[_ ]progress" || { echo "$out"; exit 0; }
i=$((i+1)); [ $i -lt 8 ] && sleep 2
done; ./sb upgrade list' 2>&1 || true
}
# cmd_tail tails the upgrade log for one server or all servers in parallel.
cmd_tail() {
local target="$1"
local targets
targets=$(resolve_target_servers "$target")
if [ "$target" = "all" ] || is_channel "$target" || is_group "$target"; then
local pids=()
for server in $targets; do
cmd_tail_one "$server" &
pids+=($!)
done
wait "${pids[@]}"
else
cmd_tail_one "$targets"
fi
}
cmd_install_one() {
# Idempotent install flow:
# install → ensure_service_started
#
# Re-running `./cloud.sh install <server>` after any partial failure
# (SSH drop, Ctrl-C, transient error) is safe — every step is rerun-safe.
# The `./sb install` dispatcher handles the running service itself
# (STATBUS-039): a stale flag reconciles via StateCrashedUpgrade; a
# crash-looping unit is taken over with a SIGKILL-class quiesce; a
# genuinely-progressing upgrade makes install REFUSE — cloud.sh then
# reports the failure and the operator retries later. Do NOT stop the
# service first: systemctl stop is SIGTERM, which an in-flight upgrade
# catches and answers with a rollback (snapshot restore) — the deploy-
# stop footgun (STATBUS-040/-041). No binary-replacement path needs a
# stop either (all are atomic renames).
#
# ensure_service_started runs at the end (and on the failure-return path)
# so a cloud.sh exit always leaves the server with the upgrader
# running, not stopped.
local server="$1"
local version="${2:-}"
local exit_code=0
local install_out=""
# Resolve trust key user: explicit env var first, then remote .env.config
# written by a prior successful run — operator sets it once, remembered forever.
local resolved_trust_user="$FLEET_TRUST_KEY_USER"
if [ -z "$resolved_trust_user" ]; then
resolved_trust_user=$(ssh_entry "$server" \
"cd statbus && ./sb dotenv -f .env.config get TRUST_GITHUB_USER 2>/dev/null" \
2>/dev/null || true)
fi
# Fast path: if no version is pinned, try the upgrader first.
# If it accepts the request (exit 0), tail the journal and return.
# If it fails (service not running, DB down, etc.), fall through to the
# full bootstrap install below.
# Version-skew guard: if remote binary != local binary, skip fast-path
# and always bootstrap (item #2 rc.64 fix — dev's looping service returned
# 0 on NOTIFY but never completed, blocking indefinitely).
if [ -z "$version" ]; then
local remote_commit local_commit
remote_commit=$(ssh_entry "$server" "cd statbus && ./sb --version 2>/dev/null" \
| grep -oE 'commit [a-f0-9]+' | awk '{print $2}') || remote_commit=""
local_commit=$(./sb --version 2>/dev/null | grep -oE 'commit [a-f0-9]+' | awk '{print $2}')
if [ -z "$remote_commit" ] || [ "$remote_commit" != "$local_commit" ]; then
echo "Version skew detected: remote=$remote_commit local=$local_commit — skipping fast-path, using bootstrap"
# Fall through to bootstrap block (do NOT enter the upgrader fast-path)
else
echo "Trying upgrader on $server..."
# Capture exit code WITHOUT triggering set -e. Pre-fix: a plain
# `apply_out=$(ssh ...)` assignment is part of the surrounding
# `set -euo pipefail` scope, so a non-zero SSH exit kills cloud.sh
# before line `apply_rc=$?` runs — operator sees only this echo
# and a bare prompt (anti-fail-fast). The `|| apply_rc=$?` form
# captures the failure code AND short-circuits set -e so the
# fall-through to the bootstrap install fires.
local apply_out apply_rc=0
apply_out=$(ssh_entry "$server" "cd statbus && ./sb upgrade apply-latest" 2>&1) || apply_rc=$?
echo "$apply_out"
# Skip-current short-circuit: when apply-latest detects the slot
# is already at the latest, it prints "Already at <ver> ..." and
# exits 0 without scheduling — no NOTIFY upgrade_apply, so the
# daemon doesn't run a pipeline. cmd_tail_one would tail forever
# waiting for a completion line that won't come. Detect the
# marker and return cleanly.
if [ "$apply_rc" -eq 0 ] && echo "$apply_out" | grep -q "^Already at "; then
return 0
fi
if [ "$apply_rc" -eq 0 ]; then
# Extract target version from apply-latest output, e.g.:
# Sent: NOTIFY upgrade_apply, '9bf48bb8' # commit_short
# Sent: NOTIFY upgrade_apply, 'v2026.04.0-rc.55' # release tag
# Passed to cmd_tail_one so the awk exit pattern is version-specific,
# preventing stale recovery lines for previous upgrades from terminating early.
local target_version
target_version=$(echo "$apply_out" | grep "upgrade_apply" | grep -oE "'[^']+'" | tr -d "'" | head -1)
cmd_tail_one "$server" "$target_version"
return $?
fi
echo "Upgrader not responsive — falling back to full bootstrap install..."
fi
fi
# (The edge install strategy stood here and is retired with the channel —
# King, 2026-08-19. It checked out origin/master and built `sb` from source
# when HEAD carried no published binary, which is what an always-latest box
# required. Every box now installs a NAMED candidate's published binary, so
# there is one procurement path instead of two, and no box builds from
# source during a deploy. The per-server channel READ that chose between the
# two strategies went with it — there is one strategy now, so there is
# nothing left to choose.)
if [ -n "$version" ]; then
# Pinned: verify artifacts for the specific version before touching the server.
echo "Checking release artifacts for $version are ready..."
if ! "$SCRIPT_DIR/sb" release verify-artifacts --tag "$version"; then
echo "--- Release artifacts for $version not ready. Retry later. ---"
return 1
fi
echo "Installing $server at $version via $INSTALL_URL ..."
install_out=$(ssh_entry "$server" \
"curl -fsSL ${INSTALL_URL} | bash -s -- --version $version $(trust_flag "$resolved_trust_user")" 2>&1) \
|| exit_code=$?
else
# An unpinned install follows the box's own channel, exactly as its
# upgrader does: stable boxes get the latest stable release, Norway
# and dev the latest candidate. The target named on the command line
# only chooses which boxes to visit. Unreadable metadata stops here,
# before anything is installed, rather than guessing a channel.
local box_metadata box_channel
if ! box_metadata=$(read_server_metadata "$server"); then
echo "--- Cannot read the channel $server follows; nothing was installed. Check ./sb config show on $server and retry. ---"
return 1
fi
IFS='|' read -r _ box_channel _ <<< "$box_metadata"
# Gate: verify release artifacts are fully published before touching
# the server. If CI is still uploading assets or pushing
# images, abort early — the server stays up and the operator retries.
# Rc.63: use --channel so the check resolves to the channel's current
# latest release instead of treating the channel name as a literal tag.
echo "Checking release artifacts for channel $box_channel are ready..."
if ! "$SCRIPT_DIR/sb" release verify-artifacts --channel "$box_channel"; then
echo "--- Release artifacts not ready. Retry in ~5 minutes. ---"
return 1
fi
echo "Installing $server ($box_channel channel) via $INSTALL_URL ..."
# Step 1: Run install.sh as the app user. No pre-stop: install.sh
# swaps ./sb via atomic rename (sb.tmp + mv, never ETXTBSY), and
# `./sb install` refuses-or-takes-over a running upgrade itself
# (STATBUS-039/-041).
# Exit code 42 = service needs root (not a failure).
install_out=$(ssh_entry "$server" \
"curl -fsSL ${INSTALL_URL} | bash -s -- --channel $box_channel $(trust_flag "$resolved_trust_user")" 2>&1) \
|| exit_code=$?
fi
printf '%s\n' "$install_out"
if [ "$exit_code" -ne 0 ]; then
echo "--- $server install FAILED (exit code $exit_code) ---"
if [ -z "$resolved_trust_user" ] && printf '%s\n' "$install_out" | grep -Eqi 'failed signature verification|signature verification failed|release signature could not be verified'; then
echo ""
echo "If this failed because of an invalid signing key, re-run with:"
echo " FLEET_TRUST_KEY_USER=jhf ./cloud.sh install $server"
echo ""
fi
# Do NOT call ensure_service_started on failure — starting the upgrade
# service after a broken install can hang (systemctl waiting on a broken
# binary/DB). The operator re-runs ./cloud.sh install which calls it on success.
return 1
fi
# Persist trust user for future runs so CLOUD_TRUST_KEY_USER need not
# be set again. Idempotent — safe to re-write the same value.
if [ -n "$resolved_trust_user" ]; then
ssh_entry "$server" \
"cd statbus && ./sb dotenv -f .env.config set TRUST_GITHUB_USER '$resolved_trust_user'" \
2>/dev/null || true
fi
# Regenerate config so VERSION in .env matches the checked-out code.
# Must use 'up -d' not 'restart' — restart doesn't re-read .env.
echo "Regenerating config and restarting app..."
ssh_entry "$server" "cd statbus && ./sb config generate && docker compose up -d app" 2>&1
# Always leave the upgrader running on success, regardless of
# whether install's own service-install step fired (e.g., when running
# without root and the user-level path was used).
ensure_service_started "$server"
echo "--- $server install complete ---"
}
cmd_wipe_one() {
local target="$1" confirm phrase
phrase="wipe $(entry_domain "$target")"
echo "WARNING: This will DELETE the database on $target and recreate from scratch."
echo "ALL DATA WILL BE LOST."
echo "Type '$phrase' to confirm:"
read -r confirm
if [ "$confirm" != "$phrase" ]; then
echo "Aborted."
exit 1
fi
echo "Wiping $target..."
ssh_entry "$target" "cd statbus && ./dev.sh recreate-database && ./sb start all" 2>&1
echo "--- $target wipe complete ---"
}
cmd_wipe() {
local target="$1" code group ran=0
if [ "$target" != all ]; then
validate_code "$target"
# One box: a standalone box gets its production-safe wipe
# (STATBUS-587); a cloud slot its dev.sh recreate-database.
if [ "$(entry_group "$target")" = standalone ]; then
cmd_standalone_wipe "$target"
else
cmd_wipe_one "$target"
fi
return
fi
for code in $(all_codes); do
group=$(entry_group "$code")
if [ "$group" != cloud ]; then
echo "$code: skipped (wipe is only available for cloud targets)"
continue
fi
cmd_wipe_one "$code"
ran=1
done
[ "$ran" -eq 1 ]
}
cmd_create_one() {
local code="$1"
local name="$2"
local version="$3"
"$SCRIPT_DIR/ops/create-new-statbus-installation.sh" "$code" "$name" "$version"
}
cmd_create() {
local target="$1" name="$2" version="$3" code group ran=0
if [ "$target" != all ]; then
cmd_create_one "$target" "$name" "$version"
return
fi
for code in $(all_codes); do
group=$(entry_group "$code")
if [ "$group" != cloud ]; then
echo "$code: skipped (create is only available for cloud targets)"
continue
fi
cmd_create_one "$code" "$name" "$version"
ran=1
done
[ "$ran" -eq 1 ]
}
cmd_inspect() {
if [ "${1:-cloud}" = all ]; then
echo "inspect all is not supported; use 'inspect cloud'." >&2
return 2
fi
exec "$SCRIPT_DIR/ops/inspect-cloud-installations.sh"
}
cmd_ssh() {
local target
if [ "$1" = all ]; then
echo "ssh all is not supported; specify one standalone box." >&2
return 2
fi
target=$(resolve_single_target "$1")
echo "Connecting to $(entry_ssh_target "$target") ..."
ssh_entry "$target"
}
cmd_standalone_wipe() {
# Production-safe DB wipe for standalone hosts. Does NOT use dev.sh —
# dev.sh assumes the Go toolchain is installed (for the sb-rebuild
# check) and also calls `./sb build all_except_app` + `./dev.sh
# create-test-template`, neither of which belong on a pinned-release
# production host. Instead:
#
# 1. ./sb stop all — stop app/worker/rest/proxy/db
# 2. docker volume rm <db-data> — blow away the PG data volume
# 3. ./sb install --non-interactive
# — 14-step dispatcher handles
# start db + seed restore +
# migrate up + JWT secret + users
# + trusted signers + upgrade svc
# all idempotently.
#
# End state is identical to `dev.sh recreate-database` on a dev box,
# minus the test-template db (which we never want on prod anyway).
local target="$1"
if [ "$target" = "all" ]; then
echo "ERROR: wipe all is not supported. Wipe deployments one at a time."
exit 1
fi
local resolved_target
resolved_target=$(resolve_single_target "$target")
local fqdn dom
fqdn=$(entry_ssh_target "$resolved_target")
dom=$(entry_domain "$resolved_target")
echo "WARNING: This will DELETE the database for the '$resolved_target' deployment"
echo " (serving $dom, hosted on $fqdn) and recreate it from scratch."
echo "ALL DATA WILL BE LOST."
# The confirmation names the action and the domain: a bare box code is no
# answer, since Norway's code is the word "no" (STATBUS-587).
echo "Type 'wipe $dom' to confirm:"
read -r confirm
if [ "$confirm" != "wipe $dom" ]; then
echo "Aborted."
exit 1
fi
echo "Wiping $resolved_target..."
ssh_entry "$resolved_target" "set -e
cd statbus
echo '--- stopping services ---'
./sb stop all
echo '--- removing DB docker volume ---'
INSTANCE_NAME=\$(./sb dotenv -f .env get COMPOSE_INSTANCE_NAME 2>/dev/null || echo '')
if [ -z \"\$INSTANCE_NAME\" ]; then
echo 'ERROR: COMPOSE_INSTANCE_NAME not set in .env — cannot identify DB volume'
exit 1
fi
VOL=\"\${INSTANCE_NAME}-db-data\"
if docker volume inspect \"\$VOL\" >/dev/null 2>&1; then
docker volume rm \"\$VOL\"
echo \"Removed volume \$VOL\"
else
echo \"Volume \$VOL already absent\"
fi
echo '--- re-running ./sb install (step-table populates empty DB) ---'
./sb install --non-interactive" 2>&1
echo "--- $resolved_target wipe complete ---"
}
# resolve_user_email returns the email to use for cmd_import.
# Precedence: explicit positional email > STATBUS_REIMPORT_USER_EMAIL env >
# interactive prompt (TTY only). Aborts with a clear message on a closed
# stdin (e.g. piped invocation in CI) so the caller fixes the call site
# rather than silently importing as a default user.
resolve_user_email() {
local flag_value="${1:-}"
if [ -n "$flag_value" ]; then
echo "$flag_value"
return 0
fi
if [ -n "${STATBUS_REIMPORT_USER_EMAIL:-}" ]; then
echo "$STATBUS_REIMPORT_USER_EMAIL"
return 0
fi
if [ ! -t 0 ]; then
echo "ERROR: import email required (or set STATBUS_REIMPORT_USER_EMAIL)" >&2
echo " The BRREG import script verifies the email exists in public.user;" >&2
echo " the import_job rows are then attributed to that user." >&2
exit 1
fi
local email
read -r -p "User email (must exist in public.user on the target): " email </dev/tty
if [ -z "$email" ]; then
echo "Aborted: empty email." >&2
exit 1
fi
echo "$email"
}
# cmd_import schedules BRREG import jobs on the target deployment by
# running the appropriate sample script remotely with USER_EMAIL
# exported. After scheduling, the worker picks up the import_job rows
# via LISTEN and processes them asynchronously; this command does NOT
# wait for completion.
#
# Two variants:
# selection — small dataset, CSVs ship in-repo under
# samples/norway/{legal_unit,establishment,legal_relationship}/.
# Loaded by samples/norway/brreg/brreg-import-selection.sh.
# No external download needed; runs on a fresh wipe.
# downloads — full BRREG dataset, CSVs must already be on the target
# host under ~statbus/statbus/tmp/. Loaded by
# samples/norway/brreg/brreg-import-downloads-from-tmp.sh.
# These CSVs are preserved across wipes (host filesystem,
# not the Docker volume).
#