Repository navigation
Expand file tree
/
Copy pathdev.sh
More file actions
executable file
·4010 lines (3720 loc) · 201 KB
/
Copy pathdev.sh
File metadata and controls
executable file
·4010 lines (3720 loc) · 201 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#!/bin/bash
# dev.sh — Development-only commands for StatBus
#
# These commands are for local development and are NOT available in production.
# For production/ops commands, use ./sb (the Go CLI).
#
# Usage: ./dev.sh <command> [args...]
#
set -euo pipefail
if [ "${DEBUG:-}" = "true" ] || [ "${DEBUG:-}" = "1" ]; then
set -x
fi
# Ensure Homebrew tools (Go, etc.) are in PATH on servers
if [ -f /home/linuxbrew/.linuxbrew/bin/brew ]; then
eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"
fi
WORKSPACE="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
cd "$WORKSPACE"
source "$WORKSPACE/ops/dev-seed-restore.sh"
# Activate repo git hooks. `.githooks/pre-push` blocks hand-rolled release
# tags and guards the postgres/Dockerfile pgrx-builder stages. A fresh
# clone has core.hooksPath unset (defaults to .git/hooks, which is empty
# here), so the guards silently wouldn't run until a developer manually
# ran the config. Setting it on every dev.sh invocation is idempotent and
# ensures the guards are live for anyone who uses dev.sh at all.
if [ "$(git config core.hooksPath 2>/dev/null || true)" != ".githooks" ]; then
git config core.hooksPath .githooks
fi
# Rebuild ./sb when EITHER drift axis fires — they need different evidence,
# and neither tool can see the other's axis:
# - the binary doesn't exist, OR
# - HOT-EDIT axis: any cli/**/*.go is newer than the binary (local WIP edit).
# Only a file mtime catches this — git can't tell whether the binary was
# built from the current *uncommitted* bytes. Self-clears on rebuild.
# - COMMITTED axis: the binary's build commit differs from HEAD with cli/
# changes between. `git commit`/`checkout`/`pull` move HEAD WITHOUT touching
# working-tree mtimes, so the mtime check above is blind to it. `./sb
# committed-drift` answers from the binary's baked-in build commit vs live
# HEAD (reliable even when the binary is stale) and exits non-zero on drift
# (or if it cannot confirm freshness). It is guard-exempt, so it never warns
# into this check.
sb_needs_rebuild=false
sb_has_dirty_cli=false
if [ -n "$(git status --porcelain --untracked-files=all -- cli/ 2>/dev/null)" ]; then
# Git, not mtimes, is the authority for unpublished source bytes. This must
# precede the missing-binary case: a fresh clone with a dirty cli/ and no sb
# must build those bytes, never procure clean HEAD and silently ignore them.
sb_needs_rebuild=true
sb_has_dirty_cli=true
elif ! test -x ./sb; then
sb_needs_rebuild=true
elif [ -n "$(find cli -name '*.go' -newer ./sb -print -quit 2>/dev/null)" ]; then
sb_needs_rebuild=true
elif ! ./sb committed-drift; then
sb_needs_rebuild=true
fi
if [ "$sb_needs_rebuild" = true ]; then
sb_procured=false
run_bounded() {
local seconds="$1" log="$2"
shift 2
"$@" >"$log" 2>&1 &
local command_pid=$!
(
sleep "$seconds"
if kill -0 "$command_pid" 2>/dev/null; then
echo "Timed out after ${seconds}s: $*" >>"$log"
# Shell-script test doubles and some CLIs spawn a child process;
# terminate children first so the deadline cannot leave an
# orphan holding pipes or network connections open.
pkill -TERM -P "$command_pid" 2>/dev/null || true
kill -TERM "$command_pid" 2>/dev/null || true
sleep 1
pkill -KILL -P "$command_pid" 2>/dev/null || true
kill -KILL "$command_pid" 2>/dev/null || true
fi
) &
local timer_pid=$!
local command_rc=0
wait "$command_pid" || command_rc=$?
kill "$timer_pid" 2>/dev/null || true
wait "$timer_pid" 2>/dev/null || true
return "$command_rc"
}
# A clean checkout should consume the exact per-commit binary Images already
# published, not ask GOTOOLCHAIN=auto to download go.mod's toolchain merely
# to enumerate tests. This is the shell bootstrap equivalent of
# cli/internal/sbimage. Any git-visible cli/ edit deliberately bypasses it:
# only a source build can include bytes that are not in HEAD's image.
_SB_HOST_OS=$(uname -s)
if [ "$sb_has_dirty_cli" = false ] && [ "$_SB_HOST_OS" != "Darwin" ] && command -v docker >/dev/null 2>&1; then
_SB_SHORT=$(git rev-parse --short=8 HEAD 2>/dev/null || true)
_SB_IMAGE="ghcr.io/statisticsnorway/statbus-sb:${_SB_SHORT}"
_SB_IMAGE_LOG=$(mktemp)
if [ -n "$_SB_SHORT" ]; then
# Probe before pull. Unpushed/non-master commits have no image, and
# `docker pull` can wait for minutes before saying so. Discovery jobs
# get a 15s existence decision, then keep the source-build fallback.
if run_bounded 15 "$_SB_IMAGE_LOG" docker manifest inspect "$_SB_IMAGE"; then
echo "Procuring sb from image ${_SB_IMAGE} (no host toolchain)..." >&2
if ! run_bounded 120 "$_SB_IMAGE_LOG" docker pull "$_SB_IMAGE"; then
cat "$_SB_IMAGE_LOG" >&2
rm -f "$_SB_IMAGE_LOG"
echo "::error title=Infrastructure: sb image pull failed::manifest exists for ${_SB_IMAGE}, but its bounded pull failed. Retry after GHCR/CDN recovers; refusing a host-Go fallback for a published artifact." >&2
exit 1
fi
cat "$_SB_IMAGE_LOG" >&2
_SB_CID=$(docker create "$_SB_IMAGE")
_SB_TMP=$(mktemp "$WORKSPACE/.sb.procure.XXXXXX")
if docker cp "${_SB_CID}:/sb" "$_SB_TMP" && chmod +x "$_SB_TMP" && mv "$_SB_TMP" ./sb; then
sb_procured=true
else
rm -f "$_SB_TMP"
docker rm -f "$_SB_CID" >/dev/null 2>&1 || true
rm -f "$_SB_IMAGE_LOG"
echo "::error title=Infrastructure: sb image extraction failed::pulled ${_SB_IMAGE}, but could not extract /sb. Refusing to invoke Go for a published commit artifact." >&2
exit 1
fi
docker rm "$_SB_CID" >/dev/null
else
cat "$_SB_IMAGE_LOG" >&2
echo "No published sb image confirmed for ${_SB_SHORT} within 15s; falling back to a local source build." >&2
fi
fi
rm -f "$_SB_IMAGE_LOG"
elif [ "$sb_has_dirty_cli" = true ]; then
echo "Uncommitted cli/ changes detected; building sb from source instead of procuring clean HEAD." >&2
elif [ "$_SB_HOST_OS" = "Darwin" ]; then
echo "Darwin host detected; building sb from source instead of extracting the Linux image binary." >&2
fi
if [ "$sb_procured" = false ] && command -v go >/dev/null 2>&1; then
echo "Building sb from source..." >&2
# Inject version from git describe verbatim — it carries the leading "v"
# (the canonical CommitVersion form stored in public.upgrade.commit_version
# and printed by ./sb --version). No strip/re-prepend dance (STATBUS-064).
# --match 'v[0-9]*' restricts git describe to release tags. The moving
# install-verified tag was deleted in rc.62; this filter remains as
# defense against any stray non-release tags landing in the refs/tags/ space.
_SB_VERSION=$(git describe --tags --always --match 'v[0-9]*' 2>/dev/null || echo "dev")
# Full 40-char commit_sha for cmd.commit ldflag — equality-compared
# against public.upgrade.commit_sha in the upgrader's
# ground-truth check. Display-only trimming happens via
# upgrade.ShortForDisplay() / commitShort() in Go (rc.63 canonical).
_SB_COMMIT=$(git rev-parse HEAD 2>/dev/null || echo "unknown")
_SB_LDFLAGS="-X 'github.com/statisticsnorway/statbus/cli/cmd.version=${_SB_VERSION}' -X 'github.com/statisticsnorway/statbus/cli/cmd.commit=${_SB_COMMIT}'"
(cd cli && go build -ldflags "$_SB_LDFLAGS" -o ../sb .)
elif [ "$sb_procured" = false ]; then
echo "Error: ./sb binary not found or out of date. Build it with: cd cli && go build -o ../sb ." >&2
exit 1
fi
fi
# Auto-fetch DB seed if not cached locally.
# Intent: speeds up create-db from ~294 migrations to one pg_restore (~2 seconds).
# Uses ./sb db seed fetch — one implementation in Go, shared by dev.sh and ./sb install.
# Placed AFTER the rebuild block so the current binary is always used.
#
# NON-FATAL: the seed is a pure speed optimization. `./sb db seed fetch` exits 1
# ("manifest unknown") when the published statbus-seed:<commit_short> image does
# not exist for this commit — a freshly-pushed commit whose Images run hasn't
# built the seed yet, or any non-master ref Images never builds at all. Under
# `set -e` an unguarded failure here aborts the WHOLE ./dev.sh invocation; that
# killed the install-recovery harness's `build-sb` discover step on a fresh
# commit (STATBUS-025 — the build phase died before any scenario ran). Mirror
# install.go's runSeedRestore: warn and proceed; create-db just replays all
# migrations instead of restoring the seed.
if [ ! -f "$WORKSPACE/.db-seed/seed.pg_dump" ] && [ -x ./sb ]; then
# postgres-variables is consumed by eval in subprocesses. Keep fetch's
# human-facing stdout off that machine-readable stream (including when a
# live test replaces docker with a PATH shim).
./sb db seed fetch >&2 || echo "Note: no seed image for this commit — proceeding without the seed cache (create-db will replay all migrations)." >&2
fi
# Set TTY_INPUT to /dev/tty if available (interactive), otherwise /dev/null
if [ -e /dev/tty ]; then
export TTY_INPUT=/dev/tty
else
export TTY_INPUT=/dev/null
fi
# ---- Tier-1 stamp guard ----
#
# Used by `./dev.sh test fast`, `./dev.sh generate-doc-db`, and
# (via a parallel Go implementation in cli/cmd/types.go) `./sb types generate`.
# Three outcomes, each printed verbatim to stdout with reason + evidence +
# override hint:
#
# REFUSED any file inside the caller's content scope has uncommitted
# changes. A stamp written now would not honestly reflect HEAD
# (dirty files aren't in the commit the stamp records), so refuse
# before doing any work.
# SKIPPED the stamp file exists, points to an ancestor of HEAD, and no
# file in the caller's content scope has changed between that
# ancestor and HEAD — re-running the command would produce an
# identical result.
# RUNNING normal execution. No stamp, or the stamp is orphaned (not an
# ancestor of HEAD — branch switch, rebase, unknown commit), or
# in-scope content drifted since the stamp.
#
# Escape hatches:
# FORCE=1 bypass all guards, always run + stamp.
# rm tmp/<stamp> force next invocation from SKIP to RUN.
#
# Arguments: <command_label> <stamp_basename> <scope_path...>
# REFUSE and SKIP share the same scope — the files the command actually
# consumes. Fast-test passes "migrations test"; types/db-docs pass just
# "migrations". Non-strict baseline paths (test/expected/explain,
# test/expected/performance) are always excluded from dirty-checks: they
# drift with environment and shouldn't block a release.
#
# Return codes: 0 = RUN (continue), 1 = SKIP (caller should exit 0),
# 2 = REFUSE (caller should exit 1).
check_stamp_guard() {
local label="$1"
local stamp_basename="$2"
shift 2
local scopes=("$@")
local stamp_path="$WORKSPACE/tmp/$stamp_basename"
# Non-strict baselines: routine environment drift, never block release.
local excludes=(':!test/expected/explain/' ':!test/expected/performance/')
if [ "${FORCE:-}" = "1" ] || [ "${FORCE:-}" = "true" ]; then
echo "RUNNING: $label"
echo "Reason: FORCE=1 — guard bypassed."
return 0
fi
# RUN_NO_STAMP (rc 3): any file inside the scope (minus excludes) has
# uncommitted staged or unstaged changes — you're (almost certainly) landing
# a migration. RUN the work but DO NOT write the freshness stamp: a stamp from
# a dirty tree can't honestly point at a commit (FORCE=1's old escape did
# exactly that). The release preflight RE-DERIVES freshness (release.go
# checkMigrationStamp), so withholding is fail-safe — after the commit a
# clean-tree re-run writes an honest stamp. No override needed.
local dirty
dirty=$(git -C "$WORKSPACE" status --porcelain -- "${scopes[@]}" "${excludes[@]}" 2>/dev/null)
if [ -n "$dirty" ]; then
echo "RUNNING: $label — freshness stamp DEFERRED"
echo "Reason: uncommitted changes in ${scopes[*]} — running now, but NOT writing the"
echo " freshness stamp (a stamp from a dirty tree can't honestly point at a"
echo " commit). Release preflight re-derives freshness, so after you commit,"
echo " re-run this on a clean tree to write the stamp."
echo "If you're landing a migration, the full no-override flow is:"
echo " 1. ./sb migrate up --target seed && ./dev.sh create-test-template # seed->HEAD (non-destructive)"
echo " 2. ./dev.sh generate-doc-db && ./sb types generate # regenerate"
echo " 3. git add migrations/ doc/db/ app/src/lib/database.types.ts <your code>"
echo " 4. git commit # pre-commit pairs migration+regen"
echo " 5. after commit (clean tree) re-run step 2 once -> writes the release stamp"
echo "Do NOT FORCE=1 to land a migration — it writes a stamp from a dirty tree, the exact"
echo "lie this guard prevents. FORCE=1 is only for regenerating against an already-committed"
echo "schema (e.g. a generator change with no new migration)."
echo "Evidence (uncommitted in ${scopes[*]}):"
printf '%s\n' "$dirty" | sed 's/^/ /'
return 3
fi
if [ ! -f "$stamp_path" ]; then
echo "RUNNING: $label"
echo "Reason: no stamp at tmp/$stamp_basename — no prior successful run to skip."
return 0
fi
# Two-line stamp format: line 1 = HEAD SHA, line 2 = source DB
# migration_version. Extract line 1 for the freshness check below.
# Pre-upgrade legacy stamps have only line 1; the count check after
# this one catches and force-RUNs them so the generator upgrades the
# stamp format on next run. Without that short-circuit, an operator
# with a legacy stamp + no migrations-changed gets stuck in a SKIP
# loop: preflight refuses the stamp ("legacy single-line"), generator
# SKIPs because the stamp's SHA still matches HEAD's migrations.
local stamp_sha
stamp_sha=$(head -n 1 "$stamp_path" 2>/dev/null | tr -d '[:space:]')
if [ -z "$stamp_sha" ]; then
echo "RUNNING: $label"
echo "Reason: stamp tmp/$stamp_basename is empty."
return 0
fi
# Legacy single-line stamp detection. Two-line stamps have ≥2
# non-blank lines (SHA + migration_version); legacy stamps have 1.
# Force RUN so the generator writes the upgraded format. awk 'NF>0'
# filters non-blank lines, wc -l counts them.
local non_blank_lines
non_blank_lines=$(awk 'NF>0' "$stamp_path" 2>/dev/null | wc -l | tr -d ' ')
if [ "${non_blank_lines:-0}" -lt 2 ]; then
echo "RUNNING: $label"
echo "Reason: legacy single-line stamp at tmp/$stamp_basename — two-line format required by preflight."
return 0
fi
if ! git -C "$WORKSPACE" merge-base --is-ancestor "$stamp_sha" HEAD 2>/dev/null; then
echo "RUNNING: $label"
echo "Reason: stamp SHA $stamp_sha is not an ancestor of HEAD (branch switch, rebase, or unknown commit)."
return 0
fi
local head_sha
head_sha=$(git -C "$WORKSPACE" rev-parse HEAD 2>/dev/null)
local changed
changed=$(git -C "$WORKSPACE" diff --name-only "$stamp_sha" HEAD -- "${scopes[@]}" 2>/dev/null)
if [ -z "$changed" ]; then
# Version-coherence check: the SHA-diff axis says "no change" but the
# stamp's source-DB version (line 2) must ALSO match HEAD's on-disk max
# migration. If they diverge, release.go preflight would refuse this
# stamp — the "catch-22" scenario where a migration was applied to dev
# DB then later reverted (file deleted from tree). RUNNING so regen
# writes a fresh, coherent stamp and breaks the loop.
local stamp_version disk_max_version
stamp_version=$(sed -n '2p' "$stamp_path" 2>/dev/null | tr -d '[:space:]')
disk_max_version=$(find "$WORKSPACE/migrations" -maxdepth 1 \
\( -name '*.up.sql' -o -name '*.up.psql' \) 2>/dev/null \
| sed -E 's|.*/([0-9]{14})_.*|\1|' | sort -r | head -1)
if [ -n "$stamp_version" ] && [ -n "$disk_max_version" ] && \
[ "$stamp_version" != "$disk_max_version" ]; then
echo "RUNNING: $label"
echo "Reason: SHA-diff is empty but stamp's source-DB version ($stamp_version)"
echo " != HEAD's on-disk max migration ($disk_max_version)."
echo " release.go preflight would refuse this stamp; regen must run."
echo " (catch-22: migration applied to dev DB then reverted from tree)"
echo "Evidence:"
echo " stamp SHA: $stamp_sha"
echo " stamp source-DB version: $stamp_version"
echo " HEAD on-disk max migration: $disk_max_version"
return 0
fi
echo "SKIPPED: $label"
echo "Reason: stamp tmp/$stamp_basename points to a commit whose ${scopes[*]} content matches HEAD — re-running would produce an identical result."
echo "Evidence:"
echo " stamp SHA: $stamp_sha"
echo " HEAD SHA: $head_sha"
echo " files changed in scope (${scopes[*]}): 0"
echo "Override: rm tmp/$stamp_basename, or set FORCE=1."
return 1
fi
echo "RUNNING: $label"
echo "Reason: in-scope content has drifted since stamp."
echo "Evidence:"
echo " stamp SHA: $stamp_sha"
echo " HEAD SHA: $head_sha"
echo " files changed in scope (${scopes[*]}):"
printf '%s\n' "$changed" | sed 's/^/ /'
return 0
}
# the bash assert_db_at_head() helper has moved to Go.
# Single source of truth lives in cli/internal/migrate/at_head.go and is
# exposed at the CLI surface as `./sb assert-db-at-head <db_name> <caller>`.
# Reasons for the move:
# - The Go path can hold a PG advisory lock during the query window
# (shared variant of hashtext('statbus_seed_mutate')), serialising
# against `./sb db with-seed-lock --exclusive` so a parallel
# recreate-seed doesn't drop statbus_seed mid-query.
# - Two implementations of the same diagnostic-shape contract is one
# too many. Drift between the bash and Go copies has bitten us
# before (template-vs-seed targeting, preflight ordering); the
# bash copy is now retired.
action=${1:-}
shift || true
# ── Expected-output guardrail ─────────────────────────────────────────
#
# Used by `./dev.sh test ... --update-expected` and `make-all-failed-test-
# results-expected`. Detects NEW ERROR/FATAL/PANIC lines that the regen
# would introduce into a test/expected/*.out file. If any new error
# lacks a WANTED-context marker in the surrounding result (SAVEPOINT,
# \set ON_ERROR_STOP off, DO $$ EXCEPTION block, or "-- expected to
# fail" comment), BLOCK the regen with an educational message.
#
# Rationale: silently baselining a real error into expected hides bugs
# from CI. Future runs match the bug; everyone forgets it's there; it
# grows roots. This forces a deliberate decision at regen time.
#
# Cascade errors ("current transaction is aborted, commands ignored")
# are skipped — consequences of an earlier ERROR, not new failures.
#
# Override (rare, deliberate): ACCEPT_NEW_ERRORS=true
_unmarked_new_errors=""
_has_wanted_marker() {
local file="$1" center_line="$2"
local start=$((center_line - 10))
[ $start -lt 1 ] && start=1
local end=$((center_line + 5))
sed -n "${start},${end}p" "$file" | grep -qE '\\set ON_ERROR_STOP off|^SAVEPOINT[[:space:]]+[a-zA-Z_][a-zA-Z0-9_]*|^DO[[:space:]]+\$\$|--[[:space:]]*(expected|will|should)[[:space:]]+(to[[:space:]]+)?(fail|error)'
}
_check_new_errors() {
local expected="$1" result="$2"
_unmarked_new_errors=""
local error_linenums
if [ ! -f "$expected" ]; then
error_linenums=$(grep -nE '^(ERROR|FATAL|PANIC):' "$result" 2>/dev/null | cut -d: -f1 || true)
else
error_linenums=$(diff -u "$expected" "$result" 2>/dev/null | awk '
/^@@/ {
for (i = 1; i <= NF; i++) {
if ($i ~ /^\+/) {
sub(/^\+/, "", $i)
split($i, a, ",")
new_line = a[1] - 1
break
}
}
next
}
/^---/ || /^\+\+\+/ { next }
/^\+/ {
new_line++
if (/^\+(ERROR|FATAL|PANIC):/) print new_line
next
}
/^-/ { next }
{ new_line++ }
' || true)
fi
[ -z "$error_linenums" ] && return 0
local lineno line_text
while IFS= read -r lineno; do
[ -z "$lineno" ] && continue
line_text=$(sed -n "${lineno}p" "$result")
if echo "$line_text" | grep -q "current transaction is aborted"; then
continue
fi
if ! _has_wanted_marker "$result" "$lineno"; then
_unmarked_new_errors="${_unmarked_new_errors} ${expected#$WORKSPACE/}:${lineno}: ${line_text}"$'\n'
fi
done <<< "$error_linenums"
[ -z "$_unmarked_new_errors" ] && return 0
return 1
}
safe_update_expected() {
local result="$1" expected="$2"
if _check_new_errors "$expected" "$result"; then
cp "$result" "$expected"
return 0
fi
if [ "${ACCEPT_NEW_ERRORS:-false}" = "true" ]; then
echo " WARNING (ACCEPT_NEW_ERRORS=true override): baselining unmarked error(s):" >&2
printf '%s' "$_unmarked_new_errors" >&2
cp "$result" "$expected"
return 0
fi
cat >&2 <<EOF
═══════════════════════════════════════════════════════════════════════
BLOCKED: ${expected#$WORKSPACE/} would gain ERROR/FATAL/PANIC line(s) without WANTED-context markers.
Unmarked new error(s):
${_unmarked_new_errors}
WHY: silently baselining unexpected errors into expected files hides
bugs from CI. Future runs match the bug; everyone forgets it's there;
it grows roots.
FIX OPTIONS:
1. INTENTIONAL (bug-capture for TDD, or testing a failure path)
— add an explicit marker to the test SQL:
- Wrap in SAVEPOINT sp_<name> / ROLLBACK TO SAVEPOINT sp_<name>, OR
- Bracket with \\set ON_ERROR_STOP off ... \\set ON_ERROR_STOP on, OR
- Use DO \$\$ ... EXCEPTION WHEN ... END \$\$ for assertions, OR
- Add a comment immediately before the failing line:
-- expected to fail with <error class>
Then re-run regen. The marker travels into the expected output
and the guardrail recognises the intent.
2. ACCIDENTAL (a real bug surfacing) — FIX THE TEST SQL.
Don't baseline the bug. Investigate the failure first.
3. Deliberate override (rare; cite reason in commit message):
ACCEPT_NEW_ERRORS=true ./dev.sh ${action} ...
Hook source: dev.sh safe_update_expected
═══════════════════════════════════════════════════════════════════════
EOF
exit 1
}
# ── Test-run serialization lock ───────────────────────────────────────
#
# NORTH STAR: two invocations that touch the shared pg_regress state
# (statbus_seed / statbus_test_template / the dev database) must never
# overlap — concurrent clones and drops corrupt each other and manufacture
# "flaky" failures that are not real ("there are no flaky tests"). We
# serialize DIRECTLY: an entrypoint that touches shared state takes an
# exclusive lock first; on contention it FAILS LOUDLY naming the holder —
# it never queues silently and never proceeds.
#
# MECHANISM: a real kernel lock — flock(2). flock(1) is absent on macOS
# (`command -v flock` exits non-zero here) and we can't call syscall.Flock
# from bash, but perl (5.x, always shipped on macOS + Linux) can flock a file
# descriptor INHERITED from the shell. We `exec 9>>lockfile` in bash, then
# perl fdopens fd 9 and takes flock(LOCK_EX|LOCK_NB). The lock lives with the
# open file DESCRIPTION behind fd 9, which gives three properties a userspace
# mkdir/pidfile lock cannot:
# - No TOCTOU. The kernel grants the lock atomically; there is no
# read-holder-then-reclaim window where two runs can both win.
# - Released on process-tree death, even SIGKILL — the kernel drops the lock
# when the LAST descriptor on the description closes. No EXIT trap needed.
# - Held while any CHILD of a killed run is still alive: children inherit
# fd 9, so a SIGKILL'd parent whose psql/docker child is still mutating the
# DB keeps the lock until that child exits. (kill -0 on the parent pid, by
# contrast, would false-reclaim mid-mutation, and PID reuse could false-block.)
# The holder file is now purely INFORMATIONAL — it names the holder in the
# contention banner; it is NOT the mutex, so a race to write it is harmless.
#
# RE-ENTRANCY: composite entrypoints spawn leaf ones as child `./dev.sh`
# processes (migrate-and-test → recreate-seed/create-test-template/test;
# create-db → create-db-structure/recreate-seed/create-test-template;
# recreate-database → delete-db/create-db). The outermost process exports
# STATBUS_TEST_LOCK_HELD; children see it and pass through (they already
# inherited fd 9, which keeps the lock held for their lifetime), so the lock
# is taken once and released once, by the top of the process tree.
_TEST_LOCK_FILE="$WORKSPACE/tmp/.test-run.lock"
_TEST_LOCK_INFO="$WORKSPACE/tmp/.test-run.lock.info"
_TEST_LOCK_OWNED=false # true ONLY in the process that took the lock
release_test_run_lock() {
# Idempotent; a no-op unless THIS process took the lock. Child processes
# inherited STATBUS_TEST_LOCK_HELD and never acquired, so their
# _TEST_LOCK_OWNED stays false and they cannot release the parent's lock.
# The kernel also releases on exit; closing fd 9 here makes it prompt on a
# clean exit and clears the informational holder file.
if [ "$_TEST_LOCK_OWNED" = true ]; then
# NB: no `2>/dev/null` on a no-command `exec` — it would permanently
# redirect THIS shell's stderr, not just silence the close. fd 9 is
# guaranteed open here (we own the lock), so the bare close won't error.
exec 9>&- || true
rm -f "$_TEST_LOCK_INFO" 2>/dev/null || true
_TEST_LOCK_OWNED=false
fi
}
# ── STATBUS-188 AC#4: long-test ownership + timeout policy ──
#
# Test executions are OWNED by the TESTER agent (single-owner serialization,
# King-ruled 2026-07-30) — no other agent runs a test suite against the
# shared db, full stop. This dev.sh has no timeout knob of its own for a
# regeneration run (grep-verified — none exists); the discipline is entirely
# in HOW the owning agent invokes it.
#
# 400/401-class regenerations (a full expected-file rebuild against a large
# fixture, historically ~20-28 minutes for 401) MUST run either detached
# (backgrounded, no wrapper-imposed budget) or under an EXPLICITLY raised
# timeout long enough to let the real run finish. A generic background-task
# runner's default budget killing a regeneration MID-FLIGHT is exactly what
# manufactures a straggler: the killed run's pg_regress/psql orphans inside
# the db container (the flock above releases on host-process death, but that
# fd is never inherited into the container — see the straggler guard just
# below). This is not hypothetical: it is the STATBUS-175 batch-3 external-
# kill incident (2026-07-29) AND the original 401 chain-starter (2026-07-14)
# that opened this ticket. Size the run's own timeout to the workload, or
# detach it — never let an unrelated runner-budget default decide.
#
# ── STATBUS-158: straggler pg_regress guard + NUL-corruption tripwire ──
#
# WHY: the flock above lives on a HOST file descriptor and is released the
# instant this process tree dies, even via SIGKILL — but pg_regress runs
# INSIDE the db container via `docker compose exec`, spawned by containerd,
# and does NOT inherit that fd. Kill a harness run after pg_regress starts
# and the flock frees immediately while pg_regress (or its psql child, if
# pg_regress itself died first) keeps running and writing in the container.
# The next invocation then acquires the lock legitimately and starts a
# SECOND pg_regress into the same --outputdir — two writers on one .out
# file corrupt it silently: writer B's fopen truncates and writes its head,
# writer A's next flush lands at its own saved offset, and the kernel fills
# the gap between — observed (STATBUS-286, SEEK_HOLE probe on two corrupted
# artifacts) as a ZERO-FILLED gap, page-aligned start, arbitrary end;
# NEITHER probe found an actual sparse hole, so "sparse" is unproven — do
# not re-assert it, only that the gap reads as zeros with correct content
# on both sides and no error from either writer.
#
# MATCH: 'pg_regress' catches the parent binary while it's still alive;
# 'HIDE_TABLEAM' catches its regress-runner psql children even after
# pg_regress itself has died and the child was reparented — empirically
# confirmed live (2026-07-12): a running pg_regress's psql child carries
# exactly `-v HIDE_TABLEAM=on -v HIDE_TOAST_COMPRESSION=on` and nothing
# else identifying it as a regress child (pg_regress redirects the child's
# stdin/stdout via fork+dup2, not a shell string, so the child's own argv
# never mentions the outputdir — matching on the path alone would miss an
# orphaned child once its parent is gone). HIDE_TABLEAM/HIDE_TOAST_COMPRESSION
# are psql -v vars only pg_regress injects; no real usage collides with them.
#
# NO AUTO-KILL (no-standing-self-heal rule): refuse loudly, name the exact
# pids + kill command, and let the operator decide — recurrence must fail
# loudly with the fix named, never be quietly repaired out from under them.
# ── STATBUS-282: THE POSTMASTER IS THE AUTHORITY ────────────────────────────
#
# WHAT MUST SURVIVE A FAILURE CANNOT LIVE INSIDE THE THING THAT FAILS. The
# failure here IS the death of the host process tree, so no host-side fact can
# be the remedy — which is why the flock cannot be the authority. It frees
# because the KERNEL closes the fd, not because our code ran, and the
# container-side writer never held that fd at all.
#
# The writer is a Postgres client, and the postmaster already knows exactly and
# continuously which clients are alive. That fact:
# - lives in the WRITER's failure domain, so host death cannot make it lie;
# - is released automatically on client death — no cleanup code to skip;
# - needs no signalling, so STATBUS-188 is satisfied by construction;
# - observes a CONNECTION, not a process, so a dead pg_regress whose psql
# child still holds the database is still visible.
#
# THE PROPERTY THAT MAKES THIS STRICTLY BETTER THAN THE CONTAINER-SIDE pgrep:
# the observation channel IS the work channel. If we cannot reach the database
# to ask, we cannot reach it to run the tests either — so "blind but proceeding"
# stops being a reachable state, rather than being a state we must remember to
# guard against.
#
# The flock stays as a cheap host-side mutex. It may only ever produce a false
# BUSY (harmless: someone waits), never a false FREE.
check_no_live_test_backends() {
# ── DB-NOT-RUNNING IS A CLEAR, NOT AN OBSERVATION FAILURE (STATBUS-282) ──
# A postmaster that does not exist has no backends. That is not a case we
# failed to observe, it is a case that cannot occur: there is nothing for a
# test database to have a live connection TO. The fail-closed refusal below
# is for "a RUNNING db we cannot ask" — a genuinely unknown state — and
# applying it to "no db at all" refuses a run that is provably safe.
#
# This gap was found by CI, not by review: the runner brings the db up AFTER
# this check, so the authority refused every pg_regress run on the runner
# with "service db is not running". The sibling straggler guard has carried
# this pre-gate since it was written; this function was missing it.
#
# WHY THE DOCKER EXIT STATUS IS CHECKED SEPARATELY rather than the shorter
# `... | grep -qx db || return 0`: that one-liner returns CLEAR when the
# docker command ITSELF fails, which is the same failure-to-observe-treated-
# as-absence defect this authority exists to prevent — a transiently
# unavailable docker CLI (Docker Desktop's documented unresponsiveness on
# macOS) would silently authorise a run beside a live one. Here, only
# "docker answered, and db is not among the running services" is a clear;
# if docker could not answer we fall through to the psql probe, which
# refuses loudly on a db it cannot reach.
local _psvc _psrc=0
_psvc=$(docker compose ps --status running --format '{{.Service}}' 2>&1) || _psrc=$?
if [ "$_psrc" -eq 0 ] && ! printf '%s\n' "$_psvc" | grep -qx db; then
return 0
fi
# Errexit-safe capture (STATBUS-261). Under this script's `set -e` a bare
# `_out=$(cmd); _rc=$?` DIES on the assignment for ANY non-zero exit — the
# landmine that killed CI at 4fdea9a2b when it sat in this very file.
local _out _rc=0
_out=$(./sb psql -d postgres -t -A -c \
"SELECT datname || ' pid=' || pid || ' app=' || coalesce(application_name,'?') \
FROM pg_stat_activity \
WHERE datname IS NOT NULL AND starts_with(datname, 'test_shared_') \
ORDER BY datname, pid" 2>&1) || _rc=$?
# A FAILURE TO OBSERVE IS NOT EVIDENCE OF ABSENCE — the same defect the
# straggler guard carried. But here it is self-limiting rather than merely
# guarded against: this query runs over the very connection the tests need,
# so if it cannot run, neither can they. We still refuse explicitly, because
# a clear message beats a confusing failure three steps later.
if [ "$_rc" -ne 0 ]; then
echo "REFUSING: could not ask the postmaster which test databases are in use (exit $_rc)." >&2
echo " $_out" >&2
echo " This is the authority for 'may I start a run' — without an answer there is no run." >&2
exit 1
fi
# starts_with(), not LIKE 'test\_shared\_%': in LIKE an unescaped
# underscore is a single-character WILDCARD, so the obvious pattern also
# matches names this prefix never produced. starts_with has no metacharacters
# to get wrong.
[ -n "$(printf '%s' "$_out" | tr -d '[:space:]')" ] || return 0
cat >&2 <<EOF
═══════════════════════════════════════════════════════════════════════
BLOCKED: a test database still has live backends.
The postmaster reports these connections to test_shared_* databases:
$_out
That is a test run still in flight — possibly one whose HOST process is
already gone (a killed harness, a closed ssh session, an expired timeout),
which is exactly the case a host-side lock cannot see: the flock frees when
the kernel closes the fd, while the container-side client keeps writing.
WHAT TO DO — and NOT to kill it (STATBUS-188): a pg_regress reparented to
the postmaster dies badly if signalled. Wait for it; it finishes on its own.
Watch it drain:
./dev.sh psql -d postgres -c "SELECT datname, pid, state, query \
FROM pg_stat_activity WHERE starts_with(datname, 'test_shared_')"
Then re-run this command. Orphaned databases with NO backends are dropped
automatically — they need no action from you.
═══════════════════════════════════════════════════════════════════════
EOF
exit 1
}
# ── STATBUS-282: orphan cleanup that survives host death ────────────────────
#
# The same fact answers the leak. Cleanup used to live in the host-side wrapper,
# so a killed host left clones behind — six of them on 2026-08-27, each a full
# copy of the migrated template.
#
# A test_shared_* database with ZERO backends is an orphan by the same authority
# that governs the lock, and DROP DATABASE is the postmaster's own teardown, not
# a signal. One WITH backends is a live run and is never touched.
#
# DROP ... (FORCE) IS BANNED HERE, and the ban is STRUCTURAL rather than
# stylistic. It terminates backends — signalling by another name, forbidden by
# STATBUS-188 — but that is only the first reason. The second is stronger:
#
# THE PROTECTION HERE IS TWO-LAYERED, AND THE LOWER LAYER IS NOT OURS.
# 1. the NOT EXISTS clause below never ASKS to drop a database in use;
# 2. PostgreSQL itself REFUSES even when asked:
# ERROR: database "..." is being accessed by other users
# DETAIL: There is 1 other session using the database.
# (measured 2026-08-28 against the real cluster, not assumed.)
#
# FORCE is precisely the switch that DISABLES layer 2. Plain DROP inherits the
# postmaster's own guarantee; FORCE opts out of it and kills the sessions
# instead. So the ban is what keeps the guarantee in the postmaster's hands
# rather than in our discipline — which is this whole ticket's thesis.
#
# COROLLARY FOR ANYONE TEMPTED TO SIMPLIFY: the NOT EXISTS clause is defence in
# depth, not the floor. Removing it would not immediately break anything (layer
# 2 still holds) — which is exactly what makes removing it attractive and
# wrong: it would leave the sweep ASKING to drop live databases and relying
# entirely on the refusal, turning a clean no-op into a stream of errors, and
# leaving nothing at all if a future FORCE ever crept in.
#
# NEVER DERIVE OWNERSHIP FROM THE NAME. The clone is named test_shared_<host pid>
# for humans, but host pids are RECYCLED: after host death that name identifies
# nobody, and a later run can compute a name already held by a straggler. The
# prefix is used only as set membership — which databases are candidates — never
# to decide whose a database is. That question has exactly one answer, and it
# comes from pg_stat_activity.
drop_orphan_test_databases() {
# Same pre-gate as the authority, and now REQUIRED rather than merely tidy.
# This used to lean on the authority above having "established reachability"
# before it ran — but with that function returning early on a stopped db,
# that premise no longer holds and this would be the first thing to touch a
# db that isn't there. Nothing to sweep when there is no postmaster: skip
# silently, since a stopped db is a normal state here, not a problem to
# report.
local _psvc _psrc=0
_psvc=$(docker compose ps --status running --format '{{.Service}}' 2>&1) || _psrc=$?
if [ "$_psrc" -eq 0 ] && ! printf '%s\n' "$_psvc" | grep -qx db; then
return 0
fi
local _out _rc=0
_out=$(./sb psql -d postgres -t -A -c \
"SELECT d.datname FROM pg_database AS d \
WHERE starts_with(d.datname, 'test_shared_') \
AND NOT EXISTS (SELECT 1 FROM pg_stat_activity AS a WHERE a.datname = d.datname) \
ORDER BY d.datname" 2>&1) || _rc=$?
if [ "$_rc" -ne 0 ]; then
# Non-fatal: failing to reclaim disk must not block a run. The authority
# above already refused if the database was unreachable.
echo "note: could not list orphaned test databases (exit $_rc) — skipping cleanup" >&2
return 0
fi
local _db
for _db in $_out; do
[ -n "$_db" ] || continue
# Plain DROP. If a backend attached between the query and here, the drop
# FAILS — and that failure is correct: it means the database stopped
# being an orphan, and losing the race in this direction is harmless.
if ./sb psql -d postgres -v ON_ERROR_STOP=1 -c "DROP DATABASE \"$_db\";" >/dev/null 2>&1; then
echo "Dropped orphaned test database: $_db"
else
echo "note: left $_db in place (it acquired a backend, or the drop was refused)" >&2
fi
done
}
check_no_straggler_pg_regress() {
# If the db service isn't even running there is nothing to race with.
#
# SAME FORM AS THE AUTHORITY (STATBUS-282). This was
# `... | grep -qx db || return 0`, which returns CLEAR when the docker
# command ITSELF fails — the exact failure-to-observe-treated-as-absence
# defect the case statement below was written to remove, sitting three lines
# above it in the same function. A transiently unresponsive Docker Desktop
# would silently authorise a run beside a live straggler, and the careful
# 0/1/* branching underneath would never be reached to say otherwise.
#
# Only "docker answered, and db is not among the running services" is a
# clear. If docker could not answer we fall through to the pgrep probe,
# which refuses loudly on a container it cannot observe.
local _psvc _psrc=0
_psvc=$(docker compose ps --status running --format '{{.Service}}' 2>&1) || _psrc=$?
if [ "$_psrc" -eq 0 ] && ! printf '%s\n' "$_psvc" | grep -qx db; then
return 0
fi
# STATBUS-282 MUST-FIX: a failure to OBSERVE is not evidence of absence.
# pgrep exit 1 means "looked, found nothing" — genuinely clear. Any OTHER
# non-zero exit (docker compose exec rejected, daemon busy, container
# restarting) means the guard never looked at all. The old form —
# `2>/dev/null) || return 0` — collapsed both into the same silent
# return-clear, discarding stderr, the one thing that would have told
# them apart. On macOS Docker Desktop's documented unresponsiveness
# (CLAUDE.md has its own restart procedure for it) that reads as "no
# straggler" when the true answer is "could not check."
# Capture must be errexit-safe (STATBUS-261's idiom): under this
# script's `set -e`, a bare `_out=$(cmd); _rc=$?` DIES on the
# assignment when cmd exits non-zero — pgrep's exit 1 (the NORMAL
# clean-container case) would silently abort the whole run before
# _rc is ever read. Proven in CI at 4fdea9a2b: fast tests exited 1
# with no output the moment this guard first ran on a clean runner.
local _out _rc=0
_out=$(docker compose exec -T db pgrep -af 'pg_regress|HIDE_TABLEAM' 2>&1) || _rc=$?
case $_rc in
0) ;; # straggler found -> BLOCK below
1) return 0 ;; # pgrep looked, found nothing -> genuinely clear
*)
echo "REFUSING: could not observe the db container (exit $_rc): $_out" >&2
exit 1
;;
esac
local _straggler="$_out"
[ -n "$_straggler" ] || return 0
local _pids
_pids=$(printf '%s\n' "$_straggler" | awk '{print $1}' | tr '\n' ' ')
cat >&2 <<EOF
═══════════════════════════════════════════════════════════════════════
BLOCKED: a straggler pg_regress is still running in the db container.
This command is about to start pg_regress against the shared outputdir
(test/results/). A PREVIOUS harness run's pg_regress (or its psql child)
is still alive in the db container even though the host-side test-run
lock is free — the lock lives on a host file descriptor and does not
reach into the container, so a killed run can leave pg_regress orphaned
there, still writing into the same .out files this run is about to
reuse. Two writers on one .out file corrupt it silently, with no error
from either side (STATBUS-158).
Straggler process(es) found in the db container:
$_straggler
WHAT TO DO (STATBUS-188 evidence-based rule — observed mechanism, not caution):
1. Report this and WAIT. Waiting is not the safe option — it is the ONLY
safe option. An orphaned pg_regress/psql client self-clears within
minutes every time this was directly observed (2026-07-29, 2026-07-31),
and a self-clearing orphan exits cleanly — no recovery follows.
Re-check with:
docker compose exec db pgrep -af 'pg_regress|HIDE_TABLEAM'
2. NEVER signal these processes — not SIGTERM, not -9, nothing. Postgres
runs as PID 1 in this container, so an ORPHANED client is REPARENTED
to the postmaster, and postgres treats ANY adopted child that dies by
signal as a backend crash: it kills every live session and runs full
crash recovery. OBSERVED 2026-07-31 13:43 ('untracked child process
(PID 33798) was terminated by signal 15' -> 'terminating any other
active server processes' -> recovery) — and this same mechanism is the
resolved cause of the 2026-07-14 crash-recovery cycles (STATBUS-188).
3. If an orphan genuinely persists past ~30 minutes, escalate to a human —
do not signal it yourself.
Hook source: dev.sh check_no_straggler_pg_regress
═══════════════════════════════════════════════════════════════════════
EOF
exit 1
}
# tracked_files_status_snapshot — STATBUS-278 Part 2. Prints `git status
# --porcelain` restricted to TRACKED files (--untracked-files=no): a
# developer's own new, untracked scratch files are out of scope by design —
# only files git already knows about can "gain a modification".
#
# EXEMPTION (STATBUS-278 amendment), named narrowly: test/expected/performance/
# and test/expected/explain/ — those two subdirectories BY NAME, never
# test/expected/ itself. test/expected/ otherwise holds the strict-test
# oracles this guard MUST keep policing: under a plain `./dev.sh test` they
# must never change; only an explicit `--update-expected` legitimately
# touches them. The two exempted subdirectories are different in kind —
# baseline snapshots the suite REWRITES BY DESIGN on every run (test/sql/109,
# 303, 400, 401 write `\o :perf_file` / `\o test/expected/explain/*.txt`;
# see .claude/rules/testing.md: "Performance and explain baselines are not
# strict tests... review the diff... discard trivial drift"). A guard that
# fires on the expected, by-design case is not a guard. The gap this opens
# (a baseline could still be NUL-corrupted, same as database.types.ts was)
# is closed instead by extending check_results_for_nul_corruption's surface
# to these same two subdirectories, below — that check polices WHAT got
# written; this one polices WHETHER a write should exist at all. Each
# mechanism does what it is good at.
tracked_files_status_snapshot() {
git -C "$WORKSPACE" status --porcelain --untracked-files=no -- \
':!test/expected/performance/' ':!test/expected/explain/'
}
# check_no_new_tracked_file_modifications <before-snapshot-file> <label> —
# STATBUS-278 Part 2: the mechanism-INDEPENDENT guard the architect ruled
# for (comment #1 on the ticket) instead of a widened NUL tripwire or a
# `\o`-only grep. A grep for `\o` would catch one write mechanism while
# `\copy ... TO`, `COPY ... TO`, and `\!` remain fully able to write a
# tracked file — a partial check that would read as total. Observing
# `git status --porcelain` before and after the suite catches ANY of them,
# and anything nobody has thought of yet, without enumerating write syntaxes.
#
# Compares the BEFORE/AFTER porcelain snapshots as sets of lines (a status
# line encodes both the file path and its exact status) — a line present
# AFTER but not BEFORE means that file's tracked status changed during this
# run: it went from clean to dirty, or its dirty status itself changed
# (rename, deletion, or a new kind of modification). A file that was
# ALREADY dirty before the run and stays dirty with the SAME status line
# does not trip this — this is a before/after DELTA, not a demand for a
# clean tree, so a developer's own uncommitted work is never the trigger.
# Untracked files never appear in either snapshot (see
# tracked_files_status_snapshot above), so they are out of scope entirely.
check_no_new_tracked_file_modifications() {
local _before_file="$1" _label="$2"
local _after _new
_after=$(tracked_files_status_snapshot)
_new=$(comm -13 <(sort "$_before_file") <(printf '%s\n' "$_after" | sort))
[ -n "$_new" ] || return 0
cat >&2 <<EOF
═══════════════════════════════════════════════════════════════════════
TRACKED FILE MODIFIED DURING TEST RUN: $_label
A tracked file's \`git status --porcelain\` line changed during this run —
present AFTER the suite, absent BEFORE it started:
$_new
A test observes and asserts; it must never write a tracked file (STATBUS-278
— the category correction applied to test/sql/016, 008, and 015, each of
which used to overwrite a tracked doc/type file directly). Whatever
mechanism wrote the file above (\\o, \\copy ... TO, COPY ... TO, \\! or
anything else), find it and remove the write — redirect to a results-side
artifact (test/results/, gitignored) and compare against the tracked file
with a loud failure on mismatch, the same shape 016/008/015 now use.
Hook source: dev.sh check_no_new_tracked_file_modifications
═══════════════════════════════════════════════════════════════════════
EOF
return 1
}
# check_results_for_nul_corruption — the tripwire half of STATBUS-158. An
# embedded NUL in a pg_regress .out file is never legitimate output — psql
# and postgres never emit one; it is the fingerprint of two writers racing
# the same file (see check_no_straggler_pg_regress above) or any other
# write-layer fault. Preserve the corrupted bytes BEFORE anything reruns
# and overwrites them (the original STATBUS-158 incident lost its own byte
# schedule to exactly that overwrite), then fail with a verdict distinct
# from an ordinary test diff, naming the straggler check as the first thing
# to run. Args: <PG_REGRESS_DIR> <test_basename>...
#
# SURFACE EXTENSION (STATBUS-278 amendment): also scans each test's baseline
# artifacts under test/expected/performance/ and test/expected/explain/ —
# the exact two subdirectories the tree-diff guard (tracked_files_status_
# snapshot, above) exempts because the suite rewrites them BY DESIGN. That
# exemption means the guard no longer sees a write there at all — so a
# baseline can still be NUL-corrupted (the same 507,904-page-aligned-NUL
# 286 signature database.types.ts hit) with nothing else positioned to
# notice. This check polices WHAT got written; the guard polices WHETHER a
# write should exist. Neither mechanism alone covers both questions.
check_results_for_nul_corruption() {
local _dir="$1"; shift
local _test _file _label _full _stripped _preserved _corrupted="" _candidates _glob
for _test in "$@"; do
_candidates="$_dir/results/$_test.out"
for _glob in "$_dir/expected/performance/$_test"*.perf "$_dir/expected/explain/$_test"*.txt; do
[ -f "$_glob" ] && _candidates="$_candidates
$_glob"
done
while IFS= read -r _file; do
[ -n "$_file" ] || continue
[ -f "$_file" ] || continue
_label=$(printf '%s' "${_file#"$_dir"/}" | tr '/' '_')
_full=$(wc -c < "$_file" | tr -d ' ')
_stripped=$(LC_ALL=C tr -d '\000' < "$_file" | wc -c | tr -d ' ')
if [ "$_full" != "$_stripped" ]; then
mkdir -p "$WORKSPACE/tmp"
_preserved="$WORKSPACE/tmp/corrupted-$_label-$(date '+%Y%m%d%H%M%S' 2>/dev/null || echo unknown).out"
# ── STATBUS-286 deliverable #2: CAPTURE THE STATE AT FIRE TIME ──