W5 · Phase 8B profiling half: SEARCHES_BIJECTOR arm, bijector_ab driver with pre-registered F1–F5, A100 array (#162)

hpc/batch_gpu/submit_phase8b_bijector_a100

@@ -0,0 +1,128 @@
+#!/bin/bash -l
+#
+# Inference programme W5 / Phase 8B (issue #162) — bijector (log-coordinate)
+# A/B on the free-regularization NaN wall / conditioning. Pre-registration,
+# arm table and falsification criteria (F1-F5) are in
+# `scripts/misc/searches/bijector_ab.py`'s module docstring; read that before
+# touching this file. Cloned from
+# `submit_search_multi_start_prodigy_autoconv_imaging_mge_a100_hst_fp64_n16_diagnostic_theta_e`'s
+# shape (gres gpu:1, JAX_ENABLE_X64, prior_box, scaler none).
+#
+# ARMS — one SLURM array task per (cell, log_det_method, bijector, seed),
+# generated to match `bijector_ab.build_arms()` EXACTLY (see that function's
+# docstring for the "why" of each count):
+#
+#   delaunay_adapt_split : {cholesky, slogdet} x {none, log_reg} x seed 0-4 = 20
+#   knn                  : {none, log_reg} x seed 0-4 = 10, + logit x seed 0-4 = 5  -> 15
+#   mge                  : {none, log_reg} x seed {0, 1}  (F4 control)         = 4
+#   total                                                                       = 39
+#
+# Every task: multi_start_prodigy (fixed-step: check_for_convergence=False),
+# n_starts=16, n_steps=3000 (the #117-validated pixelized-cell budget),
+# batch_size=4 (mandatory on pixelized cells), clipper=prior_box, scaler=none,
+# SEARCHES_LANE_HISTORY=1. SEARCHES_TRACE_PARAMS is set on the two
+# regularization-coefficient paths for delaunay_adapt_split/knn (mge has
+# none). SEARCHES_LOG_DET_METHOD is only exported for delaunay_adapt_split
+# (LOG_DETS[i]="auto" -> unset, uses the W8-resolved default).
+#
+# ESTIMATED WALL — 16 starts x 3000 steps at the #117-validated pixelized
+# throughput is ~5 min including compile per task (matches the diagnostic_
+# theta_e submit's n16/3000-step citation); --time below gives it 6x headroom.
+#
+# RESULTS — one JSON+PNG per task under
+#   results/searches/multi_start_prodigy/imaging/<cell>/hst/phase8b/
+# via --output-dir, config-name carrying log_det+bijector+seed so no two
+# tasks' JSONs collide. Score with:
+#   python3 scripts/misc/searches/bijector_ab.py --score
+#
+# DO NOT SUBMIT without first confirming the RAL PyAutoFit checkout is on a
+# commit >= PR#1525 (`bijector.py`, `MultiStartGradient(bijector=...,
+# record_lane_nan_history=..., trace_param_indices=...)`) — run `HPCPullPyAuto`
+# and check first; an older PyAutoFit raises on the `bijector=` kwarg.
+
+#SBATCH -J phase8b_bijector_ab
+#SBATCH --partition=gpu
+#SBATCH --gres=gpu:1
+#SBATCH --ntasks=1
+#SBATCH --cpus-per-task=4
+#SBATCH --mem=64gb
+#SBATCH --time=0:30:00
+#SBATCH --array=0-38
+#SBATCH -o output/output.%A_%a.out
+#SBATCH -e error/error.%A_%a.err
+#SBATCH --mail-type=END,FAIL
+#SBATCH --mail-user=james.w.nightingale@durham.ac.uk
+
+export AP_ROOT=/mnt/ral/jnightin/autolens_profiling
+source $AP_ROOT/activate.sh
+
+export JAX_PLATFORM_NAME=cuda
+export JAX_PLATFORMS=cuda,cpu
+export XLA_PYTHON_CLIENT_PREALLOCATE=false
+export JAX_ENABLE_X64=True
+export NUMBA_CACHE_DIR=/tmp/numba_cache
+export MPLCONFIGDIR=/tmp/matplotlib
+
+# Generated from `bijector_ab.build_arms()` — index i is task i's
+# (cell, log_det_method, bijector, seed). "auto" in LOG_DETS means
+# SEARCHES_LOG_DET_METHOD is left unset (W8-resolved default).
+CELLS=(delaunay_adapt_split delaunay_adapt_split delaunay_adapt_split delaunay_adapt_split delaunay_adapt_split delaunay_adapt_split delaunay_adapt_split delaunay_adapt_split delaunay_adapt_split delaunay_adapt_split delaunay_adapt_split delaunay_adapt_split delaunay_adapt_split delaunay_adapt_split delaunay_adapt_split delaunay_adapt_split delaunay_adapt_split delaunay_adapt_split delaunay_adapt_split delaunay_adapt_split knn knn knn knn knn knn knn knn knn knn knn knn knn knn knn mge mge mge mge)
+LOG_DETS=(cholesky cholesky cholesky cholesky cholesky cholesky cholesky cholesky cholesky cholesky slogdet slogdet slogdet slogdet slogdet slogdet slogdet slogdet slogdet slogdet auto auto auto auto auto auto auto auto auto auto auto auto auto auto auto auto auto auto auto)
+BIJECTORS=(none none none none none log_reg log_reg log_reg log_reg log_reg none none none none none log_reg log_reg log_reg log_reg log_reg none none none none none log_reg log_reg log_reg log_reg log_reg logit logit logit logit logit none none log_reg log_reg)
+SEEDS=(0 1 2 3 4 0 1 2 3 4 0 1 2 3 4 0 1 2 3 4 0 1 2 3 4 0 1 2 3 4 0 1 2 3 4 0 1 0 1)
+
+I=$SLURM_ARRAY_TASK_ID
+CELL=${CELLS[$I]}
+LOG_DET=${LOG_DETS[$I]}
+BIJECTOR=${BIJECTORS[$I]}
+SEED=${SEEDS[$I]}
+
+N_STARTS=16
+N_STEPS=3000
+
+export SEARCHES_N_STARTS=$N_STARTS
+export SEARCHES_N_STEPS=$N_STEPS
+export SEARCHES_BATCH_SIZE=4
+export SEARCHES_SEED=$SEED
+export SEARCHES_CLIPPER=prior_box
+export SEARCHES_SCALER=none
+export SEARCHES_BIJECTOR=$BIJECTOR
+export SEARCHES_LANE_HISTORY=1
+export SEARCHES_DISABLE_VIZ=1
+
+if [ "$LOG_DET" != "auto" ]; then
+    export SEARCHES_LOG_DET_METHOD=$LOG_DET
+else
+    unset SEARCHES_LOG_DET_METHOD
+fi
+
+if [ "$CELL" == "mge" ]; then
+    unset SEARCHES_TRACE_PARAMS
+else
+    export SEARCHES_TRACE_PARAMS="galaxies.source.pixelization.regularization.inner_coefficient,galaxies.source.pixelization.regularization.outer_coefficient"
+fi
+
+CONFIG_NAME="${CELL}_${LOG_DET}_${BIJECTOR}_seed${SEED}"
+OUT_DIR="$AP_ROOT/results/searches/multi_start_prodigy/imaging/${CELL}/hst/phase8b"
+
+nvidia-smi
+
+echo "=========================================="
+date
+echo "Task:        $I / 38"
+echo "Cell:        searches/multi_start_prodigy/imaging/$CELL"
+echo "Instrument:  hst"
+echo "Precision:   fp64"
+echo "log_det:     $LOG_DET"
+echo "bijector:    $BIJECTOR"
+echo "seed:        $SEED"
+echo "n_starts=$N_STARTS  n_steps=$N_STEPS  batch_size=4  clipper=prior_box  positions=off"
+
+cd $AP_ROOT
+python3 scripts/imaging/searches/multi_start_prodigy/${CELL}.py \
+    --instrument hst \
+    --config-name $CONFIG_NAME \
+    --output-dir $OUT_DIR
+
+echo "Finished."
+date

results/notes/inference/PROGRAMME.md

@@ -33,7 +33,9 @@ expectation · **[CONTRADICTED]** existing evidence pushed back on the brief ·
 | Phase 2 NSS mainline re-tune | **scan complete 2026-08-24 (CP-2 ✅, CP-5 ✅)** — H2.1 closed: logZ bias was inner-kernel under-mixing (inner=30: 5/5 seeds +1.0 ± 0.4 nats vs Nautilus); operating point n200/nd100/inner30 costs 5.0× the Nautilus sampler wall on MGE, 18.4× on Delaunay (same-night re-baselines: Nautilus mge 707 s, delaunay 1,891 s). Sample economy: Kish ESS 1,315 vs Nautilus 4,121 per run; ~940 vs ~15 evals/ESS. **GATE A CALLED 2026-08-24 — Nautilus stays baseline; af.NSS = tuned alternative** | `phase_02_nss_mainline/RESULTS.md` + DECISIONS.md 2026-08-24 |
 | Phase 3 Prodigy reliability | **wave 1 complete + adversarially reviewed (CP-3 ✅ 2026-08-23, positions-off)** — p̂_hit = 0.048 [0.037, 0.061] (lower bound; n256 tier, ~1,280 distinct draws — tiers share lane-index draws, n-dependence unmeasured); demonstrated ≥99% reliability at **n=256 only** (joint-95% worst case), 3.4–4.5× under the viz-stripped Nautilus wall; zero parameter-recovery impostors; ~half of lanes end pinned (H3.3, measured not closed); θ_E diagnostic uninformative for H3.1 (withdrawn). **Gate B pt 1 CALLED (human-ratified 2026-08-23** — DECISIONS.md): Prodigy(n=256, prior_box, autoconv, positions-off) ratified as global MAP searcher on MGE; pt 2 (PositionsLH) open. Fresh-seed tier 2026-08-24: n128 5/5, n256 15/15 cumulative (Wilson-95 lower 0.80); Nautilus re-baseline 707 s → 3.1–4.3× | `phase_03_prodigy_reliability/RESULTS.md` + `ADVERSARIAL_REVIEW.md` |
 | Phase 8A slogdet A/B (CP-4) | **complete 2026-08-24 — FAIL as pre-registered.** knn stressor never walls (VOID both tiers); on Delaunay+AdaptSplit slogdet rescues 64–73 % of NaNs, 0 regressions, but 20–32 NaN-under-both, λ-transect grads non-finite, marginal-band Δ up to 9,619 nats (A100), CPU 3.7×. Human call: adopt as this repo's GPU gradient-cell default (W8), library stays opt-in, residual NaNs → W7, 8B in parallel — re-scored 2026-08-24 (W7): per-draw attribution (170 draws classified) finds two harvest/driver bugs (negative-coefficient + dead-lane descent rows; zero-anchor λ-transect gradient) now fixed; CP-4 re-scored on the clean subset excluding both — **verdict unchanged, FAIL on both tiers** (residual failures are the genuinely-singular λ⁴ population, 53–80% of sampled `nan_both` draws); tier-dependence traced to `cond ~ 1e15-1e19` where the reconstruction solve NaNs despite finite log-det terms. W8 adoption stands; W9 unblocked | `phase_08_regularization/RESULTS.md` + DECISIONS.md 2026-08-24 |
-| Phases 1, 4–7, 8B–13 | not started | — |
+| Phase 8B log-coordinate bijector A/B (W5) | **pre-registered, A100 submit prepared 2026-08-24** — PyAutoFit half merged (PR#1525: `bijector.py`, `MultiStartGradient(bijector=...)`, opt-in lane-NaN-history/trace-param diagnostics); driver `scripts/misc/searches/bijector_ab.py` (39-arm table, F1–F5 falsification scorer); submit `hpc/batch_gpu/submit_phase8b_bijector_a100` (39-task array, not yet dispatched) | `phase_08_regularization/RESULTS.md` "8B" |
+| Phase 1 targets registry / schema v2 | **shipped 2026-08-24 (W4, PR#170)** — `_targets.py` + `target_id`, schema v2 (Kish ESS, reject-inclusive evals), `delaunay_nn` / `slam_source_pix(_nn)`; InferenceRefs_v1 mge+delaunay retro-certified, 11 reference rows queued (RAL 340210) | DECISIONS.md 2026-08-24 W4 |
+| Phases 4–7, 9–13 | not started (Phase 4 Stage 1 shipped PR#168; Stage 2 queued RAL 340114/5) | — |
 | Gates A–F | **A CALLED 2026-08-24** (Nautilus stays nested baseline; af.NSS tuned alternative); **B part 1 CALLED 2026-08-23** (Prodigy n=256 global MAP, MGE, positions-off; caveat (a) n=256-only stands 2026-08-24); B pt 2, C–F open — C criterion reworded 2026-08-24 (batched-pipeline value, not single-fit ESS/s vs Nautilus) | DECISIONS.md |
 
 ---
@@ -967,7 +969,7 @@ ship as autolens_profiling issues/PRs, not Mind prompts).
 | W7 | CP-4 follow-up: NaN-under-both draws, transect gradients, marginal-band tier dependence | #164 | laptop / A100 replay | — |
 | W4 | Phase 1 targets registry, schema v2 (ESS + reject-inclusive evals), `slam_source_pix`, reference baselines — **IN PROGRESS 2026-08-24: registry/schema/tests shipped; 2/13 targets have a certified baseline (both retro-adopted), 11 SUBMIT_LIST rows still queued, array script not submitted** | #161 | CPU + one A100 bake | rides behind W1/W6 |
 | W2 | Phase 4 Stage 2 — Nautilus / Prodigy ± positions × 5 seeds on MGE (NSS arm dropped per Gate A) | #160 | A100 ~5 GPU-h | W1 (+ W6 for n_batch) |
-| W5 | Phase 8B log-coordinate stepping (Scaler → bijector) | #162 | source + laptop | — (parallel smoothing arc) |
+| W5 | Phase 8B log-coordinate stepping (Scaler → bijector) | #162 | source + laptop; A100 submit prepared 2026-08-24, not yet dispatched | — (parallel smoothing arc) |
 | W9 | REMINDER: make slogdet standard in PyAutoArray | #166 | source | W7 |
 
 Then: Phase 5 mesh campaign (Nautilus + Prodigy only) + Phase 6 warm-start

results/notes/inference/phase_08_regularization/RESULTS.md

@@ -285,3 +285,81 @@ tiers. The residual failures are the genuinely-singular λ⁴ population
 (established below: `prior`-source draws with a regularization coefficient
 in ~4e5-9e5) — exactly the systems CP-4 set out to measure, not artefacts of
 how the replay set was built.
+
+- `coefficient_min = −5.2` in the A100 marginal band is the driver
+  reporting the log-coefficient axis for transect draws; not a negative
+  regularization.
+
+## 8B — pre-registration (2026-08-24)
+
+W5 (issue #162). PyAutoFit half merged to main (PR#1525 — `bijector.py`,
+`MultiStartGradient(bijector=...)`, opt-in `record_lane_nan_history` /
+`trace_param_indices`). Driver: `scripts/misc/searches/bijector_ab.py`
+(full pre-registration, arm table and readouts in its module docstring).
+Submit prepared, **not yet run**: `hpc/batch_gpu/submit_phase8b_bijector_a100`
+(A100, 39-task array, `--time=0:30:00` each). Submit id: **TBD** (fill in
+once dispatched).
+
+**Question.** Does stepping in `log(lambda)` for the free AdaptSplit
+regularization coefficients (`log_reg`: `af.BijectorPerPath` restricted to
+`"regularization."` paths backed by a `LogUniformPrior`) move the NaN-wall
+position, speed up free-regularization convergence, or reduce time spent at
+high coefficients — without moving the physical objective (a category-1
+reparameterization; see `autofit.non_linear.bijector`'s equivalence
+argument)?
+
+**Arms (39 tasks)**:
+
+| cell | log_det_method | bijector | seeds | n |
+|---|---|---|---|---:|
+| `delaunay_adapt_split` (the NaN wall — Phase 8A/CP-4) | cholesky, slogdet | none, log_reg | 0-4 | 20 |
+| `knn` (finite over-regularized floor; the cell 8B's text names) | auto (W8-resolved) | none, log_reg | 0-4 | 10 |
+| `knn` (secondary arm) | auto | logit | 0-4 | 5 |
+| `mge` (F4 control — no regularization coefficients at all) | auto | none, log_reg | 0-1 | 4 |
+
+Every arm: `multi_start_prodigy` (fixed-step), `n_starts=16`, `n_steps=3000`
+(#117-validated pixelized budget), `batch_size=4`, `clipper=prior_box`,
+`scaler=none`, `record_lane_nan_history=True`; `knn`/`delaunay_adapt_split`
+also trace the two regularization coefficients
+(`SEARCHES_TRACE_PARAMS`).
+
+**Pre-registered falsification** (any two of F1-F4 -> 8B falsified; F5 halts
+and is scored first — a trip means the bijector changed the physical
+objective, a bug, not a science finding):
+
+- **F1** — median first value-NaN step under `log_reg` not earlier than
+  `none` on `delaunay_adapt_split`, OR value-NaN lane-steps do not fall
+  below 50% of `none`'s.
+- **F2** — steps-to-within-10-nats of a reference log-posterior not reduced
+  >= 2x at matched seeds on `knn`. Framed against the historical
+  free-vs-fixed-regularization convergence figures cited in
+  `PROGRAMME.md:579` (**2,200 steps free vs 98 steps fixed**) — but see the
+  driver's module docstring: this campaign has no dedicated fixed-reg
+  control arm, so the actual reference used is the max `none`-arm
+  log-posterior per (cell, log_det_method) group, and the artifact records
+  the resolved value rather than replaying the bare 2,200/98 figures
+  literally. **This is a documented deviation from a literal reading of the
+  pre-registration**, flagged here for a human to confirm or override before
+  the verdict is treated as final.
+- **F3** — `log_reg` lanes spend >= the same fraction of steps at
+  lambda > 1e4 as `none`, on either wall cell.
+- **F4** — the `mge` control differs by any bit between `none`/`log_reg`
+  (it must not — MGE carries zero `LogUniformPrior` "regularization." paths,
+  so `log_reg` resolves to an empty `BijectorPerPath` map), OR the `knn`
+  `logit` arm reproduces a pinned-lane-to-infinity pathology.
+- **F5** — figure of merit at the shared initial broad-start draw (step-0
+  global-best fom) differs by more than 1e-9 relative between arms at a
+  matched seed.
+
+**Not yet run.** This section will be replaced with the verdict table once
+`--stage run` completes on RAL and `--stage score` (or `--score`) produces
+`results/notes/inference/phase_08_regularization/bijector_ab/verdict_<hardware>.json`.
+
+**Known gap, recorded rather than hidden**: the driver cannot recover
+Prodigy's internal step-scale estimate ("final `d`",
+`optax.contrib.ProdigyState.estim_lr`) from the standard results JSON
+pipeline — `search_internal["opt_state"]` is not serialized by
+`_per_lane.per_lane_block` (an unbounded, non-JSON-safe JAX pytree), and each
+arm runs in its own subprocess (matching the real SLURM array), so there is
+no in-process handle either. Every scored row carries `final_d: null` with
+this note rather than a silently-dropped field.

scripts/imaging/searches/multi_start_prodigy/delaunay_adapt_split.py

@@ -0,0 +1,51 @@
+"""First-class af.MultiStartProdigy search profiling — imaging Delaunay + free
+AdaptSplit regularization.
+
+Added for W5 Phase 8B (issue #162): the NaN-wall cell for the bijector A/B
+(``bijector_ab.py``). Unlike ``delaunay.py`` (``model_type="delaunay_matern"``,
+the REGISTERED gradient cell — free Matérn regularization, which degrades
+gracefully), this uses ``model_type="delaunay_adapt_split"``
+(``searches._setup._delaunay_adapt_split_model``): Delaunay mesh + free
+``AdaptSplit`` regularization, the pairing Phase 8A (CP-4, ``slogdet_ab.py``)
+identified as the actual **NaN wall** — the #104 doubly-squared lambda^4
+fragility, lanes dying instead of learning at high coefficients. It is a
+DIAGNOSTIC cell, not a sampler-benchmark recommendation (see
+``_setup._delaunay_adapt_split_model``'s docstring): it exists so the bijector
+A/B can be run against the configuration that actually walls, alongside
+``knn`` (the cell the original pre-registration named, whose high-coefficient
+region is a finite floor rather than a wall).
+"""
+
+from __future__ import annotations
+
+import sys as _sys
+from pathlib import Path as _Path
+
+
+def _profiling_root() -> _Path:
+    for _p in _Path(__file__).resolve().parents:
+        if (_p / "ruff.toml").exists():
+            return _p
+    raise RuntimeError("autolens_profiling root (ruff.toml) not found")
+
+
+_misc_dir = str(_profiling_root() / "scripts" / "misc")
+if _misc_dir not in _sys.path:
+    _sys.path.insert(0, _misc_dir)
+
+
+import sys
+from pathlib import Path
+
+_REPO_ROOT = _profiling_root()
+if str(_REPO_ROOT) not in sys.path:
+    sys.path.insert(0, str(_REPO_ROOT))
+
+from searches._runner import run_search  # noqa: E402
+
+run_search(
+    sampler="multi_start_prodigy",
+    dataset_class="imaging",
+    model_type="delaunay_adapt_split",
+    default_instrument="hst",
+)

scripts/imaging/searches/multi_start_prodigy/mge.py

@@ -0,0 +1,48 @@
+"""First-class af.MultiStartProdigy search profiling — imaging MGE.
+
+Added for W5 Phase 8B (issue #162): the MGE control cell for the bijector A/B
+(``bijector_ab.py``). MGE has no pixelization / regularization coefficients at
+all, so it carries no ``LogUniformPrior`` under a ``"regularization."`` path —
+the ``log_reg`` bijector arm (``searches._samplers._bijector_object``)
+therefore resolves to an empty ``kind_by_path`` and is bit-identical to
+``BijectorNone`` here. That is the point: MGE is the falsification control for
+F4 ("MGE control differs by any bit"), run alongside the pixelized cells
+rather than through a bespoke path.
+
+See ``multi_start_adam/mge.py`` for the same model under a different
+optimizer, and ``searches/README.md`` for the design.
+"""
+
+from __future__ import annotations
+
+import sys as _sys
+from pathlib import Path as _Path
+
+
+def _profiling_root() -> _Path:
+    for _p in _Path(__file__).resolve().parents:
+        if (_p / "ruff.toml").exists():
+            return _p
+    raise RuntimeError("autolens_profiling root (ruff.toml) not found")
+
+
+_misc_dir = str(_profiling_root() / "scripts" / "misc")
+if _misc_dir not in _sys.path:
+    _sys.path.insert(0, _misc_dir)
+
+
+import sys
+from pathlib import Path
+
+_REPO_ROOT = _profiling_root()
+if str(_REPO_ROOT) not in sys.path:
+    sys.path.insert(0, str(_REPO_ROOT))
+
+from searches._runner import run_search  # noqa: E402
+
+run_search(
+    sampler="multi_start_prodigy",
+    dataset_class="imaging",
+    model_type="mge",
+    default_instrument="hst",
+)