Pairing check warns and records DR6-only losses; empty tiles produce empty products

astra.yaml

@@ -469,31 +469,27 @@ analyses:
           release: on the DR5 image of a tile only ~95% of detections pair.
           Unpaired detections still stop the run beyond both 2% and 20
           objects, preserving the check for other pixels. External-only
-          losses stop beyond both 20 objects and 2% of the external count
-          plus an additive 50-object allowance (only when pairs exist).
-          Four production edge tiles have 49, 59, 55 and 101 external-only
-          objects among 1680-2861 objects, concentrated in bright-star
-          halos/spikes at normal positive weight; only 0-3 detections lack
-          partners and paired positions agree to 7e-5 px. Local correlated
-          deblending losses do not scale with total tile population. The
-          allowance is empirical, rounded above the largest observed excess
-          over 2% (44 objects), not a binomial confidence bound. It weakens
-          sensitivity to modest external-side detection-config losses on
-          sparse tiles; there are too few uncensored sparse tiles to
-          calibrate a false-rejection rate. A zero-weight exemption would
-          not explain these losses, and spatial exemptions would add mask
-          and clustering choices. Only the acceptance gate changes: no
-          external-only object gains a shape and every previously accepted
-          catalogue retains identical membership and measurements.
+          losses never reject a tile: warn and record, then cut downstream.
+          The warning threshold is max(20, 0.02 N + 50), without the 50 if
+          no pairs exist; it no longer shares the SExtractor guard settings.
+          Sparse edge tiles 058.220 and 207.238 exceed the former budget,
+          despite almost every detection pairing. Local correlated
+          deblending losses need not scale with tile population.
+          Every joined tile records the five match counts, external paired
+          fraction and warning budget in its detection catalogue headers,
+          then in tile_detect and the published tile_make_cat.json manifest
+          beside final_cat. Join its unit (RRR.DDD) as a float to TILE_ID in
+          the final catalogue to select the population downstream.
+          No external-only object gains a shape; previously accepted
+          catalogue rows retain identical membership and measurements.
           PSF stars come from the exposure-level
           SExtractor run in both options. Image simulations, which have no
           UNIONS catalogue, keep SExtractor's own NUMBER.
           Values:
           input_types.data.tile_detection = unions_catalogue;
           config_tile_Sx.ini#SEXTRACTOR_RUNNER.MATCH_RADIUS = 1.0;
           config_tile_Sx.ini#SEXTRACTOR_RUNNER.MATCH_MIN_FRACTION = 0.98;
-          config_tile_Sx.ini#SEXTRACTOR_RUNNER.MATCH_TOLERATED_UNPAIRED = 20;
-          config_tile_Sx.ini#SEXTRACTOR_RUNNER.MATCH_EXTERNAL_UNPAIRED_ALLOWANCE = 50.
+          config_tile_Sx.ini#SEXTRACTOR_RUNNER.MATCH_TOLERATED_UNPAIRED = 20.
         default: unions_catalogue
         options:
           unions_catalogue:

src/shapepipe/modules/make_cat_package/make_cat.py

@@ -364,7 +364,7 @@ def _save_ngmix_data(self, ngmix_cat_path, moments=False):
         # A low match fraction can be valid (e.g. missing PSFs). Warn but
         # continue; unmatched detections retain their pre-filled sentinels.
         # Completeness gates reject premature or incomplete merges.
-        if len(ngmix_n_epoch) / self._cat_size_target < 0.1:
+        if self._cat_size_target and len(ngmix_n_epoch) / self._cat_size_target < 0.1:
             warning = (
                 f"Merged shape catalogue {ngmix_cat_path} contains only "
                 f"{len(ngmix_n_epoch)} of {self._cat_size_target} target "
@@ -638,7 +638,7 @@ def _save_psf_data(
         if not epoch_slots:
             n_slots = 0
         elif n_epoch_slots is None:
-            n_slots = np.max(self._final_cat["N_EPOCH"]) + 1
+            n_slots = int(np.max(self._final_cat["N_EPOCH"], initial=0)) + 1
         else:
             n_slots = n_epoch_slots
 

src/shapepipe/modules/sextractor_package/match_catalogue.py

@@ -16,6 +16,8 @@
 
 """
 
+import logging
+
 import numpy as np
 from astropy.io import ascii as asc
 from astropy.io import fits
@@ -30,6 +32,21 @@
 # The largest NUMBER the int32 NUMBER and SEG_VIGNET columns hold.
 MAX_NUMBER = np.iinfo(np.int32).max
 
+# Carried through tile_detect into the published tile_make_cat.json manifest.
+MATCH_HEADER = dict(
+    n_sextractor="MCHNSEX", n_external="MCHNEXT", n_paired="MCHNPAIR",
+    n_dropped="MCHNDROP", n_external_only="MCHNONLY",
+    external_paired_fraction="MCHFRAC", external_warning_budget="MCHBUDG",
+)
+
+
+def read_match_record(cat_path):
+    """Read join diagnostics, or None for a catalogue with no external join."""
+    header = fits.getheader(cat_path, "LDAC_OBJECTS")
+    if MATCH_HEADER["n_sextractor"] not in header:
+        return None
+    return {name: header[key] for name, key in MATCH_HEADER.items()}
+
 
 def mutual_nearest(x_a, y_a, x_b, y_b, radius):
     """One-to-one pairs of mutual nearest neighbours closer than ``radius``.
@@ -97,8 +114,7 @@ def relabel(seg_vignets, old_number, new_number):
 
 
 def match_catalogue(cat_path, ext_cat_path, radius=1.0, min_fraction=0.98,
-                    tolerated_unpaired=20, w_log=None,
-                    external_unpaired_allowance=50):
+                    tolerated_unpaired=20, w_log=None):
     """Take membership and ``NUMBER`` from an external catalogue.
 
     Each SExtractor row is paired with its mutual nearest neighbour in the
@@ -119,15 +135,11 @@ def match_catalogue(cat_path, ext_cat_path, radius=1.0, min_fraction=0.98,
     the DR6 catalogue) only ~95% pair. The guard is there to catch that: the
     SExtractor-side guard stops when unpaired detections exceed both
     ``(1 - min_fraction)`` of that side and ``tolerated_unpaired``.
-    External-only losses instead get an additive local deblending allowance
-    on top of the fractional budget (and retain the absolute tolerance).
-    Sparse DR6 edge tiles can lose dozens of children in one bright-star
-    halo while almost every detection pairs; those losses do not scale with
-    the tile's total population. This is an empirical allowance, not an
-    independent-object confidence bound. It weakens sensitivity to modest
-    external-side detection-config losses on sparse tiles, but does not
-    relax the SExtractor-side check for other pixels. With no pairs there
-    is no evidence for a local loss, so no additive allowance is granted.
+    External-only losses never reject a tile: warn above max(20, 0.02 N +
+    50), the former rejection budget (without the 50 when no pairs exist).
+    Record every tile's counts, external paired fraction and warning budget
+    in LDAC_OBJECTS headers for publication in the make-cat manifest, so
+    analysts can cut downstream. Sparse edge losses need not scale with N.
 
     Parameters
     ----------
@@ -139,31 +151,27 @@ def match_catalogue(cat_path, ext_cat_path, radius=1.0, min_fraction=0.98,
     radius : float, optional
         Largest separation of a pair, in pixels; default 1
     min_fraction : float, optional
-        Baseline paired fraction on each side; default 0.98. External-only
-        losses additionally receive ``external_unpaired_allowance``.
+        Minimum SExtractor paired fraction; default 0.98.
     tolerated_unpaired : int, optional
-        Number of unpaired rows on either side accepted whatever the
+        Number of unpaired SExtractor rows accepted whatever the
         fraction; default 20
     w_log : logging.Logger, optional
         Pipeline logger
-    external_unpaired_allowance : int, optional
-        Additive external-only loss budget, in objects; default 50. Only
-        applied when at least one pair exists; zero restores the old guard.
-
     Returns
     -------
     dict
         ``n_sextractor``, ``n_external``, ``n_paired``, ``n_dropped``
         (SExtractor rows without a partner) and ``n_external_only``
-        (external objects without a SExtractor row)
+        (external objects without a SExtractor row), plus
+        ``external_paired_fraction`` (zero for an empty external catalogue)
+        and ``external_warning_budget``
 
     Raises
     ------
     ValueError
         If the external ``NUMBER`` repeats or is not an integer in
         ``[1, MAX_NUMBER]``, or if unpaired rows exceed the absolute tolerance
-        and the side's loss budget: fractional only for SExtractor,
-        fractional plus the local allowance for the external catalogue
+        and the SExtractor fractional loss budget
 
     @sc [decision:detection.tile_detection]
     """
@@ -194,10 +202,10 @@ def match_catalogue(cat_path, ext_cat_path, radius=1.0, min_fraction=0.98,
         n_external_only=len(ext) - len(i_sex),
     )
     summary = ", ".join(f"{k}={v}" for k, v in counts.items())
-    def too_many_unpaired(n_total, allowance=0):
+    def too_many_unpaired(n_total):
         unpaired = n_total - len(i_sex)
         return (unpaired > tolerated_unpaired
-                and unpaired > (1 - min_fraction) * n_total + allowance)
+                and unpaired > (1 - min_fraction) * n_total)
 
     fraction = len(i_sex) / max(len(data), 1)
     if too_many_unpaired(len(data)):
@@ -210,16 +218,15 @@ def too_many_unpaired(n_total, allowance=0):
             + " release of the catalogue (DR6)."
         )
     fraction_ext = len(i_sex) / max(len(ext), 1)
-    allowance = external_unpaired_allowance if len(i_sex) else 0
-    if too_many_unpaired(len(ext), allowance):
-        raise ValueError(
+    budget = max(20, 0.02 * len(ext) + (50 if len(i_sex) else 0))
+    counts.update(external_paired_fraction=fraction_ext,
+                  external_warning_budget=budget)
+    if counts["n_external_only"] > budget:
+        (w_log or logging.getLogger(__name__)).warning(
             f"Only {fraction_ext:.4f} of the objects of {ext_cat_path} pair"
-            + f" with a row of {cat_path} within {radius} px (baseline"
-            + f" {min_fraction} beyond {tolerated_unpaired} unpaired;"
-            + f" additive external allowance {allowance};"
-            + f" {summary}). SExtractor finds fewer objects"
-            + " than the catalogue: check that the detection configuration"
-            + " is the catalogue's."
+            + f" with a row of {cat_path} within {radius} px (external-only"
+            + f" warning budget {budget}; {summary}). Recording for a"
+            + " downstream catalogue cut; the tile is not rejected."
         )
 
     old_number = np.asarray(data["NUMBER"])
@@ -241,6 +248,8 @@ def too_many_unpaired(n_total, allowance=0):
     new = fits.BinTableHDU.from_columns(
         columns, header=objects.header, name="LDAC_OBJECTS",
     )
+    for name, key in MATCH_HEADER.items():
+        new.header[key] = counts[name]
     fits.HDUList(
         [new if h is objects else h for h in hdus]
     ).writeto(cat_path, overwrite=True)

src/shapepipe/modules/sextractor_package/sextractor_script.py

@@ -76,7 +76,7 @@ def seg_vignet_column(seg_vignets):
     return fits.Column(
         name="SEG_VIGNET",
         format=f"{ny * nx}J",
-        array=seg_vignets.astype(np.int32, copy=False).reshape(n_obj, -1),
+        array=seg_vignets.astype(np.int32, copy=False).reshape(n_obj, ny * nx),
         dim=f"({nx},{ny})",
     )
 
@@ -193,7 +193,8 @@ def add_seg_vignet(cat_path, seg_path, w_log=None):
     ).writeto(cat_path, overwrite=True)
     if w_log:
         centre = size // 2
-        own = np.mean(seg_vignets[:, centre, centre] == data["NUMBER"])
+        own = (np.mean(seg_vignets[:, centre, centre] == data["NUMBER"])
+               if len(col) else np.nan)
         w_log.info(
             f"SEG_VIGNET cut from {seg_path} for {len(col)} objects;"
             + f" centre label = NUMBER for {own:.4f}"
@@ -430,7 +431,7 @@ def make_post_process(cat_path, f_wcs_path, pos_params, ccd_size, w_log=None):
     # (laid out as FITSCatalogue.add_col lays it out), then the EPOCH HDUs.
     new = fits.BinTableHDU.from_columns(
         objects.data.columns + fits.ColDefs([column("N_EPOCH", n_epoch)]),
-        name="LDAC_OBJECTS",
+        header=objects.header, name="LDAC_OBJECTS",
     )
     fits.HDUList(
         [new if h is objects else h for h in hdus] + epochs

src/shapepipe/modules/sextractor_runner.py

@@ -177,10 +177,6 @@ def sextractor_runner(
                 tolerated_unpaired=config.getint(
                     module_config_sec, "MATCH_TOLERATED_UNPAIRED"
                 ),
-                external_unpaired_allowance=config.getint(
-                    module_config_sec, "MATCH_EXTERNAL_UNPAIRED_ALLOWANCE",
-                    fallback=50,
-                ),
                 w_log=w_log,
             )
 

src/shapepipe/modules/vignetmaker_package/vignetmaker.py

@@ -335,6 +335,8 @@ def _get_stamp_me(self, image_dirs, image_pattern):
                     + f" HDU #{hdu_index}"
                 )
             if len(exp_names) == 0:
+                if len(all_id) == 0:
+                    continue  # A valid zero-object tile has empty EPOCH HDUs.
                 raise IndexError(
                     f"EXP_NAME list is empty in file {self._galcat_path}"     
                     + f" HDU #{hdu_index}"