Autodoc

.github/workflows/ci.yml

@@ -213,8 +213,37 @@ jobs:
           # re-run here too -- cheap relative to the cosmosis run, and it
           # confirms they also pass with the full stack installed (not just
           # in the stripped-down fast env).
-          pytest tests -v
+          pytest tests -v --cov --cov-branch --cov-report=xml
+        env:
+          DISPLAY: test
 
+      - name: Upload coverage reports to Codecov
+        uses: codecov/codecov-action@v5
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+
+
+  doc-valid:
+    name: Documentation Validation
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: conda-incubator/setup-miniconda@v3
+        with:
+          activate-environment: clpipe-full
+          environment-file: ${{ env.FULL_ENV_FILE }}
+      - uses: actions/cache@v4
+        with:
+          path: ${{ env.CONDA }}/pkgs
+          key: ${{ runner.os }}-conda-${{ env.CONDA_ENV_NAME }}
+      - name: Install the package
+        shell: bash -l {0}
+        run: |
+          pip install . -vv --no-deps --no-build-isolation
+      - name: Run the docs
+        shell: bash -l {0}
+        run: |
+          make -C docs/ html
   # ------------------------------------------------------------------ #
   # (Optional, disabled) lint. Enable in a separate formatting PR once  #
   # the pipeline files are actually black/isort-clean, so the           #

.github/workflows/docs.yml

@@ -0,0 +1,58 @@
+name: docs
+
+# Builds the Sphinx HTML docs on every PR/push touching them (as a check),
+# and publishes to GitHub Pages on pushes to main.
+# One-time setup: Settings -> Pages -> Source: "GitHub Actions".
+
+on:
+  push:
+    branches: [autodoc, main]
+    paths: ["docs/**", "clpipe/**", ".github/workflows/docs.yml"]
+  pull_request:
+    paths: ["docs/**", "clpipe/**", ".github/workflows/docs.yml"]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: pages-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+          cache: pip
+          cache-dependency-path: docs/requirements.txt
+
+      - name: Install doc dependencies
+        run: pip install -r docs/requirements.txt
+
+      - name: Build HTML docs
+        working-directory: docs
+        run: make html
+
+      - name: Upload artifact (for inspection and Pages)
+        uses: actions/upload-pages-artifact@v3
+        with:
+          path: docs/_build/html
+
+  deploy:
+    needs: build
+    if: github.event_name != 'pull_request' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/autodoc')
+    runs-on: ubuntu-latest
+    permissions:
+      pages: write
+      id-token: write
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    steps:
+      - id: deployment
+        uses: actions/deploy-pages@v4

.gitignore

@@ -4,6 +4,7 @@
 __pycache__/
 *.py[cod]
 *.so
+.coverage
 
 # ─────────────────────────────────────────
 # Jupyter
@@ -28,3 +29,7 @@ slurm*
 *.tmp
 *.swp
 *~
+# ─────────────────────────────────────────
+# Documentation
+# ─────────────────────────────────────────
+docs/_build

README.md

@@ -1,6 +1,31 @@
+<p align="center">
+<img src="./docs/source/clpipe_logo.png" alt="drawing" width="500" style="display: block; margin: auto; "/>
+</p>
+
+
+
 # CLPipe
 
-Repository dedicated to the Cluster Working Group of the DESC-LSST collaboration.
+[![Build and Check](https://github.com/LSSTDESC/CLPipe/actions/workflows/ci.yml/badge.svg)](https://github.com/LSSTDESC/CLPipe/actions/workflows/ci.yml)
+[![Coverage Status](https://codecov.io/gh/LSSTDESC/CLPipe/branch/main/graph/badge.svg)](https://codecov.io/gh/LSSTDESC/CLPipe)
+[![Documentation](https://github.com/LSSTDESC/CLPipe/actions/workflows/docs.yml/badge.svg)](https://lsstdesc.github.io/CLPipe/)
+
+CLPipe is the current implementation of the LSST DESC Cluster Pipeline,
+developed by the Cluster Working Group of the LSST Dark Energy Science
+Collaboration (DESC) to prepare cosmological analyses of galaxy clusters with
+Rubin LSST data. This repository does not reimplement the analysis tools: it
+runs the existing DESC tools together, in a single automated and reproducible
+chain, from galaxy and cluster catalogs to cosmological parameter constraints.
+TXPipe measures the cluster abundance and the stacked weak lensing profiles,
+TJPCov computes the theoretical covariance of the cluster counts, and
+Firecrown builds the likelihood, with theory predictions from CROW (built on
+CCL and CLMM), which CosmoSIS then samples.
+
+The stages are connected with [ceci](https://github.com/LSSTDESC/ceci) through
+their input and output files, so a stage can be replaced or extended without
+changing the rest of the pipeline. The current version runs a binned analysis
+of cluster counts and stacked weak lensing profiles, and has been validated on
+mock data and on the cosmoDC2 redMaPPer cluster catalog.
 
 ## Installation
 
@@ -19,7 +44,7 @@ Because some Python packages required for Firecrown are incompatible with those
 # IN2P3
 conda activate /sps/lsst/groups/clusters/cl_pipeline_project/conda_envs/firecrown_clp
 
-# NERSC
+# NERSCstyle="display: block; margin: auto; "
 conda activate /global/cfs/projectdirs/lsst/groups/CL/cl_pipeline_project/conda_envs/firecrown_clp
 ```
 
@@ -92,7 +117,7 @@ Firecrown → inference inputs
 A separate job must be run to execute the final Firecrown inference — it is
 not launched automatically as part of the ceci pipeline.
 
-> **Note:** due to missing components in TJPCov, the Firecrown stage
+> **Note:** due to missing components in TJPCov, the CLPCovariance stage
 > currently performs additional computations to rescale the covariance
 > matrix.
 

clpipe/clp_covariance.py

@@ -42,7 +42,6 @@ class CLPCovariance(PipelineStage):
         "replace_tjpcov_cov": StageParameter(bool, True, msg="Replace TJPCov cluster-count covariance with CROW covariance."),
         # Selection function
         "sel_func": StageParameter(bool, True, msg="Include purity and completeness selection functions."),
-        "wazp_catalog": StageParameter(bool, False, msg="Use WaZP completeness parameters instead of Aguena+16 defaults (extra, non-default)."),
         "diagonal_shear_covariance": StageParameter(
             bool, True,
             msg=(
@@ -255,7 +254,6 @@ def replace_crow_counts(self, config_dict, sacc_full, cov_terms, full_cov, cosmo
 
         # This function should not exist as it should be implemented in TJPCov.
         # This is temporary and so most of the options and configurations are fixed.
-        is_wazp = config_dict.get("wazp_catalog", False)
         sel_func = config_dict.get("sel_func", True)
         mor_params = config_dict["mor_parameters"]
         hmf = ccl.halos.MassFuncDespali16(mass_def="200c")
@@ -274,12 +272,6 @@ def replace_crow_counts(self, config_dict, sacc_full, cov_terms, full_cov, cosmo
         cl_abundance          = ClusterAbundance(cosmo, hmf)
         purity_aguena         = None#purity_models.PurityAguena16LnProxy()
         completeness_aguena   = completeness_models.CompletenessAguena16()
-        if is_wazp:
-            purity_aguena = None
-            completeness_aguena['a_n'] = 1.570597
-            completeness_aguena['b_n'] = -0.028690
-            completeness_aguena['a_logm_piv'] = 14.264386
-            completeness_aguena['b_logm_piv'] = 0.029814
         redshift_distribution = kernel.SpectroscopicRedshift()
         if not sel_func:
             completeness_aguena = None

clpipe/clp_firecrown.py

@@ -30,6 +30,7 @@ class CLPFirecrown(PipelineStage):
     """Firecrown pipeline stage for cluster cosmology analysis.
 
     This stage:
+
     - Builds a Firecrown likelihood from a SACC file
     - Generates the corresponding CosmoSIS configuration
     - Writes parameter files for sampling, defaulting the cosmological
@@ -38,6 +39,7 @@ class CLPFirecrown(PipelineStage):
       how a parameter gets sampled instead of held fixed
 
     Key configuration groups:
+
     - Modeling options (hmf, mass range, redshift range)
     - Observable selection (cluster counts, shear)
     - Systematics (purity, completeness)
@@ -92,10 +94,10 @@ class CLPFirecrown(PipelineStage):
         "emcee_walkers": StageParameter(int, 100, msg="Number of emcee walkers."),
         "emcee_samples": StageParameter(int, 20000, msg="Number of emcee samples."),
         "emcee_nsteps": StageParameter(int, 20, msg="Number of emcee steps per sample."),
-        "polycord_live_points": StageParameter(int, 500, msg="Number of PolyChord live points."),
-        "polycord_num_repeats": StageParameter(int, 30, msg="Number of PolyChord repeats."),
-        "polycord_tolerance": StageParameter(float, 0.05, msg="PolyChord evidence tolerance."),
-        "polycord_feedback": StageParameter(int, 1, msg="PolyChord feedback level."),
+        "polychord_live_points": StageParameter(int, 500, msg="Number of PolyChord live points."),
+        "polychord_num_repeats": StageParameter(int, 30, msg="Number of PolyChord repeats."),
+        "polychord_tolerance": StageParameter(float, 0.05, msg="PolyChord evidence tolerance."),
+        "polychord_feedback": StageParameter(int, 1, msg="PolyChord feedback level."),
         "resume": StageParameter(bool, False, msg="If True, CosmoSIS appends to the existing chain at `filename` instead of starting fresh"),
         # Cosmology -- tau is a CAMB input (reionization optical depth), not
         # a CCL cosmology parameter, so it isn't in the fiducial cosmology
@@ -240,7 +242,10 @@ def generate_python_file(self, path_name):
                 # Core CROW imports
                 f.write("from crow import ClusterAbundance, ClusterShearProfile, kernel, mass_proxy\n")
                 f.write("from crow.properties import ClusterProperty\n")
-                f.write("from crow.recipes.binned_grid import GridBinnedClusterRecipe\n\n")
+                if use_grid:
+                    f.write("from crow.recipes.binned_grid import GridBinnedClusterRecipe\n\n")
+                else:
+                    f.write("from crow.recipes.binned_exact import ExactBinnedClusterRecipe\n\n")
                 f.write(
                     "from crow import purity_models, completeness_models\n"
                 )
@@ -411,10 +416,10 @@ def generate_ini_file(self, output_ini_path, likelihood_source_name, values_file
             emcee_samples = cfg["emcee_samples"]
             emcee_nsteps = cfg["emcee_nsteps"]
 
-            polycord_live_points = cfg["polycord_live_points"]
-            polycord_num_repeats = cfg["polycord_num_repeats"]
-            polycord_tolerance = cfg["polycord_tolerance"]
-            polycord_feedback = cfg["polycord_feedback"]
+            polychord_live_points = cfg["polychord_live_points"]
+            polychord_num_repeats = cfg["polychord_num_repeats"]
+            polychord_tolerance = cfg["polychord_tolerance"]
+            polychord_feedback = cfg["polychord_feedback"]
 
             beta_parameters = cfg["beta_parameters"]
             with open(output_ini_path, 'w') as f:
@@ -478,10 +483,10 @@ def generate_ini_file(self, output_ini_path, likelihood_source_name, values_file
                 f.write(f"nsteps = {emcee_nsteps}\n")
 
                 f.write("[polychord]\n")
-                f.write(f"live_points = {polycord_live_points}\n")
-                f.write(f"num_repeats = {polycord_num_repeats}\n")
-                f.write(f"tolerance = {polycord_tolerance}\n")
-                f.write(f"feedback = {polycord_feedback}\n")
+                f.write(f"live_points = {polychord_live_points}\n")
+                f.write(f"num_repeats = {polychord_num_repeats}\n")
+                f.write(f"tolerance = {polychord_tolerance}\n")
+                f.write(f"feedback = {polychord_feedback}\n")
             print(f"INI file written to {output_ini_path}")
             return True
 

clpipe/cluster_theory_pred.py

@@ -159,6 +159,7 @@ def build_cluster_recipes_from_config(yml_file: str, sacc_file: str, set_params:
     Build cluster counts and (optionally) shear recipes from a YAML configuration and return theory/data vectors.
 
     This function:
+
     - Loads configuration from `yml_file` (supports a top-level CLPFirecrown key).
     - Builds a pyccl halo mass function and pyccl Cosmology (via build_ccl_cosmology_from_config).
     - Constructs completeness/purity models if requested.