From f80b26160f3f2f9a914e9659159b1d993f5427bc Mon Sep 17 00:00:00 2001 From: dariarom94 Date: Sun, 19 Jul 2026 23:22:50 +0200 Subject: [PATCH 01/60] Register cellposev4 in benchmark run scripts The cellposev4 segmentation component exists but was not listed in any run script, so it never ran. Add it to segmentation_methods alongside cellpose (active in the seqeracloud scripts, commented in the local/test scripts, matching the existing convention). Co-Authored-By: Claude Opus 4.8 --- scripts/run_benchmark/run_full_local.sh | 1 + scripts/run_benchmark/run_full_seqeracloud.sh | 1 + scripts/run_benchmark/run_test_local.sh | 1 + scripts/run_benchmark/run_test_seqeracloud.sh | 1 + 4 files changed, 4 insertions(+) diff --git a/scripts/run_benchmark/run_full_local.sh b/scripts/run_benchmark/run_full_local.sh index db9ba0457..b6fe94888 100755 --- a/scripts/run_benchmark/run_full_local.sh +++ b/scripts/run_benchmark/run_full_local.sh @@ -33,6 +33,7 @@ default_methods: segmentation_methods: - custom_segmentation # - cellpose + # - cellposev4 - binning # - stardist # - watershed diff --git a/scripts/run_benchmark/run_full_seqeracloud.sh b/scripts/run_benchmark/run_full_seqeracloud.sh index 65d353b09..ae3d5a3f3 100755 --- a/scripts/run_benchmark/run_full_seqeracloud.sh +++ b/scripts/run_benchmark/run_full_seqeracloud.sh @@ -25,6 +25,7 @@ default_methods: segmentation_methods: - custom_segmentation - cellpose + - cellposev4 - binning - stardist - watershed diff --git a/scripts/run_benchmark/run_test_local.sh b/scripts/run_benchmark/run_test_local.sh index 75ba1ac4a..b20058497 100755 --- a/scripts/run_benchmark/run_test_local.sh +++ b/scripts/run_benchmark/run_test_local.sh @@ -28,6 +28,7 @@ default_methods: segmentation_methods: - custom_segmentation # - cellpose + # - cellposev4 - binning # - stardist # - watershed diff --git a/scripts/run_benchmark/run_test_seqeracloud.sh b/scripts/run_benchmark/run_test_seqeracloud.sh index 86bef47e8..1a3124961 100755 --- a/scripts/run_benchmark/run_test_seqeracloud.sh +++ b/scripts/run_benchmark/run_test_seqeracloud.sh @@ -24,6 +24,7 @@ default_methods: segmentation_methods: - custom_segmentation - cellpose + - cellposev4 - binning - stardist - watershed From 1a2fa097b73295ad2f88671906ce53605403f1b3 Mon Sep 17 00:00:00 2001 From: dariarom94 Date: Mon, 20 Jul 2026 00:11:37 +0200 Subject: [PATCH 02/60] fix anndata version mismatch with txsim --- .../basic_count_aggregation/script.py | 60 ++++++++++++++++++- 1 file changed, 58 insertions(+), 2 deletions(-) diff --git a/src/methods_count_aggregation/basic_count_aggregation/script.py b/src/methods_count_aggregation/basic_count_aggregation/script.py index 5be4265e9..a1ac203f8 100644 --- a/src/methods_count_aggregation/basic_count_aggregation/script.py +++ b/src/methods_count_aggregation/basic_count_aggregation/script.py @@ -1,5 +1,61 @@ +import numpy as np +import pandas as pd +import anndata as ad import spatialdata as sd -import txsim as tx +from scipy.sparse import csr_matrix + + +def generate_adata(input_spots, cell_id_col="cell", gene_col="Gene"): + """Aggregate per-transcript assignments into a cell x gene AnnData. + + Reimplementation of txsim.preprocessing.generate_adata: txsim's version is + incompatible with anndata>=0.12 (it passes the removed `dtype=` argument to + AnnData() and uses per-row item assignment `adata[cell, :] = ...`, which + anndata no longer supports). This fills the count matrix directly and + produces an identical output structure (X, layers['raw_counts'], + obs/var stats, uns['spots'/'pct_noise']). + """ + spots = input_spots.copy() + pct_noise = sum(spots[cell_id_col] <= 0) / len(spots[cell_id_col]) + spots_raw = spots.copy() # kept in uns['spots']; 0 (background) -> None + spots_raw.loc[spots_raw[cell_id_col] == 0, cell_id_col] = None + spots = spots[spots[cell_id_col] > 0] + + cell_ids = pd.unique(spots[cell_id_col]) + genes = pd.unique(spots[gene_col]) + cell_pos = {c: i for i, c in enumerate(cell_ids)} + + # Populate the count matrix + centroids per cell (no AnnData item assignment). + # feature_name may be categorical, so value_counts() can return unobserved + # categories; reindex to `genes` (fill 0) to align to the var order, mirroring + # txsim's `.reindex(var_names, fill_value=0)`. + X = np.zeros((len(cell_ids), len(genes)), dtype=np.float32) + centroid_x = np.zeros(len(cell_ids)) + centroid_y = np.zeros(len(cell_ids)) + for cell_id, grp in spots.groupby(cell_id_col, sort=False): + row = cell_pos[cell_id] + cts = grp[gene_col].value_counts().reindex(genes, fill_value=0) + X[row, :] = cts.values + centroid_x[row] = grp["x"].mean() + centroid_y[row] = grp["y"].mean() + + adata = ad.AnnData(csr_matrix(X)) + adata.obs["cell_id"] = cell_ids + adata.obs_names = [f"{i:d}" for i in cell_ids] + adata.var_names = genes + adata.obs["centroid_x"] = centroid_x + adata.obs["centroid_y"] = centroid_y + + adata.uns["spots"] = spots_raw + adata.uns["pct_noise"] = pct_noise + adata.layers["raw_counts"] = adata.X.copy() + + adata.obs["n_counts"] = np.ravel(adata.layers["raw_counts"].sum(axis=1)) + adata.obs["n_genes"] = adata.layers["raw_counts"].getnnz(axis=1) + adata.var["n_counts"] = np.ravel(adata.layers["raw_counts"].sum(axis=0)) + adata.var["n_cells"] = adata.layers["raw_counts"].getnnz(axis=0) + return adata + ## VIASH START par = { @@ -14,7 +70,7 @@ sdata = sd.read_zarr(par['input']) df = sdata['transcripts'].compute() # TODO: Could optimize tx.preprocessing.generate_adata to work on spatialdata -adata = tx.preprocessing.generate_adata(df, cell_id_col='cell_id', gene_col='feature_name') #TODO: x and y refers to a specific coordinate system. Decide which space we want to use here. (probably should be handled in the previous assignment step) +adata = generate_adata(df, cell_id_col='cell_id', gene_col='feature_name') #TODO: x and y refers to a specific coordinate system. Decide which space we want to use here. (probably should be handled in the previous assignment step) adata.layers['counts'] = adata.layers['raw_counts'] del adata.layers['raw_counts'] adata.var["gene_name"] = adata.var_names From 82add803b17430c35df08b6b0222b76783bb1bb2 Mon Sep 17 00:00:00 2001 From: dariarom94 Date: Mon, 20 Jul 2026 00:29:37 +0200 Subject: [PATCH 03/60] add segger to workflow (test) --- src/workflows/run_benchmark/config.vsh.yaml | 3 ++- src/workflows/run_benchmark/main.nf | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/workflows/run_benchmark/config.vsh.yaml b/src/workflows/run_benchmark/config.vsh.yaml index dc79d85e2..d0a478f06 100644 --- a/src/workflows/run_benchmark/config.vsh.yaml +++ b/src/workflows/run_benchmark/config.vsh.yaml @@ -68,7 +68,7 @@ argument_groups: A list of transcript assignment methods to run. type: string multiple: true - default: "basic_transcript_assignment:baysor:clustermap:pciseq:comseg:proseg" + default: "basic_transcript_assignment:baysor:clustermap:pciseq:comseg:proseg:segger" - name: "--count_aggregation_methods" description: | A list of count aggregation methods to run. @@ -160,6 +160,7 @@ dependencies: - name: methods_transcript_assignment/pciseq - name: methods_transcript_assignment/comseg - name: methods_transcript_assignment/proseg + - name: methods_transcript_assignment/segger - name: methods_count_aggregation/basic_count_aggregation - name: methods_qc_filter/basic_qc_filter - name: methods_calculate_cell_volume/alpha_shapes diff --git a/src/workflows/run_benchmark/main.nf b/src/workflows/run_benchmark/main.nf index 64cfc7f82..f5cbd27c2 100644 --- a/src/workflows/run_benchmark/main.nf +++ b/src/workflows/run_benchmark/main.nf @@ -138,7 +138,8 @@ workflow run_wf { clustermap, pciseq, comseg, - proseg + proseg, + segger ] segm_ass_ch = segm_ch From 53e172839fb060747a63f09d304a9f51ed16df08 Mon Sep 17 00:00:00 2001 From: dariarom94 Date: Mon, 20 Jul 2026 00:43:15 +0200 Subject: [PATCH 04/60] duplicates when FOV stiching cleaned up --- src/datasets/loaders/allen_brain_cell_atlas_merfish/script.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/datasets/loaders/allen_brain_cell_atlas_merfish/script.py b/src/datasets/loaders/allen_brain_cell_atlas_merfish/script.py index f9e4fe979..056bebb2d 100644 --- a/src/datasets/loaders/allen_brain_cell_atlas_merfish/script.py +++ b/src/datasets/loaders/allen_brain_cell_atlas_merfish/script.py @@ -458,6 +458,10 @@ def parse_polygon_z0(row): how="left", predicate="within", ) +# A spot lying inside overlapping cell polygons yields multiple sjoin rows, +# making `joined` longer than `spots_df`. Keep one match per spot (the first) +# and realign to the original spot index so the assignment stays 1:1. +joined = joined[~joined.index.duplicated(keep="first")].reindex(spots_gdf.index) spots_df["cell_id"] = joined["cell_id"].values print( datetime.now() - t0, From 1186b7a09dbad16d2d4dc96a40fddb23fde9a978 Mon Sep 17 00:00:00 2001 From: dariarom94 Date: Mon, 20 Jul 2026 09:48:14 +0200 Subject: [PATCH 05/60] chunks issue atera --- src/data_processors/process_dataset/script.py | 35 +++++++++++-------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/src/data_processors/process_dataset/script.py b/src/data_processors/process_dataset/script.py index 28f7e57b8..68f32cb8e 100644 --- a/src/data_processors/process_dataset/script.py +++ b/src/data_processors/process_dataset/script.py @@ -8,6 +8,11 @@ import tempfile from pathlib import Path +try: + from xarray import DataTree +except ImportError: # older xarray ships DataTree as a separate package + from datatree import DataTree + # The 10x Atera loader (newer spatialdata/zarr) writes image arrays with # rectilinear (variable-size) chunk grids, which zarr>=3 gates behind an # experimental flag and refuses to read by default. Enable reading them so @@ -85,21 +90,21 @@ def rechunk_sdata(sdata, CHUNK_SIZE=1024): """ - for key in list(sdata.images.keys()): - image = sdata.images[key] - coords = list(image["scale0"].coords.keys()) - rechunk_strategy = {c: CHUNK_SIZE for c in coords} - if "c" in coords: - rechunk_strategy["c"] = image["scale0"]["image"].chunks[0][0] - image = image.chunk(rechunk_strategy) - sdata.images[key] = image - - for key in list(sdata.labels.keys()): - label_image = sdata.labels[key] - coords = list(label_image.coords.keys()) - rechunk_strategy = {c: CHUNK_SIZE for c in coords} - label_image = label_image.chunk(rechunk_strategy) - sdata.labels[key] = label_image + # Chunk only the spatial dims to a uniform size. Reading coords from the + # element directly fails for multiscale rasters (DataTree), whose root + # exposes no y/x coords -> an empty strategy leaves the coarse pyramid + # levels with their irregular (rectilinear) chunks, which zarr>=3 cannot + # write. Read coords from scale0 for DataTrees; keep the channel axis whole. + spatial = ("z", "y", "x") + for group in (sdata.images, sdata.labels): + for key in list(group.keys()): + elem = group[key] + ref = elem["scale0"] if isinstance(elem, DataTree) else elem + coords = list(ref.coords.keys()) + rechunk_strategy = {c: CHUNK_SIZE for c in coords if c in spatial} + if "c" in coords: # keep the channel axis as a single chunk + rechunk_strategy["c"] = -1 + group[key] = elem.chunk(rechunk_strategy) def subsample_adata_group_balanced(adata, group_key, n_samples, seed=0): From 18644d7a62a0667876aea1c075cb5a25920ab01c Mon Sep 17 00:00:00 2001 From: dariarom94 Date: Mon, 20 Jul 2026 09:52:22 +0200 Subject: [PATCH 06/60] segger update image --- .../segger/config.vsh.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/methods_transcript_assignment/segger/config.vsh.yaml b/src/methods_transcript_assignment/segger/config.vsh.yaml index bb8288d8c..a3b72b8f4 100644 --- a/src/methods_transcript_assignment/segger/config.vsh.yaml +++ b/src/methods_transcript_assignment/segger/config.vsh.yaml @@ -69,6 +69,18 @@ engines: - type: docker run: | pip install --no-cache-dir --extra-index-url=https://pypi.nvidia.com cuspatial-cu12 + # segger imports torch_geometric (at import time via _patches), lightning + # (segger segment -> Trainer) and torch_scatter (models: scatter_max) but + # declares NONE of them in its pyproject, so the github install below does + # not pull them. torch_geometric/lightning are pure Python. torch_scatter + # has no prebuilt wheel for the NGC image's custom torch build, so it must + # compile against the image's torch (--no-build-isolation). The build host + # has no GPU, so force a CUDA build for the target GPU archs (A100=8.0, + # A10/A40=8.6, L4/L40=8.9, H100=9.0). + - type: docker + run: | + pip install --no-cache-dir torch_geometric lightning + FORCE_CUDA=1 TORCH_CUDA_ARCH_LIST="8.0;8.6;8.9;9.0" pip install --no-cache-dir --no-build-isolation torch_scatter # segger trunk installed last so its (unpinned) deps resolve against the # RAPIDS/torch stack already present in the image. - type: python From ecb302d31df9d88d6851d94afbcd64eff4df9282 Mon Sep 17 00:00:00 2001 From: dariarom94 Date: Mon, 20 Jul 2026 15:01:20 +0200 Subject: [PATCH 07/60] claude fix for segger --- .../allen_brain_cell_atlas_merfish/script.py | 7 +++- .../segger/config.vsh.yaml | 41 +++++++++++-------- 2 files changed, 31 insertions(+), 17 deletions(-) diff --git a/src/datasets/loaders/allen_brain_cell_atlas_merfish/script.py b/src/datasets/loaders/allen_brain_cell_atlas_merfish/script.py index 056bebb2d..48d2984f3 100644 --- a/src/datasets/loaders/allen_brain_cell_atlas_merfish/script.py +++ b/src/datasets/loaders/allen_brain_cell_atlas_merfish/script.py @@ -75,7 +75,12 @@ def download_if_missing(url, path): if not Path(path).exists(): print(datetime.now() - t0, f"Downloading {url}", flush=True) - urllib.request.urlretrieve(url, path) + # Route required downloads through the retrying helper so a transient + # network hiccup (e.g. "Connection reset by peer" during the TLS + # handshake) does not abort the whole run. robust_urlretrieve is + # defined below but only referenced at call time, which is fine. + if not robust_urlretrieve(url, path): + raise RuntimeError(f"Failed to download required file after retries: {url}") def check_url(url): diff --git a/src/methods_transcript_assignment/segger/config.vsh.yaml b/src/methods_transcript_assignment/segger/config.vsh.yaml index a3b72b8f4..3c6a5ecaf 100644 --- a/src/methods_transcript_assignment/segger/config.vsh.yaml +++ b/src/methods_transcript_assignment/segger/config.vsh.yaml @@ -45,7 +45,12 @@ engines: # (cudf, cuspatial, torch). Develop/test on a CUDA host; `viash test` # will not run on a machine without an NVIDIA GPU. - type: docker - image: nvcr.io/nvidia/pytorch:25.06-py3 + # NGC 26.06 ships numpy 2.1 and a torch (2.13) compiled against numpy 2.x, so + # the torch<->numpy bridge (used by the `segger segment` subprocess) stays + # alive alongside the numpy-2.x task stack (spatialdata>=0.7.3 hard-requires + # numpy>=2 via multiscale-spatial-image -> xarray-dataclass). The older 25.06 + # image shipped numpy-1.x torch, which cannot coexist with spatialdata>=0.7.3. + image: nvcr.io/nvidia/pytorch:26.06-py3 setup: - type: apt packages: [procps, git, libxcb1] @@ -61,30 +66,34 @@ engines: - shapely - "rasterio>=1.3" - scikit-image - # cuspatial is required by segger.geometry.conversion at import time and is - # NOT shipped in the NGC PyTorch base image (only cudf is). No version pin, - # so pip resolves it against whatever cudf the image already has installed - # (pinning would risk diverging from the image's cudf). The NVIDIA index is - # required: cuspatial's libcudf-cu12/libcuspatial-cu12 deps are not on PyPI. + # cuspatial is imported by segger.geometry.conversion and is NOT shipped in + # the 26.06 image (unlike 25.06). Install from the NVIDIA index (its + # libcudf-cu12/libcuspatial-cu12 deps are not on PyPI); it pulls a matching + # cudf-cu12. Unpinned -> resolves to the newest cuda-12 build (25.4, the last + # cuda-12 cuspatial-cu12 on the index). - type: docker run: | pip install --no-cache-dir --extra-index-url=https://pypi.nvidia.com cuspatial-cu12 # segger imports torch_geometric (at import time via _patches), lightning # (segger segment -> Trainer) and torch_scatter (models: scatter_max) but # declares NONE of them in its pyproject, so the github install below does - # not pull them. torch_geometric/lightning are pure Python. torch_scatter - # has no prebuilt wheel for the NGC image's custom torch build, so it must - # compile against the image's torch (--no-build-isolation). The build host - # has no GPU, so force a CUDA build for the target GPU archs (A100=8.0, - # A10/A40=8.6, L4/L40=8.9, H100=9.0). + # not pull them. torch_geometric/lightning are pure Python. torch_scatter has + # no prebuilt wheel for the NGC custom torch build, so compile against it + # (--no-build-isolation). The build host has no GPU, so force a CUDA build for + # the target archs (A100=8.0, A10/A40=8.6, L4/L40=8.9, H100=9.0). - type: docker - run: | - pip install --no-cache-dir torch_geometric lightning - FORCE_CUDA=1 TORCH_CUDA_ARCH_LIST="8.0;8.6;8.9;9.0" pip install --no-cache-dir --no-build-isolation torch_scatter - # segger trunk installed last so its (unpinned) deps resolve against the - # RAPIDS/torch stack already present in the image. + run: + - pip install --no-cache-dir torch_geometric lightning + - FORCE_CUDA=1 TORCH_CUDA_ARCH_LIST="8.0;8.6;8.9;9.0" pip install --no-cache-dir --no-build-isolation torch_scatter - type: python github: dpeerlab/segger + # cudf 25.4 needs pandas<2.2.4, but segger (via scanpy 1.12 / anndata 0.13) + # pulls pandas 3.0, which removed pandas.api.types.is_interval that cudf + # imports -> `import cudf` crashes. Pin pandas back into cudf's supported + # range as the LAST step so it wins over the deps above. numpy stays 2.x. + - type: docker + run: + - pip install --no-cache-dir "pandas>=2.0,<2.2.4" - type: native runners: From d400ebec013786e3b02b274fe1b87008c3f469c7 Mon Sep 17 00:00:00 2001 From: dariarom94 Date: Mon, 20 Jul 2026 18:10:23 +0200 Subject: [PATCH 08/60] atera version fix --- src/datasets/loaders/tenx_atera/config.vsh.yaml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/datasets/loaders/tenx_atera/config.vsh.yaml b/src/datasets/loaders/tenx_atera/config.vsh.yaml index ed74ffeac..fd96307de 100644 --- a/src/datasets/loaders/tenx_atera/config.vsh.yaml +++ b/src/datasets/loaders/tenx_atera/config.vsh.yaml @@ -60,8 +60,15 @@ engines: setup: - type: python pypi: - - spatialdata-io==0.6.0 - - spatialdata==0.7.2 + # spatialdata 0.7.2 + zarr>=3.2 writes multiscale rasters (the xenium + # cell/nucleus label pyramids) as rectilinear chunk grids, which zarr + # gates behind an experimental flag and which spatialdata cannot read + # back (rechunking to a uniform grid does NOT help -- dask's to_zarr + # passes per-chunk sizes so the grid is always rectilinear). spatialdata + # 0.8.0 (with ome-zarr 0.18 + dask>=2026.3) emits a regular grid again + # and round-trips cleanly. Verified on the cluster against the real data. + - spatialdata-io==0.7.1 + - spatialdata==0.8.0 - type: native runners: From 64d7b4e534ce4d9100c74af25efe094765de8c05 Mon Sep 17 00:00:00 2001 From: dariarom94 Date: Mon, 20 Jul 2026 18:11:19 +0200 Subject: [PATCH 09/60] wf for the custom rnaseq scripts --- .../process_scrnaseq/config.vsh.yaml | 59 +++++++++++++++ .../workflows/process_scrnaseq/main.nf | 72 +++++++++++++++++++ 2 files changed, 131 insertions(+) create mode 100644 src/datasets/workflows/process_scrnaseq/config.vsh.yaml create mode 100644 src/datasets/workflows/process_scrnaseq/main.nf diff --git a/src/datasets/workflows/process_scrnaseq/config.vsh.yaml b/src/datasets/workflows/process_scrnaseq/config.vsh.yaml new file mode 100644 index 000000000..9841d81ab --- /dev/null +++ b/src/datasets/workflows/process_scrnaseq/config.vsh.yaml @@ -0,0 +1,59 @@ +name: process_scrnaseq +namespace: datasets/workflows + +# Generic single-cell RNA-seq processing workflow. +# +# Unlike the per-dataset process__sc workflows, this one is loader-agnostic: +# it takes an already-loaded / standardized raw SC dataset as --input and runs the +# shared processing chain (log_cp -> hvg -> pca -> knn -> extract_uns_metadata) to +# produce a file_common_scrnaseq dataset (with a 'normalized' layer, HVG, PCA and +# kNN) usable as the input_sc reference for process_datasets. +# +# Use it for custom references that have no dedicated loader (e.g. the MPII human +# lung annotation standardized by scripts/create_resources/sc/standardize_mpii_human_lung_sc.py). + +argument_groups: + - name: Inputs + arguments: + - type: file + name: --input + required: true + description: | + A raw single-cell RNA-seq dataset (loader-equivalent h5ad): a 'counts' + layer, cell_type annotations in .obs, feature_name (gene symbols) in .var + and dataset_* metadata in .uns. + - name: Outputs + arguments: + - name: "--output_dataset" + __merge__: /src/api/file_common_scrnaseq.yaml + direction: output + required: true + default: "$id/dataset.h5ad" + - name: "--output_meta" + direction: output + type: file + description: "Dataset metadata" + default: "$id/dataset_meta.yaml" + +resources: + - type: nextflow_script + path: main.nf + entrypoint: run_wf + - path: /common/nextflow_helpers/helper.nf + +dependencies: + - name: normalization/log_cp + repository: datasets + - name: processors/pca + repository: datasets + - name: processors/hvg + repository: datasets + - name: processors/knn + repository: datasets + - name: utils/extract_uns_metadata + repository: openproblems + +runners: + - type: nextflow + directives: + label: [midcpu, midmem, hightime] diff --git a/src/datasets/workflows/process_scrnaseq/main.nf b/src/datasets/workflows/process_scrnaseq/main.nf new file mode 100644 index 000000000..4db197416 --- /dev/null +++ b/src/datasets/workflows/process_scrnaseq/main.nf @@ -0,0 +1,72 @@ +include { findArgumentSchema } from "${meta.resources_dir}/helper.nf" + +workflow auto { + findStates(params, meta.config) + | meta.workflow.run( + auto: [publish: "state"] + ) +} + +workflow run_wf { + take: + input_ch + + main: + output_ch = input_ch + + // Normalize (log CP10k). Reads the raw counts from --input. + | log_cp.run( + key: "log_cp10k", + fromState: [ + "input": "input" + ], + args: [ + "normalization_id": "log_cp10k", + "n_cp": 10000 + ], + toState: [ + "output_normalized": "output" + ] + ) + | hvg.run( + fromState: ["input": "output_normalized"], + toState: ["output_hvg": "output"] + ) + + | pca.run( + fromState: ["input": "output_hvg"], + toState: ["output_pca": "output" ] + ) + + | knn.run( + fromState: ["input": "output_pca"], + toState: ["output_knn": "output"] + ) + // add synonym + | map{ id, state -> + [id, state + [output_dataset: state.output_knn]] + } + + | extract_uns_metadata.run( + fromState: { id, state -> + def schema = findArgumentSchema(meta.config, "output_dataset") + // workaround: convert GString to String + schema = iterateMap(schema, { it instanceof GString ? it.toString() : it }) + def schemaYaml = tempFile("schema.yaml") + writeYaml(schema, schemaYaml) + [ + "input": state.output_dataset, + "schema": schemaYaml + ] + }, + toState: ["output_meta": "output"] + ) + + | setState([ + "output_dataset": "output_dataset", + "output_meta": "output_meta" + ]) + + emit: + output_ch +} From 3edfbf192629d8b5c63038dc94392d49d1664514 Mon Sep 17 00:00:00 2001 From: dariarom94 Date: Mon, 20 Jul 2026 23:32:55 +0200 Subject: [PATCH 10/60] adjust the loader image name --- src/base/labels_nebius.config | 22 +++++++++++++--------- src/datasets/loaders/tenx_atera/script.py | 12 ++++++++++-- src/datasets/loaders/tenx_xenium/script.py | 12 ++++++++++-- 3 files changed, 33 insertions(+), 13 deletions(-) diff --git a/src/base/labels_nebius.config b/src/base/labels_nebius.config index 741e88a40..f9eb03fc1 100644 --- a/src/base/labels_nebius.config +++ b/src/base/labels_nebius.config @@ -33,7 +33,11 @@ process { // Default disk space disk = 50.GB - // Always pull the latest image digest so nodes never serve a stale cached image + // Always pull the latest image digest so nodes never serve a stale cached image. + // NOTE: a `pod` directive in a withLabel block REPLACES this one (Nextflow does not + // merge `pod` across selectors) — so every label that sets `pod` (e.g. a nodeSelector) + // MUST also include [imagePullPolicy: 'Always'], or its processes silently revert to + // IfNotPresent and can run a stale cached image. Keep that in sync below. pod = [[imagePullPolicy: 'Always']] // Retry for exit codes that have something to do with memory issues @@ -55,13 +59,13 @@ process { // Nebius 48vcpu-192gb nodes: 188 GB allocatable memory = { get_memory( 25.GB * task.attempt ) } disk = { 50.GB * task.attempt } - pod = [[nodeSelector: 'nebius.com/node-group-id=mk8snodegroup-e00p5wjvb7bc55b2js']] + pod = [[nodeSelector: 'nebius.com/node-group-id=mk8snodegroup-e00p5wjvb7bc55b2js'], [imagePullPolicy: 'Always']] } withLabel: midmem { // Nebius 64vcpu-256gb nodes: 251 GB allocatable memory = { get_memory( 50.GB * task.attempt ) } disk = { 100.GB * task.attempt } - pod = [[nodeSelector: 'nebius.com/node-group-id=mk8snodegroup-e00wvqfyde1nfwezs0']] + pod = [[nodeSelector: 'nebius.com/node-group-id=mk8snodegroup-e00wvqfyde1nfwezs0'], [imagePullPolicy: 'Always']] } // highmem: 200 GB base. No hard nodeSelector — the 200 GiB request itself // restricts placement to the two large cpu-d3 node groups (251 GiB × 20 and @@ -96,7 +100,7 @@ process { accelerator = 1 memory = 28.GB disk = 200.GB - pod = [[nodeSelector: 'nebius.com/node-group-id=mk8snodegroup-e00t775jb99svb7k5r']] + pod = [[nodeSelector: 'nebius.com/node-group-id=mk8snodegroup-e00t775jb99svb7k5r'], [imagePullPolicy: 'Always']] containerOptions = { workflow.containerEngine == "singularity" ? '--nv': ( workflow.containerEngine == "docker" ? '--gpus all': null ) } } @@ -107,7 +111,7 @@ process { accelerator = 1 memory = 26.GB disk = 200.GB - pod = [[nodeSelector: 'nebius.com/node-group-id=mk8snodegroup-e00t775jb99svb7k5r']] + pod = [[nodeSelector: 'nebius.com/node-group-id=mk8snodegroup-e00t775jb99svb7k5r'], [imagePullPolicy: 'Always']] containerOptions = { workflow.containerEngine == "singularity" ? '--nv': ( workflow.containerEngine == "docker" ? '--gpus all': null ) } } @@ -118,7 +122,7 @@ process { accelerator = 1 memory = 26.GB disk = 200.GB - pod = [[nodeSelector: 'nebius.com/node-group-id=mk8snodegroup-e00t775jb99svb7k5r']] + pod = [[nodeSelector: 'nebius.com/node-group-id=mk8snodegroup-e00t775jb99svb7k5r'], [imagePullPolicy: 'Always']] containerOptions = { workflow.containerEngine == "singularity" ? '--nv': ( workflow.containerEngine == "docker" ? '--gpus all': null ) } } @@ -128,7 +132,7 @@ process { accelerator = 1 memory = 26.GB disk = 200.GB - pod = [[nodeSelector: 'nebius.com/node-group-id=mk8snodegroup-e00t775jb99svb7k5r']] + pod = [[nodeSelector: 'nebius.com/node-group-id=mk8snodegroup-e00t775jb99svb7k5r'], [imagePullPolicy: 'Always']] containerOptions = { workflow.containerEngine == "singularity" ? '--nv': ( workflow.containerEngine == "docker" ? '--gpus all': null ) } } @@ -139,7 +143,7 @@ process { accelerator = 1 memory = { [ 100.GB * task.attempt, 150.GB ].min() } disk = 200.GB - pod = [[nodeSelector: 'nebius.com/node-group-id=mk8snodegroup-e00dhcgx1xqjskycvc']] + pod = [[nodeSelector: 'nebius.com/node-group-id=mk8snodegroup-e00dhcgx1xqjskycvc'], [imagePullPolicy: 'Always']] containerOptions = { workflow.containerEngine == "singularity" ? '--nv': ( workflow.containerEngine == "docker" ? '--gpus all': null ) } } @@ -150,7 +154,7 @@ process { accelerator = 1 memory = { [ 120.GB * task.attempt, 180.GB ].min() } disk = 200.GB - pod = [[nodeSelector: 'nebius.com/node-group-id=mk8snodegroup-e00jp7hyqr094tmy35']] + pod = [[nodeSelector: 'nebius.com/node-group-id=mk8snodegroup-e00jp7hyqr094tmy35'], [imagePullPolicy: 'Always']] containerOptions = { workflow.containerEngine == "singularity" ? '--nv': ( workflow.containerEngine == "docker" ? '--gpus all': null ) } } diff --git a/src/datasets/loaders/tenx_atera/script.py b/src/datasets/loaders/tenx_atera/script.py index c594fb26a..5a1a8c89e 100644 --- a/src/datasets/loaders/tenx_atera/script.py +++ b/src/datasets/loaders/tenx_atera/script.py @@ -102,8 +102,16 @@ def extract_zip(input_zip: Path, output_dir: Path, strip_root: bool = False): cells_as_circles=False, ) -# remove morphology_focus -_ = sdata.images.pop("morphology_focus") +# Keep exactly one morphology raster named "morphology_mip"; process_dataset +# renames it to the API-required "image" element. Atera output mirrors Xenium +# Onboard Analysis v4, which ships only a multi-channel "morphology_focus" +# (channel 0 is DAPI, which the segmentation methods use via image[0]) and no +# separate "morphology_mip". Prefer the MIP if present, otherwise fall back to +# morphology_focus so the dataset always has an image. +if "morphology_mip" not in sdata.images and "morphology_focus" in sdata.images: + sdata["morphology_mip"] = sdata["morphology_focus"] +if "morphology_focus" in sdata.images: + del sdata.images["morphology_focus"] print("Add uns to table", flush=True) new_uns = { diff --git a/src/datasets/loaders/tenx_xenium/script.py b/src/datasets/loaders/tenx_xenium/script.py index c17e35cf5..8422da04b 100644 --- a/src/datasets/loaders/tenx_xenium/script.py +++ b/src/datasets/loaders/tenx_xenium/script.py @@ -59,8 +59,16 @@ cells_as_circles=False, ) - # remove morphology_focus - _ = sdata.images.pop("morphology_focus") + # Keep exactly one morphology raster named "morphology_mip"; process_dataset + # renames it to the API-required "image" element. Older Xenium ship both a + # single-channel "morphology_mip" and a multi-channel "morphology_focus"; + # newer Xenium (Onboard Analysis v2+) ship only "morphology_focus" (channel 0 + # is DAPI, which the segmentation methods use via image[0]). Prefer the MIP, + # otherwise fall back to morphology_focus so the dataset always has an image. + if "morphology_mip" not in sdata.images and "morphology_focus" in sdata.images: + sdata["morphology_mip"] = sdata["morphology_focus"] + if "morphology_focus" in sdata.images: + del sdata.images["morphology_focus"] print("Add uns to table", flush=True) new_uns = { From cbd2f12216921d46a3bc3b41843a5891753af79f Mon Sep 17 00:00:00 2001 From: dariarom94 Date: Mon, 20 Jul 2026 23:55:10 +0200 Subject: [PATCH 11/60] adjust the memory --- src/base/labels_nebius.config | 14 ++++++++++---- src/datasets/loaders/tenx_atera/script.py | 13 +++++++++++++ src/datasets/loaders/tenx_xenium/script.py | 13 +++++++++++++ 3 files changed, 36 insertions(+), 4 deletions(-) diff --git a/src/base/labels_nebius.config b/src/base/labels_nebius.config index f9eb03fc1..bfe4bf2a3 100644 --- a/src/base/labels_nebius.config +++ b/src/base/labels_nebius.config @@ -58,13 +58,17 @@ process { withLabel: lowmem { // Nebius 48vcpu-192gb nodes: 188 GB allocatable memory = { get_memory( 25.GB * task.attempt ) } - disk = { 50.GB * task.attempt } + // Scale disk with retries but cap at 200 GB: disk has no maxMemory-style + // clamp, so an uncapped scaled retry can exceed node local-disk capacity + // and leave the job Pending until it times out. + disk = { [ 50.GB * task.attempt, 200.GB ].min() } pod = [[nodeSelector: 'nebius.com/node-group-id=mk8snodegroup-e00p5wjvb7bc55b2js'], [imagePullPolicy: 'Always']] } withLabel: midmem { // Nebius 64vcpu-256gb nodes: 251 GB allocatable memory = { get_memory( 50.GB * task.attempt ) } - disk = { 100.GB * task.attempt } + // Scale disk with retries but cap at 200 GB: see lowmem note above. + disk = { [ 100.GB * task.attempt, 200.GB ].min() } pod = [[nodeSelector: 'nebius.com/node-group-id=mk8snodegroup-e00wvqfyde1nfwezs0'], [imagePullPolicy: 'Always']] } // highmem: 200 GB base. No hard nodeSelector — the 200 GiB request itself @@ -75,7 +79,8 @@ process { // units are binary: 200.GB = 200 GiB, which excludes the 195/188 GiB groups.) withLabel: highmem { memory = { get_memory( 200.GB * task.attempt ) } - disk = { 200.GB * task.attempt } + // Scale disk with retries but cap at 200 GB: see lowmem note above. + disk = 200.GB } // veryhighmem: 400 GB base — genuinely larger than highmem (previously both // were 200 GB, so veryhighmem bought no extra RAM). A 400 GiB request only @@ -83,7 +88,8 @@ process { // largest jobs without a hard nodeSelector. withLabel: veryhighmem { memory = { get_memory( 400.GB * task.attempt ) } - disk = { 400.GB * task.attempt } + // Scale disk with retries but cap at 200 GB: see lowmem note above. + disk = 200.GB } withLabel: lowsharedmem { containerOptions = { workflow.containerEngine != 'singularity' ? "--shm-size ${String.format("%.0f",task.memory.mega * 0.05)}" : ""} diff --git a/src/datasets/loaders/tenx_atera/script.py b/src/datasets/loaders/tenx_atera/script.py index 5a1a8c89e..6a2009ddf 100644 --- a/src/datasets/loaders/tenx_atera/script.py +++ b/src/datasets/loaders/tenx_atera/script.py @@ -113,6 +113,19 @@ def extract_zip(input_zip: Path, output_dir: Path, strip_root: bool = False): if "morphology_focus" in sdata.images: del sdata.images["morphology_focus"] +# Log the morphology image channel names so it's visible in the run log which +# channel is which — segmentation uses channel 0 (expected to be DAPI). +try: + _mip = sdata["morphology_mip"] + try: + _arr = _mip["scale0"].image # multiscale raster + except (TypeError, KeyError): + _arr = _mip # single-scale DataArray + _channels = [str(c) for c in _arr.coords["c"].values] if "c" in _arr.coords else "" + print(f"morphology image channels (channel 0 used for segmentation): {_channels}", flush=True) +except Exception as e: + print(f"Could not read morphology image channel names: {e}", flush=True) + print("Add uns to table", flush=True) new_uns = { "dataset_id": par["dataset_id"], diff --git a/src/datasets/loaders/tenx_xenium/script.py b/src/datasets/loaders/tenx_xenium/script.py index 8422da04b..b1a2d4b7e 100644 --- a/src/datasets/loaders/tenx_xenium/script.py +++ b/src/datasets/loaders/tenx_xenium/script.py @@ -70,6 +70,19 @@ if "morphology_focus" in sdata.images: del sdata.images["morphology_focus"] + # Log the morphology image channel names so it's visible in the run log which + # channel is which — segmentation uses channel 0 (expected to be DAPI). + try: + _mip = sdata["morphology_mip"] + try: + _arr = _mip["scale0"].image # multiscale raster + except (TypeError, KeyError): + _arr = _mip # single-scale DataArray + _channels = [str(c) for c in _arr.coords["c"].values] if "c" in _arr.coords else "" + print(f"morphology image channels (channel 0 used for segmentation): {_channels}", flush=True) + except Exception as e: + print(f"Could not read morphology image channel names: {e}", flush=True) + print("Add uns to table", flush=True) new_uns = { "dataset_id": par["dataset_id"], From 184260e3cbe68629b89b4ab9439a6c44b1cf5eff Mon Sep 17 00:00:00 2001 From: dariarom94 Date: Tue, 21 Jul 2026 01:28:53 +0200 Subject: [PATCH 12/60] troubleshootig edges --- .../basic_transcript_assignment/script.py | 5 +++++ .../baysor/script.py | 6 ++++++ .../proseg/script.py | 5 +++++ .../segger/config.vsh.yaml | 16 +++++++++++----- 4 files changed, 27 insertions(+), 5 deletions(-) diff --git a/src/methods_transcript_assignment/basic_transcript_assignment/script.py b/src/methods_transcript_assignment/basic_transcript_assignment/script.py index 20377b8aa..33ff24337 100644 --- a/src/methods_transcript_assignment/basic_transcript_assignment/script.py +++ b/src/methods_transcript_assignment/basic_transcript_assignment/script.py @@ -59,6 +59,11 @@ # Assign cell ids directly to transcripts_reset (clean-index single-partition dask DataFrame). # Using sdata[par['transcripts_key']] here would reintroduce the duplicate parquet index, # causing the same "cannot reindex on an axis with duplicate labels" error at write time. +# Clamp coords to the label-image bounds: transcripts at the crop boundary can +# round a few pixels past the raster edge (see get_crop_coords in +# process_dataset). Matches segger's handling; edge/background at the border. +y_coords = np.clip(y_coords, 0, label_image.shape[0] - 1) +x_coords = np.clip(x_coords, 0, label_image.shape[1] - 1) cell_ids = label_image[y_coords, x_coords] transcripts_reset["cell_id"] = pd.Series(cell_ids) diff --git a/src/methods_transcript_assignment/baysor/script.py b/src/methods_transcript_assignment/baysor/script.py index 2e6ca0928..748644adb 100644 --- a/src/methods_transcript_assignment/baysor/script.py +++ b/src/methods_transcript_assignment/baysor/script.py @@ -80,6 +80,12 @@ else: label_image = sdata_segm["segmentation"].to_numpy() +# Clamp coords to the label-image bounds: transcripts at the crop boundary can +# round a few pixels past the raster edge (see get_crop_coords in +# process_dataset). Matches segger's handling; edge/background at the border. +y_coords = np.clip(y_coords, 0, label_image.shape[0] - 1) +x_coords = np.clip(x_coords, 0, label_image.shape[1] - 1) + cell_id_dask_series = dask.dataframe.from_dask_array( dask.array.from_array( label_image[y_coords, x_coords], chunks=tuple(sdata[par['transcripts_key']].map_partitions(len).compute()) diff --git a/src/methods_transcript_assignment/proseg/script.py b/src/methods_transcript_assignment/proseg/script.py index 2c1acd0f2..8de0afe7d 100644 --- a/src/methods_transcript_assignment/proseg/script.py +++ b/src/methods_transcript_assignment/proseg/script.py @@ -75,6 +75,11 @@ # assign transcripts to cells based on x,y coords and segmentation image y_coords = transcripts.y.compute().to_numpy(dtype=np.int64) x_coords = transcripts.x.compute().to_numpy(dtype=np.int64) +# Clamp coords to the label-image bounds: transcripts at the crop boundary can +# round a few pixels past the raster edge (see get_crop_coords in +# process_dataset). Matches segger's handling; edge/background at the border. +y_coords = np.clip(y_coords, 0, label_image.shape[0] - 1) +x_coords = np.clip(x_coords, 0, label_image.shape[1] - 1) cell_id_dask_series = dask.dataframe.from_dask_array( dask.array.from_array( label_image[y_coords, x_coords], chunks=tuple(sdata[par['transcripts_key']].map_partitions(len).compute()) diff --git a/src/methods_transcript_assignment/segger/config.vsh.yaml b/src/methods_transcript_assignment/segger/config.vsh.yaml index 3c6a5ecaf..532ecb443 100644 --- a/src/methods_transcript_assignment/segger/config.vsh.yaml +++ b/src/methods_transcript_assignment/segger/config.vsh.yaml @@ -87,13 +87,19 @@ engines: - FORCE_CUDA=1 TORCH_CUDA_ARCH_LIST="8.0;8.6;8.9;9.0" pip install --no-cache-dir --no-build-isolation torch_scatter - type: python github: dpeerlab/segger - # cudf 25.4 needs pandas<2.2.4, but segger (via scanpy 1.12 / anndata 0.13) - # pulls pandas 3.0, which removed pandas.api.types.is_interval that cudf - # imports -> `import cudf` crashes. Pin pandas back into cudf's supported - # range as the LAST step so it wins over the deps above. numpy stays 2.x. + # cudf 25.4 (cuda-12; there is no cuspatial-cu13, so we are stuck on this + # RAPIDS) hard-requires pandas<2.2.4. But segger pulls the modern single-cell + # stack — anndata 0.13 and scanpy 1.12 — which BOTH require pandas>=2.3 + # (anndata 0.13's zarr reader calls pd.StringDtype(na_value=...), an API added + # in pandas 2.3), so under the pinned pandas 2.2.3 reading ANY AnnData/zarr + # table crashes with "StringDtype.__init__() got an unexpected keyword + # argument 'na_value'". Pin pandas back into cudf's range AND hold anndata / + # scanpy at their last pandas-2.2-compatible majors (anndata 0.12.x needs + # pandas>=2.1; scanpy 1.11.x needs pandas>=1.5) as the LAST step so they win + # over the deps above. numpy stays 2.x. See NOTES.md for the full conflict. - type: docker run: - - pip install --no-cache-dir "pandas>=2.0,<2.2.4" + - pip install --no-cache-dir "pandas>=2.0,<2.2.4" "anndata>=0.12,<0.13" "scanpy>=1.11,<1.12" - type: native runners: From 36631c4f61cc71f4da6732be8038197a9d773672 Mon Sep 17 00:00:00 2001 From: dariarom94 Date: Tue, 21 Jul 2026 10:56:47 +0200 Subject: [PATCH 13/60] segger update --- .../segger/config.vsh.yaml | 20 +++++++++++++------ .../segger/script.py | 18 ++++++++++------- 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/src/methods_transcript_assignment/segger/config.vsh.yaml b/src/methods_transcript_assignment/segger/config.vsh.yaml index 532ecb443..8dd320557 100644 --- a/src/methods_transcript_assignment/segger/config.vsh.yaml +++ b/src/methods_transcript_assignment/segger/config.vsh.yaml @@ -24,17 +24,17 @@ arguments: required: false description: "Number of training epochs for the segger GNN." default: 20 - - name: --prediction_expansion_ratio + - name: --prediction_graph_buffer_ratio type: double required: false - description: "Fraction of each polygon's equivalent radius used to expand it during prediction." - default: 0.5 + description: "Fraction of each polygon's equivalent radius used to buffer (expand) it when building the prediction graph. Maps to segger's --prediction-graph-buffer-ratio (renamed from --prediction-expansion-ratio in segger 0.1.0)." + default: 0.05 - name: --prediction_mode type: string required: false choices: [nucleus, cell, uniform] description: "Which polygon set drives the prediction graph." - default: nucleus + default: cell resources: - type: python_script @@ -52,8 +52,11 @@ engines: # image shipped numpy-1.x torch, which cannot coexist with spatialdata>=0.7.3. image: nvcr.io/nvidia/pytorch:26.06-py3 setup: + # libgl1 + libglib2.0-0: segger's writer imports segger.io -> cv2 (opencv), + # which needs libGL.so.1 / libgthread at import; without them the run crashes + # at write time with "libGL.so.1: cannot open shared object file". - type: apt - packages: [procps, git, libxcb1] + packages: [procps, git, libxcb1, libgl1, libglib2.0-0] - type: python github: - openproblems-bio/core#subdirectory=packages/python/openproblems @@ -85,8 +88,13 @@ engines: run: - pip install --no-cache-dir torch_geometric lightning - FORCE_CUDA=1 TORCH_CUDA_ARCH_LIST="8.0;8.6;8.9;9.0" pip install --no-cache-dir --no-build-isolation torch_scatter + # Pin to a specific commit: segger's CLI flags AND output schema change on + # trunk (0.1.0 renamed --prediction-expansion-ratio -> --prediction-graph- + # buffer-ratio and dropped the `keep` column). script.py targets THIS commit's + # contract, so bump this deliberately and re-check `segger segment --help` + # and data/writer.py before doing so. - type: python - github: dpeerlab/segger + github: dpeerlab/segger@0233cf62eb177b6e44e49318037705550765a010 # cudf 25.4 (cuda-12; there is no cuspatial-cu13, so we are stuck on this # RAPIDS) hard-requires pandas<2.2.4. But segger pulls the modern single-cell # stack — anndata 0.13 and scanpy 1.12 — which BOTH require pandas>=2.3 diff --git a/src/methods_transcript_assignment/segger/script.py b/src/methods_transcript_assignment/segger/script.py index ff43e2662..373aa7deb 100644 --- a/src/methods_transcript_assignment/segger/script.py +++ b/src/methods_transcript_assignment/segger/script.py @@ -22,8 +22,8 @@ 'coordinate_system': 'global', 'output': './temp/methods/segger/segger_assigned_transcripts.zarr', 'n_epochs': 20, - 'prediction_expansion_ratio': 0.5, - 'prediction_mode': 'nucleus', + 'prediction_graph_buffer_ratio': 0.05, + 'prediction_mode': 'cell', } meta = { 'name': 'segger', @@ -122,8 +122,10 @@ else: label_image = sdata_segm["segmentation"].to_numpy() # Clip to the label image bounds (transcripts can sit just outside after transform). +n_oob = int(np.count_nonzero((y_coords < 0) | (y_coords >= label_image.shape[0]) | (x_coords < 0) | (x_coords >= label_image.shape[1]))) y_coords = np.clip(y_coords, 0, label_image.shape[0] - 1) x_coords = np.clip(x_coords, 0, label_image.shape[1] - 1) +print(f"Clamped {n_oob}/{len(x_coords)} transcripts outside the {label_image.shape[0]}x{label_image.shape[1]} label image to its edge", flush=True) prior_cell_id = label_image[y_coords, x_coords].astype(np.int64) # 0 == background # Canonical transcripts frame (native coordinates, clean RangeIndex). Its row @@ -162,7 +164,7 @@ "-i", str(XENIUM_DIR), "-o", str(SEGGER_OUT_DIR), "--n-epochs", str(par["n_epochs"]), - "--prediction-expansion-ratio", str(par["prediction_expansion_ratio"]), + "--prediction-graph-buffer-ratio", str(par["prediction_graph_buffer_ratio"]), "--prediction-mode", par["prediction_mode"], ] # cudf's `validate_setup` aborts `import cudf` on hosts without a usable NVIDIA @@ -185,13 +187,15 @@ print('Reading segger output', flush=True) seg = pd.read_parquet(seg_pq) -# Keep only confident assignments. Mirror segger's canonical valid_cell_id -# predicate (drop null / "-1" / "UNASSIGNED" / "NONE", case-insensitive) so -# the unassigned sentinels don't leak into the output. +# Keep only confident assignments. segger 0.1.0 no longer emits a `keep` column; +# a transcript is confidently assigned when its per-gene similarity clears the +# learned threshold (segger applies exactly this `segger_similarity >= +# similarity_threshold` filter when writing its own AnnData). Also drop +# null / "-1" / "UNASSIGNED" / "NONE" cell ids (case-insensitive). _UNASSIGNED = {"-1", "UNASSIGNED", "NONE", "NAN", ""} seg_id_str = seg["segger_cell_id"].astype(str).str.strip().str.upper() keep_mask = ( - seg["keep"].astype(bool) + (seg["segger_similarity"] >= seg["similarity_threshold"]) & seg["segger_cell_id"].notna() & ~seg_id_str.isin(_UNASSIGNED) ) From 06261271a3aa8ed64dd06674497645ef66680858 Mon Sep 17 00:00:00 2001 From: dariarom94 Date: Tue, 21 Jul 2026 10:57:43 +0200 Subject: [PATCH 14/60] cell type label correction --- .../rctd/script.R | 21 +++++++++-- .../split/script.R | 35 +++++++++++-------- 2 files changed, 40 insertions(+), 16 deletions(-) diff --git a/src/methods_cell_type_annotation/rctd/script.R b/src/methods_cell_type_annotation/rctd/script.R index bcc28cfd2..6e4524e39 100644 --- a/src/methods_cell_type_annotation/rctd/script.R +++ b/src/methods_cell_type_annotation/rctd/script.R @@ -44,8 +44,20 @@ filtered_ref <- ref[,colData(ref)$cell_type %in% valid_celltypes] ref_counts <- assay(filtered_ref, "counts") # factor to drop filtered cell types colData(filtered_ref)$cell_type <- factor(colData(filtered_ref)$cell_type) -cell_types <- colData(filtered_ref)$cell_type +cell_types <- colData(filtered_ref)$cell_type names(cell_types) <- colnames(ref_counts) + +# spacexr::check_cell_types() rejects cell-type names containing '/' +# (e.g. "Ciliated/secretory cells", "T/NK lineage"). Sanitize the factor levels +# before building the Reference, and keep a map (safe name -> original name) so +# the predicted labels can be restored below to match the scRNAseq reference. +cell_type_name_map <- levels(cell_types) +names(cell_type_name_map) <- gsub("/", "_", levels(cell_types)) +if (any(duplicated(names(cell_type_name_map)))) { + names(cell_type_name_map) <- make.unique(names(cell_type_name_map)) +} +levels(cell_types) <- names(cell_type_name_map) + reference <- Reference(ref_counts, cell_types, min_UMI = 0) # check cores @@ -75,7 +87,12 @@ names(spatial_cell_types) <- rownames(results$results_df) # colData(sce)$cell_type <- "None_sp" -colData(sce)[names(spatial_cell_types),"cell_type"] <- as.character(spatial_cell_types) +# Restore the original cell-type names (undo the '/' sanitization). "None_sp" +# and anything not in the map are left unchanged. +predicted <- as.character(spatial_cell_types) +mapped <- unname(cell_type_name_map[predicted]) +predicted[!is.na(mapped)] <- mapped[!is.na(mapped)] +colData(sce)[names(spatial_cell_types),"cell_type"] <- predicted # Write the final object to h5ad format # set to 'w', is this ok? diff --git a/src/methods_expression_correction/split/script.R b/src/methods_expression_correction/split/script.R index 5210aab16..aa0e06957 100644 --- a/src/methods_expression_correction/split/script.R +++ b/src/methods_expression_correction/split/script.R @@ -7,12 +7,12 @@ library(Seurat) library(scuttle) ## VIASH START -par <- list( - "input_spatial_with_cell_types" = "task_ist_preprocessing/resources_test/task_ist_preprocessing/mouse_brain_combined/spatial_with_celltypes.h5ad", - "input_scrnaseq_reference"= "task_ist_preprocessing/resources_test/task_ist_preprocessing/mouse_brain_combined/scrnaseq_reference.h5ad", - "output" = "task_ist_preprocessing/tmp/split_corrected.h5ad", - "keep_all_cells" = FALSE, -) +par <- list( + "input_spatial_with_cell_types" = "task_ist_preprocessing/resources_test/task_ist_preprocessing/mouse_brain_combined/spatial_with_celltypes.h5ad", + "input_scrnaseq_reference"= "task_ist_preprocessing/resources_test/task_ist_preprocessing/mouse_brain_combined/scrnaseq_reference.h5ad", + "output" = "task_ist_preprocessing/tmp/split_corrected.h5ad", + "keep_all_cells" = FALSE, +) meta <- list( 'cpus': 4, @@ -51,6 +51,13 @@ ref_counts <- assay(filtered_ref, "counts") colData(filtered_ref)$cell_type <- factor(colData(filtered_ref)$cell_type) cell_types <- colData(filtered_ref)$cell_type names(cell_types) <- colnames(ref_counts) + +# spacexr::check_cell_types() rejects cell-type names containing '/' +# (e.g. "Ciliated/secretory cells", "T/NK lineage"). Sanitize the factor levels +# before building the Reference. SPLIT uses RCTD only internally to purify +# counts and does not emit cell-type labels, so no name restoration is needed. +levels(cell_types) <- make.unique(gsub("/", "_", levels(cell_types))) + reference <- Reference(ref_counts, cell_types, min_UMI = 0) # check cores @@ -83,14 +90,14 @@ res_split <- SPLIT::purify( ) -# create corrected counts layer in original SingleCell object -cat("Normalizing counts\n") - -# Preserve original normalized values before overwriting with corrected normalization -assay(sce, "normalized_uncorrected") <- assay(sce, "normalized") - -# First copy in counts -assay(sce, "corrected_counts") <- assay(sce, "counts") +# create corrected counts layer in original SingleCell object +cat("Normalizing counts\n") + +# Preserve original normalized values before overwriting with corrected normalization +assay(sce, "normalized_uncorrected") <- assay(sce, "normalized") + +# First copy in counts +assay(sce, "corrected_counts") <- assay(sce, "counts") # Then, replace only the updated cells assay(sce, "corrected_counts")[rownames(res_split$purified_counts), colnames(res_split$purified_counts)] <- res_split$purified_counts From 318643543dd7a0301bf995c1970d83bca5a30340 Mon Sep 17 00:00:00 2001 From: dariarom94 Date: Tue, 21 Jul 2026 10:58:51 +0200 Subject: [PATCH 15/60] fix boundaries --- .../basic_transcript_assignment/script.py | 2 ++ .../baysor/script.py | 19 +++++++------- .../clustermap/script.py | 15 +++++------ .../comseg/script.py | 26 ++++++++++++++++++- .../proseg/script.py | 17 ++++++------ 5 files changed, 53 insertions(+), 26 deletions(-) diff --git a/src/methods_transcript_assignment/basic_transcript_assignment/script.py b/src/methods_transcript_assignment/basic_transcript_assignment/script.py index 33ff24337..334e2b529 100644 --- a/src/methods_transcript_assignment/basic_transcript_assignment/script.py +++ b/src/methods_transcript_assignment/basic_transcript_assignment/script.py @@ -62,8 +62,10 @@ # Clamp coords to the label-image bounds: transcripts at the crop boundary can # round a few pixels past the raster edge (see get_crop_coords in # process_dataset). Matches segger's handling; edge/background at the border. +n_oob = int(np.count_nonzero((y_coords < 0) | (y_coords >= label_image.shape[0]) | (x_coords < 0) | (x_coords >= label_image.shape[1]))) y_coords = np.clip(y_coords, 0, label_image.shape[0] - 1) x_coords = np.clip(x_coords, 0, label_image.shape[1] - 1) +print(f"Clamped {n_oob}/{len(x_coords)} transcripts outside the {label_image.shape[0]}x{label_image.shape[1]} label image to its edge", flush=True) cell_ids = label_image[y_coords, x_coords] transcripts_reset["cell_id"] = pd.Series(cell_ids) diff --git a/src/methods_transcript_assignment/baysor/script.py b/src/methods_transcript_assignment/baysor/script.py index 748644adb..5cca8e4b5 100644 --- a/src/methods_transcript_assignment/baysor/script.py +++ b/src/methods_transcript_assignment/baysor/script.py @@ -83,26 +83,27 @@ # Clamp coords to the label-image bounds: transcripts at the crop boundary can # round a few pixels past the raster edge (see get_crop_coords in # process_dataset). Matches segger's handling; edge/background at the border. +n_oob = int(np.count_nonzero((y_coords < 0) | (y_coords >= label_image.shape[0]) | (x_coords < 0) | (x_coords >= label_image.shape[1]))) y_coords = np.clip(y_coords, 0, label_image.shape[0] - 1) x_coords = np.clip(x_coords, 0, label_image.shape[1] - 1) +print(f"Clamped {n_oob}/{len(x_coords)} transcripts outside the {label_image.shape[0]}x{label_image.shape[1]} label image to its edge", flush=True) -cell_id_dask_series = dask.dataframe.from_dask_array( - dask.array.from_array( - label_image[y_coords, x_coords], chunks=tuple(sdata[par['transcripts_key']].map_partitions(len).compute()) - ), - index=sdata[par['transcripts_key']].index -) -sdata[par['transcripts_key']]["cell_id"] = cell_id_dask_series +# Assign cell ids to the clean-index single-partition frame (transcripts_reset). +# Using sdata[par['transcripts_key']] here reintroduces the duplicate parquet index +# (multi-partition parquet restarts at 0 per partition) and fails downstream with +# "cannot reindex on an axis with duplicate labels". +cell_ids = label_image[y_coords, x_coords] +transcripts_reset["cell_id"] = pd.Series(cell_ids) ######################## # Run baysor with sopa # ######################## -# Create reduced sdata +# Create reduced sdata (use the clean-index frame that carries cell_id) sdata_sopa = sd.SpatialData( points={ - "transcripts": sdata[par['transcripts_key']] + "transcripts": transcripts_reset }, ) diff --git a/src/methods_transcript_assignment/clustermap/script.py b/src/methods_transcript_assignment/clustermap/script.py index 071b1bf70..87544c1c3 100644 --- a/src/methods_transcript_assignment/clustermap/script.py +++ b/src/methods_transcript_assignment/clustermap/script.py @@ -270,13 +270,12 @@ def run_clustermap( spots_cmap['clustermap'] += 1 #assign ClusterMap output (stored in spots_cmap) to the transcripts (i.e. assign transcripts to cells) -cell_id_dask_series = dask.dataframe.from_dask_array( - dask.array.from_array( - spots_cmap["clustermap"].values, chunks=tuple(sdata[par['transcripts_key']].map_partitions(len).compute()) - ), - index=sdata[par['transcripts_key']].index -) -sdata[par['transcripts_key']]["cell_id"] = cell_id_dask_series +# Assign to the clean-index single-partition frame (transcripts_reset). Using +# sdata[par['transcripts_key']] here reintroduces the duplicate parquet index +# (multi-partition parquet restarts at 0 per partition) and fails downstream with +# "cannot reindex on an axis with duplicate labels". +cell_ids = spots_cmap["clustermap"].values +transcripts_reset["cell_id"] = pd.Series(cell_ids) #create new .obs for cells based on the segmentation output (corresponding with the transcripts 'cell_id') unique_cells = np.unique(spots_cmap["clustermap"].values) @@ -298,7 +297,7 @@ def run_clustermap( print('Subsetting to transcripts cell id data', flush=True) sdata_transcripts_only = sd.SpatialData( points={ - "transcripts": sdata[par['transcripts_key']] + "transcripts": transcripts_reset }, tables={ "table": ad.AnnData( diff --git a/src/methods_transcript_assignment/comseg/script.py b/src/methods_transcript_assignment/comseg/script.py index 00c4330f4..2829b873f 100644 --- a/src/methods_transcript_assignment/comseg/script.py +++ b/src/methods_transcript_assignment/comseg/script.py @@ -1,8 +1,10 @@ +import os import spatialdata as sd import sopa import anndata as ad import pandas as pd import numpy as np +import dask.dataframe as dd ## VIASH START par = { @@ -23,6 +25,10 @@ "norm_vector": False, "allow_disconnected_polygon": True, } +meta = { + "name": "comseg", + "cpus": 15, +} ## VIASH END @@ -31,6 +37,16 @@ sdata = sd.read_zarr(par['input_ist']) sdata_segm = sd.read_zarr(par['input_segmentation']) +# Multi-partition parquet restarts its index at 0 per partition, producing duplicate +# index labels that break sopa's cell_id assignment and the final write with +# "cannot reindex on an axis with duplicate labels". Rebuild the transcripts as a +# single-partition frame with a clean RangeIndex before any sopa op touches them. +_tx = sdata[par['transcripts_key']] +_tx_reset = dd.from_pandas(_tx.compute().reset_index(drop=True), npartitions=1) +_tx_reset.attrs.update(_tx.attrs) +del sdata[par['transcripts_key']] +sdata[par['transcripts_key']] = _tx_reset + # Convert the prior segmentation to polygons sdata["segmentation_boundaries"] = sd.to_polygons(sdata_segm["segmentation"]) del sdata["segmentation_boundaries"]["label"] # make_transcript_patches will create a new label column and fails if one exists. @@ -61,7 +77,15 @@ } -# sopa.settings.parallelization_backend = 'dask' #TODO: get parallelization running. +# ComSeg processes each transcript patch independently and is pure-Python, so it +# runs single-threaded unless a parallelization backend is enabled. Use the dask +# backend to spread patches across the allocated CPUs (see midcpu/highmem labels). +n_workers = max((meta["cpus"] or os.cpu_count() or 1) - 1, 1) +sopa.settings.parallelization_backend = "dask" +sopa.settings.dask_client_kwargs["n_workers"] = n_workers +sopa.settings.dask_client_kwargs["threads_per_worker"] = 1 # CPU-bound work, avoid GIL contention +print(f"Running ComSeg with dask backend, n_workers={n_workers}", flush=True) + sopa.segmentation.comseg(sdata, config) # Assign transcripts to cell ids diff --git a/src/methods_transcript_assignment/proseg/script.py b/src/methods_transcript_assignment/proseg/script.py index 8de0afe7d..b11049ce4 100644 --- a/src/methods_transcript_assignment/proseg/script.py +++ b/src/methods_transcript_assignment/proseg/script.py @@ -78,15 +78,16 @@ # Clamp coords to the label-image bounds: transcripts at the crop boundary can # round a few pixels past the raster edge (see get_crop_coords in # process_dataset). Matches segger's handling; edge/background at the border. +n_oob = int(np.count_nonzero((y_coords < 0) | (y_coords >= label_image.shape[0]) | (x_coords < 0) | (x_coords >= label_image.shape[1]))) y_coords = np.clip(y_coords, 0, label_image.shape[0] - 1) x_coords = np.clip(x_coords, 0, label_image.shape[1] - 1) -cell_id_dask_series = dask.dataframe.from_dask_array( - dask.array.from_array( - label_image[y_coords, x_coords], chunks=tuple(sdata[par['transcripts_key']].map_partitions(len).compute()) - ), - index=sdata[par['transcripts_key']].index -) -sdata[par['transcripts_key']]["cell_id"] = cell_id_dask_series +print(f"Clamped {n_oob}/{len(x_coords)} transcripts outside the {label_image.shape[0]}x{label_image.shape[1]} label image to its edge", flush=True) +# Assign cell ids to the clean-index single-partition frame (transcripts_reset). +# Using sdata[par['transcripts_key']] here reintroduces the duplicate parquet index +# (multi-partition parquet restarts at 0 per partition) and fails downstream with +# "cannot reindex on an axis with duplicate labels". +cell_ids = label_image[y_coords, x_coords] +transcripts_reset["cell_id"] = pd.Series(cell_ids) ### Run Proseg with sopa @@ -95,7 +96,7 @@ print("Creating sopa SpatialData object") sdata_sopa = sd.SpatialData( points={ - "transcripts": sdata[par['transcripts_key']] + "transcripts": transcripts_reset }, ) From 3505718f559c9ad52659460653190d1b47d799c7 Mon Sep 17 00:00:00 2001 From: dariarom94 Date: Tue, 21 Jul 2026 11:42:02 +0200 Subject: [PATCH 16/60] OOM fixes --- .../moscot/config.vsh.yaml | 9 +++++++++ src/methods_cell_type_annotation/moscot/script.py | 6 ++++++ .../tangram/config.vsh.yaml | 5 ++++- src/methods_cell_type_annotation/tangram/script.py | 12 ++++++++++-- 4 files changed, 29 insertions(+), 3 deletions(-) diff --git a/src/methods_cell_type_annotation/moscot/config.vsh.yaml b/src/methods_cell_type_annotation/moscot/config.vsh.yaml index ea79aa1b5..c29da2da5 100644 --- a/src/methods_cell_type_annotation/moscot/config.vsh.yaml +++ b/src/methods_cell_type_annotation/moscot/config.vsh.yaml @@ -34,6 +34,15 @@ arguments: direction: input type: integer default: 500 #5000 + - name: --batch_size + required: false + direction: input + type: integer + # Number of cost-matrix rows/cols materialized at once during the solve. + # Computes the (n_sc x n_sc) / (n_sp x n_sp) GW costs online in batches instead + # of materializing the full matrix, which otherwise exhausts GPU memory on large + # references (the 101k-cell MPII lung reference needs ~76 GiB for n_sc x n_sc). + default: 1024 - name: --mapping_mode required: false direction: input diff --git a/src/methods_cell_type_annotation/moscot/script.py b/src/methods_cell_type_annotation/moscot/script.py index 1f5adc847..d69e21e45 100644 --- a/src/methods_cell_type_annotation/moscot/script.py +++ b/src/methods_cell_type_annotation/moscot/script.py @@ -41,6 +41,7 @@ 'epsilon': 0.01, 'tau': 0.3, 'rank': 500, #5000 + 'batch_size': 1024, 'mapping_mode': 'max', } meta = { @@ -82,12 +83,17 @@ xy_callback="local-pca", ) +# batch_size computes the GW cost matrices online in batches instead of +# materializing the full (n_sc x n_sc) matrix, which otherwise exhausts GPU +# memory on large references (e.g. the 101k-cell MPII lung reference needs +# ~76 GiB just for n_sc x n_sc). Trades a little speed for a bounded footprint. mp = mp.solve( alpha=par['alpha'], epsilon=par['epsilon'], tau_a=par['tau'], tau_b=par['tau'], rank=par['rank'], + batch_size=par['batch_size'], ) # Map annotations diff --git a/src/methods_cell_type_annotation/tangram/config.vsh.yaml b/src/methods_cell_type_annotation/tangram/config.vsh.yaml index 903c82c52..f6f9ec48b 100644 --- a/src/methods_cell_type_annotation/tangram/config.vsh.yaml +++ b/src/methods_cell_type_annotation/tangram/config.vsh.yaml @@ -15,7 +15,10 @@ arguments: required: false direction: input type: string - default: "cells" + # 'clusters' maps per-cell-type clusters -> space; 'cells' maps individual + # reference cells and exhausts GPU VRAM on large references (e.g. the 101k-cell + # MPII lung reference). 'clusters' is the appropriate mode for label projection. + default: "clusters" - name: --num_epochs required: false direction: input diff --git a/src/methods_cell_type_annotation/tangram/script.py b/src/methods_cell_type_annotation/tangram/script.py index 38f883c55..ffe679ab4 100644 --- a/src/methods_cell_type_annotation/tangram/script.py +++ b/src/methods_cell_type_annotation/tangram/script.py @@ -48,14 +48,22 @@ # 'uniform', when is a ndarray, shape = (number_spots,). # use 'uniform' if the spatial voxels are at single cell resolution (e.g. MERFISH). 'rna_count_based', assumes that # cell density is proportional to the number of RNA molecules. -adata_map = tg.map_cells_to_space( +# In 'cells' mode tangram learns a (n_sc_cells x n_spatial_cells) mapping matrix +# on the GPU, which runs out of VRAM for large references (e.g. the 101k-cell +# MPII lung reference). 'clusters' mode maps per-cell-type clusters instead, +# shrinking that matrix by ~n_cells_per_type and fitting comfortably on the GPU; +# it requires the cluster_label to average cells within. +map_kwargs = dict( adata_sc=adata_sc, adata_sp=adata_sp, device=device, mode=par['mode'], num_epochs=par['num_epochs'], - density_prior='uniform' + density_prior='uniform', ) +if par['mode'] == 'clusters': + map_kwargs['cluster_label'] = par['celltype_key'] +adata_map = tg.map_cells_to_space(**map_kwargs) # Spatial prediction dataframe is saved in `obsm` `tangram_ct_pred` of the spatial AnnData tg.project_cell_annotations( From d6e110a71d1430cac56c1167851a5cdb9f12ac6e Mon Sep 17 00:00:00 2001 From: dariarom94 Date: Tue, 21 Jul 2026 11:42:15 +0200 Subject: [PATCH 17/60] fix code --- .../pciseq/script.py | 30 +++++++++++++------ 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/src/methods_transcript_assignment/pciseq/script.py b/src/methods_transcript_assignment/pciseq/script.py index d7aa84d2d..f3001003c 100644 --- a/src/methods_transcript_assignment/pciseq/script.py +++ b/src/methods_transcript_assignment/pciseq/script.py @@ -135,15 +135,27 @@ def eta_update_no_assert(self): else: label_image = sdata_segm["segmentation"].to_numpy() -# There might be spots at the border of the image, pciseq runs into an error if this is the case. -# Build the mask as a positional numpy array from the transformed coords and apply the SAME mask -# to both the pciSeq input and the full transcripts, so pciSeq's per-spot assignments line up -# row-for-row with the transcripts when cell_ids are written back below. (Filtering the original -# multi-partition dask frame by a mask derived from the reset-index transcripts raises an -# "Unalignable boolean Series" IndexingError because the two indices no longer match.) -transcripts_at_border = (x_coords > (label_image.shape[1] - 0.5)) | (y_coords > (label_image.shape[0] - 0.5)) -transcripts_dataframe = transcripts_dataframe[~transcripts_at_border] -transcripts_full = transcripts_full[~transcripts_at_border] +# pciSeq internally drops spots that fall OUTSIDE the label image, and returns +# per-spot assignments only for the kept spots; txsim.run_pciSeq then does +# `spots['cell'] = assignments`, which raises "Length of values ... does not match +# length of index" when any spot was dropped. So we must pre-filter to exactly the +# spots pciSeq keeps. pciSeq rounds coords to pixel indices and keeps round(coord) +# in [0, size); the equivalent float bounds are (-0.5, size-0.5]. Filter BOTH borders +# (a crop transform can push transcripts off either the high OR the low/negative side; +# filtering only the high border left the negative-coord spots in and caused the +# mismatch on cropped datasets like MPII). +# Build the mask as a positional numpy array from the transformed coords and apply the +# SAME mask to both the pciSeq input and the full transcripts so pciSeq's per-spot +# assignments line up row-for-row when cell_ids are written back below. +transcripts_out_of_bounds = ( + (x_coords > (label_image.shape[1] - 0.5)) | (y_coords > (label_image.shape[0] - 0.5)) | + (x_coords < -0.5) | (y_coords < -0.5) +) +n_oob = int(transcripts_out_of_bounds.sum()) +print(f"Dropping {n_oob}/{len(x_coords)} transcripts outside the " + f"{label_image.shape[0]}x{label_image.shape[1]} label image (pciSeq drops these internally)", flush=True) +transcripts_dataframe = transcripts_dataframe[~transcripts_out_of_bounds] +transcripts_full = transcripts_full[~transcripts_out_of_bounds] # Grab all the pciSeq parameters opts_keys = [#'exclude_genes', From 4660f2619a4f8df8dda4127822933bcbd67655d1 Mon Sep 17 00:00:00 2001 From: dariarom94 Date: Tue, 21 Jul 2026 22:58:52 +0200 Subject: [PATCH 18/60] RCTD --- .../rctd/script.R | 19 ++++++++++++- .../split/config.vsh.yaml | 28 +++++++++++++++++++ .../split/script.R | 15 +++++++++- 3 files changed, 60 insertions(+), 2 deletions(-) diff --git a/src/methods_cell_type_annotation/rctd/script.R b/src/methods_cell_type_annotation/rctd/script.R index 6e4524e39..93a9cc345 100644 --- a/src/methods_cell_type_annotation/rctd/script.R +++ b/src/methods_cell_type_annotation/rctd/script.R @@ -43,10 +43,27 @@ filtered_ref <- ref[,colData(ref)$cell_type %in% valid_celltypes] ref_counts <- assay(filtered_ref, "counts") # factor to drop filtered cell types -colData(filtered_ref)$cell_type <- factor(colData(filtered_ref)$cell_type) +colData(filtered_ref)$cell_type <- factor(colData(filtered_ref)$cell_type) cell_types <- colData(filtered_ref)$cell_type names(cell_types) <- colnames(ref_counts) +# --- Diagnostics: confirm what RCTD actually reads. The reference itself has +# ~475/477 panel genes with clear cross-cell-type fold-change, so a "fewer +# than 10 DE genes" failure means the data reaching RCTD is degenerate: +# cell types collapsed to ~1, non-integer/normalized counts, or few genes +# shared with the spatial puck. This block surfaces which. --- +cat("=== RCTD input diagnostics ===\n") +cat(sprintf("spatial puck: %d genes x %d cells\n", nrow(counts), ncol(counts))) +cat(sprintf("reference (>=25 cells/type): %d genes x %d cells, %d cell types\n", + nrow(ref_counts), ncol(ref_counts), length(unique(as.character(cell_types))))) +print(utils::head(sort(table(as.character(cell_types)), decreasing = TRUE), 8)) +cat(sprintf("genes shared reference<->spatial: %d\n", + length(intersect(rownames(ref_counts), rownames(counts))))) +.rcv <- if (methods::is(ref_counts, "sparseMatrix")) ref_counts@x else as.numeric(ref_counts) +if (length(.rcv)) cat(sprintf("reference counts: min=%.4g max=%.4g mean=%.4g all-integer=%s\n", + min(.rcv), max(.rcv), mean(.rcv), isTRUE(all.equal(.rcv, round(.rcv))))) +cat("==============================\n") + # spacexr::check_cell_types() rejects cell-type names containing '/' # (e.g. "Ciliated/secretory cells", "T/NK lineage"). Sanitize the factor levels # before building the Reference, and keep a map (safe name -> original name) so diff --git a/src/methods_expression_correction/split/config.vsh.yaml b/src/methods_expression_correction/split/config.vsh.yaml index d6fe7e14c..9d56d1fba 100644 --- a/src/methods_expression_correction/split/config.vsh.yaml +++ b/src/methods_expression_correction/split/config.vsh.yaml @@ -17,6 +17,34 @@ arguments: type: boolean default: false description: Whether to keep cells with 0 counts (may cause errors if set to TRUE) + # RCTD's default DE-gene / UMI thresholds are tuned for whole-transcriptome + # references; on small iST panels (a few hundred genes, many similar fine-grained + # cell types) fold-changes are compressed and RCTD finds "fewer than 10 DE genes". + # These relaxed defaults mirror the rctd component so create.RCTD below succeeds. + - name: --gene_cutoff + type: double + default: 0.0 + description: "Minimum normalized mean expression for a gene to be a platform-effect DE gene (RCTD default 0.000125)." + - name: --fc_cutoff + type: double + default: 0.1 + description: "Minimum log-fold-change for a gene to be a platform-effect DE gene (RCTD default 0.5)." + - name: --gene_cutoff_reg + type: double + default: 0.0 + description: "Minimum normalized mean expression for a gene to be a regression DE gene (RCTD default 0.0002)." + - name: --fc_cutoff_reg + type: double + default: 0.1 + description: "Minimum log-fold-change for a gene to be a regression DE gene (RCTD default 0.75)." + - name: --umi_min + type: integer + default: 20 + description: "Minimum total UMI per spatial cell to be included (RCTD default 100; iST cells are low-count)." + - name: --umi_min_sigma + type: integer + default: 20 + description: "Minimum UMI for cells used to fit the platform-effect variance (RCTD default 300)." resources: - type: r_script diff --git a/src/methods_expression_correction/split/script.R b/src/methods_expression_correction/split/script.R index aa0e06957..7bcc06f00 100644 --- a/src/methods_expression_correction/split/script.R +++ b/src/methods_expression_correction/split/script.R @@ -12,6 +12,12 @@ par <- list( "input_scrnaseq_reference"= "task_ist_preprocessing/resources_test/task_ist_preprocessing/mouse_brain_combined/scrnaseq_reference.h5ad", "output" = "task_ist_preprocessing/tmp/split_corrected.h5ad", "keep_all_cells" = FALSE, + "gene_cutoff" = 0.0, + "fc_cutoff" = 0.1, + "gene_cutoff_reg" = 0.0, + "fc_cutoff_reg" = 0.1, + "umi_min" = 20, + "umi_min_sigma" = 20 ) meta <- list( @@ -67,7 +73,14 @@ cat(sprintf("Number of cores: %s\n", cores)) # Run the algorithm cat("Running RCTD\n") -myRCTD <- create.RCTD(puck, reference, max_cores = cores) +# Relaxed DE-gene / UMI thresholds (see config): RCTD's defaults find "fewer than +# 10 DE genes" on small iST panels with many fine-grained cell types. +myRCTD <- create.RCTD( + puck, reference, max_cores = cores, + gene_cutoff = par$gene_cutoff, fc_cutoff = par$fc_cutoff, + gene_cutoff_reg = par$gene_cutoff_reg, fc_cutoff_reg = par$fc_cutoff_reg, + UMI_min = par$umi_min, UMI_min_sigma = par$umi_min_sigma +) myRCTD <- run.RCTD(myRCTD, doublet_mode = "doublet") # Get the "spot_class" annotation from RCTD From 5abd651be071216aec6a39669e8c550ef139197a Mon Sep 17 00:00:00 2001 From: dariarom94 Date: Tue, 21 Jul 2026 22:59:19 +0200 Subject: [PATCH 19/60] segger to RAPIDS --- .../segger/config.vsh.yaml | 97 +++++++++---------- 1 file changed, 45 insertions(+), 52 deletions(-) diff --git a/src/methods_transcript_assignment/segger/config.vsh.yaml b/src/methods_transcript_assignment/segger/config.vsh.yaml index 8dd320557..fdd8872c1 100644 --- a/src/methods_transcript_assignment/segger/config.vsh.yaml +++ b/src/methods_transcript_assignment/segger/config.vsh.yaml @@ -45,69 +45,62 @@ engines: # (cudf, cuspatial, torch). Develop/test on a CUDA host; `viash test` # will not run on a machine without an NVIDIA GPU. - type: docker - # NGC 26.06 ships numpy 2.1 and a torch (2.13) compiled against numpy 2.x, so - # the torch<->numpy bridge (used by the `segger segment` subprocess) stays - # alive alongside the numpy-2.x task stack (spatialdata>=0.7.3 hard-requires - # numpy>=2 via multiscale-spatial-image -> xarray-dataclass). The older 25.06 - # image shipped numpy-1.x torch, which cannot coexist with spatialdata>=0.7.3. - image: nvcr.io/nvidia/pytorch:26.06-py3 + # segger is fundamentally a RAPIDS application (cudf + cuspatial + cugraph + + # cuml). Bolting that full stack onto the NGC PyTorch base made RAPIDS's UCX + # (cugraph -> raft_dask) collide with the image's HPC-X UCX and abort with heap + # corruption (SIGABRT, exit 134) that no pip pin could fix. So we base on the + # official RAPIDS image (ships cudf/cugraph/cuml/cuspatial 25.04 + a consistent, + # working UCX/dask stack) and layer torch (GNN) + the spatialdata/anndata I/O + # stack + segger on top. numpy 2.0.2 / pandas 2.2.3 come from the base. + # + # NOTE: this image runs as the NON-root 'rapids' user, so `type: apt` will NOT + # work in a viash build (needs root). Everything below installs via conda/pip + # only (both write to the rapids-owned conda env) — do NOT add apt steps. + image: rapidsai/base:25.04-cuda12.8-py3.12 setup: - # libgl1 + libglib2.0-0: segger's writer imports segger.io -> cv2 (opencv), - # which needs libGL.so.1 / libgthread at import; without them the run crashes - # at write time with "libGL.so.1: cannot open shared object file". - - type: apt - packages: [procps, git, libxcb1, libgl1, libglib2.0-0] + # git (for the segger + openproblems github installs) via conda: apt is + # unavailable as non-root and the RAPIDS base ships no git. + - type: docker + run: conda install -y -c conda-forge git - type: python github: - openproblems-bio/core#subdirectory=packages/python/openproblems - - type: python - packages: - - "spatialdata>=0.7.3" - - "anndata>=0.12.0" - - "zarr>=3.0.0" - - geopandas - - shapely - - "rasterio>=1.3" - - scikit-image - # cuspatial is imported by segger.geometry.conversion and is NOT shipped in - # the 26.06 image (unlike 25.06). Install from the NVIDIA index (its - # libcudf-cu12/libcuspatial-cu12 deps are not on PyPI); it pulls a matching - # cudf-cu12. Unpinned -> resolves to the newest cuda-12 build (25.4, the last - # cuda-12 cuspatial-cu12 on the index). - - type: docker - run: | - pip install --no-cache-dir --extra-index-url=https://pypi.nvidia.com cuspatial-cu12 - # segger imports torch_geometric (at import time via _patches), lightning - # (segger segment -> Trainer) and torch_scatter (models: scatter_max) but - # declares NONE of them in its pyproject, so the github install below does - # not pull them. torch_geometric/lightning are pure Python. torch_scatter has - # no prebuilt wheel for the NGC custom torch build, so compile against it - # (--no-build-isolation). The build host has no GPU, so force a CUDA build for - # the target archs (A100=8.0, A10/A40=8.6, L4/L40=8.9, H100=9.0). + # torch for the GNN. Standard PyPI cu124 wheel (numpy-2 compatible) coexists + # with the RAPIDS cuda-12 libs. Unlike the NGC custom torch, torch_scatter has + # a PREBUILT wheel for this torch (the pyg index) — no CUDA compile needed. - type: docker run: + - pip install --no-cache-dir torch==2.6.0 --index-url https://download.pytorch.org/whl/cu124 - pip install --no-cache-dir torch_geometric lightning - - FORCE_CUDA=1 TORCH_CUDA_ARCH_LIST="8.0;8.6;8.9;9.0" pip install --no-cache-dir --no-build-isolation torch_scatter - # Pin to a specific commit: segger's CLI flags AND output schema change on - # trunk (0.1.0 renamed --prediction-expansion-ratio -> --prediction-graph- - # buffer-ratio and dropped the `keep` column). script.py targets THIS commit's - # contract, so bump this deliberately and re-check `segger segment --help` - # and data/writer.py before doing so. + - pip install --no-cache-dir torch_scatter -f https://data.pyg.org/whl/torch-2.6.0+cu124.html + # I/O stack. spatialdata pinned <0.8 (resolves to 0.7.1): 0.7.2+ require + # dask>=2026.x, but RAPIDS 25.04 hard-pins dask==2025.2.0. 0.7.1 accepts dask + # 2025.2.0; pin dask/distributed so spatialdata can't drag them up (which would + # break RAPIDS's dask-cudf/cugraph). We validated sd.transform works on 2025.2.0. + - type: docker + run: + - pip install --no-cache-dir "spatialdata>=0.7,<0.8" "zarr>=3.0.0" geopandas shapely "rasterio>=1.3" scikit-image "dask==2025.2.0" "distributed==2025.2.0" + # Pin segger to a commit: its CLI flags AND output schema change on trunk + # (0.1.0 renamed --prediction-expansion-ratio -> --prediction-graph-buffer- + # ratio and dropped the `keep` column). script.py targets THIS commit; bump + # deliberately and re-check `segger segment --help` and data/writer.py first. - type: python github: dpeerlab/segger@0233cf62eb177b6e44e49318037705550765a010 - # cudf 25.4 (cuda-12; there is no cuspatial-cu13, so we are stuck on this - # RAPIDS) hard-requires pandas<2.2.4. But segger pulls the modern single-cell - # stack — anndata 0.13 and scanpy 1.12 — which BOTH require pandas>=2.3 - # (anndata 0.13's zarr reader calls pd.StringDtype(na_value=...), an API added - # in pandas 2.3), so under the pinned pandas 2.2.3 reading ANY AnnData/zarr - # table crashes with "StringDtype.__init__() got an unexpected keyword - # argument 'na_value'". Pin pandas back into cudf's range AND hold anndata / - # scanpy at their last pandas-2.2-compatible majors (anndata 0.12.x needs - # pandas>=2.1; scanpy 1.11.x needs pandas>=1.5) as the LAST step so they win - # over the deps above. numpy stays 2.x. See NOTES.md for the full conflict. + # Final step, must be LAST (after segger's github install pulls its deps): + # - cudf 25.4 needs pandas<2.2.4; but anndata 0.13 / scanpy 1.12 (pulled by + # segger) need pandas>=2.3 (anndata 0.13's zarr reader uses + # pd.StringDtype(na_value=...), a pandas-2.3 API) -> reading any zarr table + # crashes under pandas 2.2.3. Hold anndata/scanpy at their last pandas-2.2 + # majors (anndata 0.12.x: pandas>=2.1; scanpy 1.11.x: pandas>=1.5), and + # re-assert spatialdata<0.8 + dask==2025.2.0 in case a dep nudged them. + # - swap opencv-python -> opencv-python-headless: segger pulls opencv-python, + # whose cv2 needs libGL.so.1 (a system lib we cannot apt-install as non-root); + # the headless build provides the same cv2 without it. See NOTES.md. - type: docker run: - - pip install --no-cache-dir "pandas>=2.0,<2.2.4" "anndata>=0.12,<0.13" "scanpy>=1.11,<1.12" + - pip install --no-cache-dir "pandas>=2.0,<2.2.4" "anndata>=0.12,<0.13" "scanpy>=1.11,<1.12" "spatialdata>=0.7,<0.8" "dask==2025.2.0" "distributed==2025.2.0" + - pip uninstall -y opencv-python || true + - pip install --no-cache-dir opencv-python-headless - type: native runners: From 0b23474e7c81dfd9b386b5114e3ecdd8bdff77e8 Mon Sep 17 00:00:00 2001 From: dariarom94 Date: Wed, 22 Jul 2026 09:02:59 +0200 Subject: [PATCH 20/60] fix rctd --- src/base/labels_nebius.config | 6 +++++- src/methods_cell_type_annotation/rctd/script.R | 13 ++++++++++++- src/methods_expression_correction/split/script.R | 13 ++++++++++++- 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/src/base/labels_nebius.config b/src/base/labels_nebius.config index bfe4bf2a3..7dd1ffcbe 100644 --- a/src/base/labels_nebius.config +++ b/src/base/labels_nebius.config @@ -106,7 +106,11 @@ process { accelerator = 1 memory = 28.GB disk = 200.GB - pod = [[nodeSelector: 'nebius.com/node-group-id=mk8snodegroup-e00t775jb99svb7k5r'], [imagePullPolicy: 'Always']] + // runAsUser: 0 — segger is based on rapidsai/base, which runs as the non-root + // 'rapids' user (uid 1001); without this the container cannot write Nextflow's + // root-owned task scratch dir (.command.log / .command.begin / .exitcode -> + // "Permission denied"). Harmless for the other, already-root GPU images. + pod = [[nodeSelector: 'nebius.com/node-group-id=mk8snodegroup-e00t775jb99svb7k5r'], [imagePullPolicy: 'Always'], [runAsUser: 0]] containerOptions = { workflow.containerEngine == "singularity" ? '--nv': ( workflow.containerEngine == "docker" ? '--gpus all': null ) } } diff --git a/src/methods_cell_type_annotation/rctd/script.R b/src/methods_cell_type_annotation/rctd/script.R index 93a9cc345..371f1556e 100644 --- a/src/methods_cell_type_annotation/rctd/script.R +++ b/src/methods_cell_type_annotation/rctd/script.R @@ -37,8 +37,19 @@ puck <- SpatialRNA(coords, counts) # Read reference scrnaseq ref <- read_h5ad(par$input_scrnaseq_reference, as = "SingleCellExperiment") +# Drop reference cells with zero total counts. After the reference is subset to the +# shared spatial panel (~few hundred genes), some cells express NONE of those genes +# (nUMI == 0). RCTD's get_cell_type_info divides each cell by its nUMI, so a zero-UMI +# cell becomes NaN; a single NaN column then poisons other_mean (via rowMeans) for +# every cell type, making get_de_genes return 0 DE genes ("fewer than 10 ... DEGs"). +nonzero_cells <- Matrix::colSums(assay(ref, "counts")) > 0 +if (any(!nonzero_cells)) { + cat(sprintf("Dropping %d reference cells with zero panel-gene counts\n", sum(!nonzero_cells))) + ref <- ref[, nonzero_cells] +} + #filter reference cell types to those with >25 cells -valid_celltypes <- names(table(colData(ref)$cell_type))[table(colData(ref)$cell_type) >= 25] +valid_celltypes <- names(table(colData(ref)$cell_type))[table(colData(ref)$cell_type) >= 25] filtered_ref <- ref[,colData(ref)$cell_type %in% valid_celltypes] ref_counts <- assay(filtered_ref, "counts") diff --git a/src/methods_expression_correction/split/script.R b/src/methods_expression_correction/split/script.R index 7bcc06f00..671438567 100644 --- a/src/methods_expression_correction/split/script.R +++ b/src/methods_expression_correction/split/script.R @@ -48,8 +48,19 @@ puck <- SpatialRNA(coords, counts) # Read reference scrnaseq ref <- read_h5ad(par$input_scrnaseq_reference, as = "SingleCellExperiment") +# Drop reference cells with zero total counts. After the reference is subset to the +# shared spatial panel (~few hundred genes), some cells express NONE of those genes +# (nUMI == 0). RCTD's get_cell_type_info divides each cell by its nUMI, so a zero-UMI +# cell becomes NaN; a single NaN column then poisons other_mean (via rowMeans) for +# every cell type, making get_de_genes return 0 DE genes ("fewer than 10 ... DEGs"). +nonzero_cells <- Matrix::colSums(assay(ref, "counts")) > 0 +if (any(!nonzero_cells)) { + cat(sprintf("Dropping %d reference cells with zero panel-gene counts\n", sum(!nonzero_cells))) + ref <- ref[, nonzero_cells] +} + #filter reference cell types to those with >25 cells (minimum for RCTD) -valid_celltypes <- names(table(colData(ref)$cell_type))[table(colData(ref)$cell_type) >= 25] +valid_celltypes <- names(table(colData(ref)$cell_type))[table(colData(ref)$cell_type) >= 25] filtered_ref <- ref[,colData(ref)$cell_type %in% valid_celltypes] ref_counts <- assay(filtered_ref, "counts") From 196ff1fb3d61ccc320c59551bcb74b308d1a3318 Mon Sep 17 00:00:00 2001 From: dariarom94 Date: Wed, 22 Jul 2026 13:54:21 +0200 Subject: [PATCH 21/60] segger debug (torchvision) --- .../segger/config.vsh.yaml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/methods_transcript_assignment/segger/config.vsh.yaml b/src/methods_transcript_assignment/segger/config.vsh.yaml index fdd8872c1..377150533 100644 --- a/src/methods_transcript_assignment/segger/config.vsh.yaml +++ b/src/methods_transcript_assignment/segger/config.vsh.yaml @@ -65,13 +65,16 @@ engines: - type: python github: - openproblems-bio/core#subdirectory=packages/python/openproblems - # torch for the GNN. Standard PyPI cu124 wheel (numpy-2 compatible) coexists - # with the RAPIDS cuda-12 libs. Unlike the NGC custom torch, torch_scatter has - # a PREBUILT wheel for this torch (the pyg index) — no CUDA compile needed. + # torch (+ torchvision, which segger's data_module imports) for the GNN. + # Standard PyPI cu124 wheels (numpy-2 compatible) coexist with the RAPIDS + # cuda-12 libs. torchvision must match torch (2.6.0 <-> 0.21.0). Unlike the NGC + # custom torch, torch_scatter has a PREBUILT wheel for this torch (pyg index) — + # no CUDA compile needed. typer + torch_geometric + lightning are pure Python + # (segger imports all three but declares none of them). - type: docker run: - - pip install --no-cache-dir torch==2.6.0 --index-url https://download.pytorch.org/whl/cu124 - - pip install --no-cache-dir torch_geometric lightning + - pip install --no-cache-dir torch==2.6.0 torchvision==0.21.0 --index-url https://download.pytorch.org/whl/cu124 + - pip install --no-cache-dir torch_geometric lightning typer - pip install --no-cache-dir torch_scatter -f https://data.pyg.org/whl/torch-2.6.0+cu124.html # I/O stack. spatialdata pinned <0.8 (resolves to 0.7.1): 0.7.2+ require # dask>=2026.x, but RAPIDS 25.04 hard-pins dask==2025.2.0. 0.7.1 accepts dask From b8d3d7b0d38565a730bc77b742b983e4b84ded7d Mon Sep 17 00:00:00 2001 From: dariarom94 Date: Wed, 22 Jul 2026 20:15:23 +0200 Subject: [PATCH 22/60] save the xenium version --- src/datasets/loaders/tenx_atera/script.py | 13 ++++++++++++ src/datasets/loaders/tenx_xenium/script.py | 13 ++++++++++++ .../segger/script.py | 21 +++++++++++++++++++ 3 files changed, 47 insertions(+) diff --git a/src/datasets/loaders/tenx_atera/script.py b/src/datasets/loaders/tenx_atera/script.py index 6a2009ddf..9c7e5f32b 100644 --- a/src/datasets/loaders/tenx_atera/script.py +++ b/src/datasets/loaders/tenx_atera/script.py @@ -1,6 +1,7 @@ ## code author: Florian Heyl import shutil import os +import json import zipfile import tempfile from pathlib import Path @@ -140,6 +141,18 @@ def extract_zip(input_zip: Path, output_dir: Path, strip_root: bool = False): for key, value in new_uns.items(): sdata.tables["table"].uns[key] = value +# Preserve the Xenium analysis software version (from the raw experiment.xenium) so +# downstream components can recover it instead of hardcoding — e.g. segger reads it to +# select its v1 vs v2+ Xenium loader. Best-effort: skip if absent/unreadable. +try: + with open(input_extracted / "experiment.xenium") as f: + xenium_sw_version = json.load(f).get("analysis_sw_version") + if xenium_sw_version: + sdata.tables["table"].uns["xenium_analysis_sw_version"] = xenium_sw_version + print(f"Xenium analysis_sw_version: {xenium_sw_version}", flush=True) +except (OSError, ValueError) as e: + print(f"(no experiment.xenium analysis_sw_version: {e})", flush=True) + # Force a regular chunk grid so the written store is readable by spatialdata's # ome-zarr reader (see rechunk_uniform). Do NOT enable array.rectilinear_chunks: # that only permits writing the unreadable rectilinear grids we are avoiding. diff --git a/src/datasets/loaders/tenx_xenium/script.py b/src/datasets/loaders/tenx_xenium/script.py index b1a2d4b7e..3a1bdeebd 100644 --- a/src/datasets/loaders/tenx_xenium/script.py +++ b/src/datasets/loaders/tenx_xenium/script.py @@ -3,6 +3,7 @@ from spatialdata_io import xenium import shutil import os +import json import zipfile import tempfile @@ -97,6 +98,18 @@ for key, value in new_uns.items(): sdata.tables["table"].uns[key] = value + # Preserve the Xenium analysis software version (from the raw experiment.xenium) + # so downstream components can recover it instead of hardcoding — e.g. segger reads + # it to select its v1 vs v2+ Xenium loader. Best-effort: skip if absent/unreadable. + try: + with open(os.path.join(par_input, "experiment.xenium")) as f: + xenium_sw_version = json.load(f).get("analysis_sw_version") + if xenium_sw_version: + sdata.tables["table"].uns["xenium_analysis_sw_version"] = xenium_sw_version + print(f"Xenium analysis_sw_version: {xenium_sw_version}", flush=True) + except (OSError, ValueError) as e: + print(f"(no experiment.xenium analysis_sw_version: {e})", flush=True) + print(f"Output: {sdata}", flush=True) print(f"Writing to '{par['output']}'", flush=True) diff --git a/src/methods_transcript_assignment/segger/script.py b/src/methods_transcript_assignment/segger/script.py index 373aa7deb..7c97e66c3 100644 --- a/src/methods_transcript_assignment/segger/script.py +++ b/src/methods_transcript_assignment/segger/script.py @@ -1,4 +1,5 @@ import os +import json import shutil import subprocess from pathlib import Path @@ -154,6 +155,26 @@ tx_out.to_parquet(XENIUM_DIR / "transcripts.parquet", index=False) del transcripts, y_coords, x_coords, label_image +# segger 0.1.0 auto-detects the platform from marker files: its Xenium loader needs an +# `experiment.xenium` JSON and reads only `analysis_sw_version` from it, which selects its +# v1 ("-1") vs v2+ ("UNASSIGNED") loader. We always write the "UNASSIGNED" sentinel above, +# so we need the v2+ loader (version >= 2). Reuse the real version the Xenium dataset +# loader stashed in the metadata table's uns when it is v2+; otherwise (v1 source, missing, +# or a non-Xenium dataset wrapped here as Xenium-layout) fall back to a v2+ default. +DEFAULT_SW_VERSION = "xenium-3.0.0" +sw_version = DEFAULT_SW_VERSION +try: + src_version = sdata.tables["metadata"].uns.get("xenium_analysis_sw_version") + if src_version and int(str(src_version).split("-")[-1].split(".")[0]) >= 2: + sw_version = str(src_version) + elif src_version: + print(f"Source Xenium version '{src_version}' is <2, but we write the " + f"UNASSIGNED sentinel; using {sw_version} instead.", flush=True) +except (KeyError, AttributeError, ValueError, IndexError): + pass +print(f"Writing experiment.xenium with analysis_sw_version={sw_version}", flush=True) +(XENIUM_DIR / "experiment.xenium").write_text(json.dumps({"analysis_sw_version": sw_version})) + ################ # Run segger # ################ From 202ac494fde2fd3a940a85e25380c5edd4b2aef0 Mon Sep 17 00:00:00 2001 From: dariarom94 Date: Wed, 22 Jul 2026 20:17:01 +0200 Subject: [PATCH 23/60] add atera to datasets --- .../combine/process_datasets_xenium_nebius.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/scripts/create_resources/combine/process_datasets_xenium_nebius.sh b/scripts/create_resources/combine/process_datasets_xenium_nebius.sh index 43dad7f5e..a630053f1 100644 --- a/scripts/create_resources/combine/process_datasets_xenium_nebius.sh +++ b/scripts/create_resources/combine/process_datasets_xenium_nebius.sh @@ -129,6 +129,17 @@ param_list: dataset_description: "Xenium V1 FFPE Human Breast IDC + 2021 Wu scRNAseq" dataset_organism: "homo_sapiens" + - id: "2026_10x_human_breast_cancer_atera_combined" + input_sp: "$input_dir/10x_atera/2026_10x_human_breast_cancer_atera/dataset.zarr" + input_sc: "$input_dir/wu_human_breast_cancer_sc/2021Wu_human_breast_cancer_sc/dataset.h5ad" + dataset_id: "2026_10x_human_breast_cancer_atera_combined" + dataset_name: "Human breast cancer combined 2026 10x Atera WTA 2021 Wu scRNAseq" + dataset_url: "https://www.10xgenomics.com/datasets/atera-wta-ffpe-human-breast-cancer" + dataset_reference: "https://doi.org/10.1038/s41588-021-00911-1" + dataset_summary: "Atera WTA FFPE Human Breast Cancer (DCIS Grade 3) + 2021 Wu scRNAseq" + dataset_description: "Atera WTA FFPE Human Breast Cancer (DCIS Grade 3) + 2021 Wu scRNAseq" + dataset_organism: "homo_sapiens" + output_sc: "\$id/output_sc.h5ad" output_sp: "\$id/output_sp.zarr" output_state: "\$id/state.yaml" From 14be8d0a76977972e892dc419a4f5cd04fdab597 Mon Sep 17 00:00:00 2001 From: Daria Romanovskaia Date: Wed, 22 Jul 2026 21:20:20 +0200 Subject: [PATCH 24/60] Add gene efficiency correction as a separate pipeline stage (#183) Promote gene_efficiency_correction from an expression-correction method to its own stage (methods_gene_efficiency_correction) that runs after expression correction, offering two options: no_correction (pass-through) and gene_efficiency_correction. - New API src/api/comp_method_gene_efficiency_correction.yaml (input and output on file_spatial_corrected_counts). - New components under methods_gene_efficiency_correction/; the moved gene_efficiency_correction now reads --input and preserves an upstream normalized_uncorrected layer. - Remove gene_efficiency_correction from methods_expression_correction. - run_benchmark: insert the gene_eff stage between expression correction and aggregate_spatial_data; it overwrites the output_correction state key so aggregate_spatial_data and the similarity metric need no rewiring and controls keep working. Add --gene_efficiency_correction_methods and alias the colliding no_correction dependency (gene_eff_no_correction). - Update run scripts' method lists for the new stage. Verified: viash config view, viash ns build (components + workflow, deps and alias resolve), and viash test on both new components (output spec-conformant). Co-authored-by: Claude Opus 4.8 --- scripts/run_benchmark/run_full_local.sh | 4 +- scripts/run_benchmark/run_full_seqeracloud.sh | 4 +- scripts/run_benchmark/run_test_local.sh | 4 +- .../run_benchmark/run_test_segger_nebius.sh | 3 ++ scripts/run_benchmark/run_test_seqeracloud.sh | 4 +- ...omp_method_gene_efficiency_correction.yaml | 31 +++++++++++++ .../config.vsh.yaml | 6 +-- .../gene_efficiency_correction/script.py | 22 +++++---- .../no_correction/config.vsh.yaml | 31 +++++++++++++ .../no_correction/script.py | 19 ++++++++ src/workflows/run_benchmark/config.vsh.yaml | 12 ++++- src/workflows/run_benchmark/main.nf | 45 +++++++++++++++++-- 12 files changed, 163 insertions(+), 22 deletions(-) create mode 100644 src/api/comp_method_gene_efficiency_correction.yaml rename src/{methods_expression_correction => methods_gene_efficiency_correction}/gene_efficiency_correction/config.vsh.yaml (90%) rename src/{methods_expression_correction => methods_gene_efficiency_correction}/gene_efficiency_correction/script.py (64%) create mode 100644 src/methods_gene_efficiency_correction/no_correction/config.vsh.yaml create mode 100644 src/methods_gene_efficiency_correction/no_correction/script.py diff --git a/scripts/run_benchmark/run_full_local.sh b/scripts/run_benchmark/run_full_local.sh index b6fe94888..f591c201e 100755 --- a/scripts/run_benchmark/run_full_local.sh +++ b/scripts/run_benchmark/run_full_local.sh @@ -64,9 +64,11 @@ celltype_annotation_methods: # - rctd expression_correction_methods: - no_correction - # - gene_efficiency_correction # - resolvi_correction # - split +gene_efficiency_correction_methods: + - no_correction + # - gene_efficiency_correction method_parameters_yaml: /tmp/method_params.yaml HERE diff --git a/scripts/run_benchmark/run_full_seqeracloud.sh b/scripts/run_benchmark/run_full_seqeracloud.sh index ae3d5a3f3..a8da75c4a 100755 --- a/scripts/run_benchmark/run_full_seqeracloud.sh +++ b/scripts/run_benchmark/run_full_seqeracloud.sh @@ -56,9 +56,11 @@ celltype_annotation_methods: - rctd expression_correction_methods: - no_correction - - gene_efficiency_correction - resolvi_correction - split +gene_efficiency_correction_methods: + - no_correction + - gene_efficiency_correction method_parameters_yaml: /tmp/method_params.yaml HERE diff --git a/scripts/run_benchmark/run_test_local.sh b/scripts/run_benchmark/run_test_local.sh index b20058497..d51b98adc 100755 --- a/scripts/run_benchmark/run_test_local.sh +++ b/scripts/run_benchmark/run_test_local.sh @@ -59,9 +59,11 @@ celltype_annotation_methods: # - rctd expression_correction_methods: - no_correction - # - gene_efficiency_correction # - resolvi_correction # - split +gene_efficiency_correction_methods: + - no_correction + # - gene_efficiency_correction method_parameters_yaml: /tmp/method_params.yaml HERE diff --git a/scripts/run_benchmark/run_test_segger_nebius.sh b/scripts/run_benchmark/run_test_segger_nebius.sh index 04bc72e90..22d1916ba 100644 --- a/scripts/run_benchmark/run_test_segger_nebius.sh +++ b/scripts/run_benchmark/run_test_segger_nebius.sh @@ -44,6 +44,9 @@ celltype_annotation_methods: - tacco expression_correction_methods: - no_correction +gene_efficiency_correction_methods: + - no_correction + # - gene_efficiency_correction HERE # Write the parameters to file (input_states version, NOTE: enable `-entry_name auto` for this) diff --git a/scripts/run_benchmark/run_test_seqeracloud.sh b/scripts/run_benchmark/run_test_seqeracloud.sh index 1a3124961..caba7137d 100755 --- a/scripts/run_benchmark/run_test_seqeracloud.sh +++ b/scripts/run_benchmark/run_test_seqeracloud.sh @@ -55,9 +55,11 @@ celltype_annotation_methods: - rctd expression_correction_methods: - no_correction - - gene_efficiency_correction - resolvi_correction - split +gene_efficiency_correction_methods: + - no_correction + - gene_efficiency_correction #method_parameters_yaml: /tmp/method_params.yaml HERE diff --git a/src/api/comp_method_gene_efficiency_correction.yaml b/src/api/comp_method_gene_efficiency_correction.yaml new file mode 100644 index 000000000..824587a29 --- /dev/null +++ b/src/api/comp_method_gene_efficiency_correction.yaml @@ -0,0 +1,31 @@ +namespace: methods_gene_efficiency_correction +info: + type: method + subtype: method_gene_efficiency_correction + type_info: + label: Gene efficiency correction + summary: Correcting per-gene detection efficiency in spatial data + description: >- + A gene efficiency correction method corrects for per-gene detection efficiency in the + (expression-)corrected spatial counts. It runs as a separate stage after expression + correction, consuming and producing the corrected-counts format. +arguments: + - name: --input + required: true + direction: input + __merge__: /src/api/file_spatial_corrected_counts.yaml + - name: --input_scrnaseq_reference + required: false + direction: input + __merge__: /src/api/file_scrnaseq_reference.yaml + - name: --output + required: true + direction: output + __merge__: /src/api/file_spatial_corrected_counts.yaml +test_resources: + - path: /resources_test/task_ist_preprocessing/mouse_brain_combined + dest: resources_test/task_ist_preprocessing/mouse_brain_combined + - type: python_script + path: /common/component_tests/run_and_check_output.py + - type: python_script + path: /common/component_tests/check_config.py diff --git a/src/methods_expression_correction/gene_efficiency_correction/config.vsh.yaml b/src/methods_gene_efficiency_correction/gene_efficiency_correction/config.vsh.yaml similarity index 90% rename from src/methods_expression_correction/gene_efficiency_correction/config.vsh.yaml rename to src/methods_gene_efficiency_correction/gene_efficiency_correction/config.vsh.yaml index 1d813fbdb..6f9ea9013 100644 --- a/src/methods_expression_correction/gene_efficiency_correction/config.vsh.yaml +++ b/src/methods_gene_efficiency_correction/gene_efficiency_correction/config.vsh.yaml @@ -1,4 +1,4 @@ -__merge__: /src/api/comp_method_expression_correction.yaml +__merge__: /src/api/comp_method_gene_efficiency_correction.yaml name: gene_efficiency_correction label: "Gene Efficiency Correction" @@ -27,7 +27,7 @@ resources: engines: - type: docker image: openproblems/base_python:1 - __merge__: + __merge__: - /src/base/setup_txsim_partial.yaml setup: - type: python @@ -38,4 +38,4 @@ runners: - type: executable - type: nextflow directives: - label: [ midtime, lowcpu, midmem ] \ No newline at end of file + label: [ midtime, lowcpu, midmem ] diff --git a/src/methods_expression_correction/gene_efficiency_correction/script.py b/src/methods_gene_efficiency_correction/gene_efficiency_correction/script.py similarity index 64% rename from src/methods_expression_correction/gene_efficiency_correction/script.py rename to src/methods_gene_efficiency_correction/gene_efficiency_correction/script.py index b2dd82266..c9a3c027d 100644 --- a/src/methods_expression_correction/gene_efficiency_correction/script.py +++ b/src/methods_gene_efficiency_correction/gene_efficiency_correction/script.py @@ -5,29 +5,33 @@ # Note: this section is auto-generated by viash at runtime. To edit it, make changes # in config.vsh.yaml and then run `viash config inject config.vsh.yaml`. par = { - 'input_spatial_with_cell_types': 'resources_test/task_ist_preprocessing/mouse_brain_combined/spatial_with_celltypes.h5ad', + 'input': 'resources_test/task_ist_preprocessing/mouse_brain_combined/spatial_corrected.h5ad', 'input_scrnaseq_reference': 'resources_test/task_ist_preprocessing/mouse_brain_combined/scrnaseq_reference.h5ad', 'celltype_key': 'cell_type', - 'output': 'spatial_corrected.h5ad', + 'output': 'spatial_gene_efficiency_corrected.h5ad', } meta = { 'name': 'gene_efficiency_correction', } ## VIASH END -# Optional parameter check: For this specific correction method the par['input_sc'] is required -assert par['input_scrnaseq_reference'] is not None, 'Single cell input is required for this expr correction method.' - +# Optional parameter check: this correction method requires the scRNA-seq reference +assert par['input_scrnaseq_reference'] is not None, 'Single cell input is required for this gene efficiency correction method.' + # Read input print('Reading input files', flush=True) -adata_sp = ad.read_h5ad(par['input_spatial_with_cell_types']) +adata_sp = ad.read_h5ad(par['input']) adata_sc = ad.read_h5ad(par['input_scrnaseq_reference']) -adata_sp.layers["normalized_uncorrected"] = adata_sp.layers["normalized"] +# Preserve the pre-correction normalized layer. An upstream expression-correction stage +# already sets it, so only initialise it here if it is missing (keeps it meaning "before +# any correction"). +if "normalized_uncorrected" not in adata_sp.layers: + adata_sp.layers["normalized_uncorrected"] = adata_sp.layers["normalized"] adata_sp_reduced = adata_sp[:,adata_sc.var_names].copy() obs_sp = adata_sp.obs.copy() # Apply gene efficiency correction -print('Annotating cell types', flush=True) +print('Applying gene efficiency correction', flush=True) adata_sp_reduced = tx.preprocessing.gene_efficiency_correction( adata_sp_reduced, adata_sc, layer_key='normalized', ct_key=par['celltype_key'] ) @@ -36,7 +40,7 @@ print('Concatenating and reordering', flush=True) gene_order = adata_sp.var_names.tolist() gene_mask = ~adata_sp.var_names.isin(adata_sp_reduced.var_names) -adata_sp = ad.concat([adata_sp[:,gene_mask], adata_sp_reduced], axis=1, join="outer") +adata_sp = ad.concat([adata_sp[:,gene_mask], adata_sp_reduced], axis=1, join="outer") adata_sp = adata_sp[:,gene_order] adata_sp.obs = obs_sp diff --git a/src/methods_gene_efficiency_correction/no_correction/config.vsh.yaml b/src/methods_gene_efficiency_correction/no_correction/config.vsh.yaml new file mode 100644 index 000000000..a211477cb --- /dev/null +++ b/src/methods_gene_efficiency_correction/no_correction/config.vsh.yaml @@ -0,0 +1,31 @@ +__merge__: /src/api/comp_method_gene_efficiency_correction.yaml + +name: no_correction +label: "No Correction" +summary: "Dummy method that does not apply gene efficiency correction" +description: >- + Dummy method that does not apply gene efficiency correction; passes the (expression-)corrected + input through unchanged. +links: + documentation: "https://github.com/openproblems-bio/task_ist_preprocessing" + repository: "https://github.com/openproblems-bio/task_ist_preprocessing" +references: + doi: "10.1101/2023.02.13.528102" + +resources: + - type: python_script + path: script.py + +engines: + - type: docker + image: openproblems/base_python:1 + setup: + - type: python + pypi: [anndata] + - type: native + +runners: + - type: executable + - type: nextflow + directives: + label: [ midtime, lowcpu, lowmem ] diff --git a/src/methods_gene_efficiency_correction/no_correction/script.py b/src/methods_gene_efficiency_correction/no_correction/script.py new file mode 100644 index 000000000..45715c353 --- /dev/null +++ b/src/methods_gene_efficiency_correction/no_correction/script.py @@ -0,0 +1,19 @@ +import anndata as ad + +## VIASH START +par = { + 'input': 'resources_test/task_ist_preprocessing/mouse_brain_combined/spatial_corrected.h5ad', + 'output': 'spatial_gene_efficiency_corrected.h5ad', +} +## VIASH END + +# Read input +print('Reading input files', flush=True) +adata_sp = ad.read_h5ad(par['input']) +# Preserve the pre-correction normalized layer only if an upstream stage hasn't set it. +if "normalized_uncorrected" not in adata_sp.layers: + adata_sp.layers["normalized_uncorrected"] = adata_sp.layers["normalized"] + +# Write output +print('Writing output', flush=True) +adata_sp.write(par['output']) diff --git a/src/workflows/run_benchmark/config.vsh.yaml b/src/workflows/run_benchmark/config.vsh.yaml index d0a478f06..a61489ffb 100644 --- a/src/workflows/run_benchmark/config.vsh.yaml +++ b/src/workflows/run_benchmark/config.vsh.yaml @@ -104,7 +104,13 @@ argument_groups: A list of expression correction methods to run. type: string multiple: true - default: "no_correction:gene_efficiency_correction:resolvi_correction:split" + default: "no_correction:resolvi_correction:split" + - name: "--gene_efficiency_correction_methods" + description: | + A list of gene efficiency correction methods to run. + type: string + multiple: true + default: "no_correction:gene_efficiency_correction" - name: Method parameters description: | Use these arguments to control the parameter sets that are run for each @@ -175,9 +181,11 @@ dependencies: - name: methods_cell_type_annotation/singler - name: methods_cell_type_annotation/rctd - name: methods_expression_correction/no_correction - - name: methods_expression_correction/gene_efficiency_correction - name: methods_expression_correction/resolvi_correction - name: methods_expression_correction/split + - name: methods_gene_efficiency_correction/no_correction + alias: gene_eff_no_correction + - name: methods_gene_efficiency_correction/gene_efficiency_correction - name: methods_data_aggregation/aggregate_spatial_data - name: metrics/similarity - name: metrics/quality diff --git a/src/workflows/run_benchmark/main.nf b/src/workflows/run_benchmark/main.nf index f5cbd27c2..f1725397b 100644 --- a/src/workflows/run_benchmark/main.nf +++ b/src/workflows/run_benchmark/main.nf @@ -415,11 +415,10 @@ workflow run_wf { ****************************************/ expr_corr_methods = [ no_correction, - gene_efficiency_correction, resolvi_correction, split ] - + expr_corr_ch = cta_ch | expandChannelWithParameterSets(expr_corr_methods, "corr", "expression_correction_methods") | runEach( @@ -445,6 +444,44 @@ workflow run_wf { ] } ) + + + /**************************************** + * GENE EFFICIENCY CORRECTION * + ****************************************/ + // Runs after expression correction on its corrected counts. The output overwrites the + // `output_correction` state key, so `aggregate_spatial_data` and the similarity metric + // (which both read `output_correction`) consume the final corrected counts unchanged. + gene_eff_methods = [ + gene_eff_no_correction, + gene_efficiency_correction + ] + + gene_eff_ch = expr_corr_ch + | expandChannelWithParameterSets(gene_eff_methods, "gene_eff", "gene_efficiency_correction_methods") + | runEach( + components: gene_eff_methods, + filter: { id, state, comp -> + comp.config.name == state.current_method_id + }, + fromState: { id, state, comp -> + [ + input: state.output_correction, + input_scrnaseq_reference: state.input_sc + ] + state.current_method_args + }, + toState: { id, out_dict, state, comp -> + removeKeys(state, ["current_method_id", "current_method_variant", "current_method_args"]) + [ + steps: state.steps + [[ + type: "gene_efficiency_correction", + component_id: state.current_method_id, + component_variant: state.current_method_variant, + run_id: id + ]], + output_correction: out_dict.output + ] + } + ) // aggregate spatial data for quality metrics | aggregate_spatial_data.run( fromState: [ @@ -469,7 +506,7 @@ workflow run_wf { * COMBINE WITH CONTROL * ****************************************/ - expr_corr_and_control_ch = expr_corr_ch.mix(control_ch) + expr_corr_and_control_ch = gene_eff_ch.mix(control_ch) /**************************************** @@ -586,7 +623,7 @@ workflow run_wf { def methods = segm_methods + segm_ass_methods + direct_ass_methods + count_aggr_methods + qc_filter_methods + cell_vol_methods + vol_norm_methods + direct_norm_methods + - cta_methods + expr_corr_methods + cta_methods + expr_corr_methods + gene_eff_methods def method_configs = methods.collect{it.config} def method_configs_yaml_blob = toYamlBlob(method_configs) def method_configs_file = tempFile("method_configs.yaml") From 0cf02434695e65cd127bfe2d7c0c8605bed13c83 Mon Sep 17 00:00:00 2001 From: dariarom94 Date: Wed, 22 Jul 2026 21:50:38 +0200 Subject: [PATCH 25/60] moscot to pca and segger troubleshooting --- src/methods_cell_type_annotation/moscot/script.py | 4 +++- src/methods_transcript_assignment/segger/script.py | 8 ++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/methods_cell_type_annotation/moscot/script.py b/src/methods_cell_type_annotation/moscot/script.py index d69e21e45..ac0b7343b 100644 --- a/src/methods_cell_type_annotation/moscot/script.py +++ b/src/methods_cell_type_annotation/moscot/script.py @@ -75,11 +75,13 @@ adata_sp.X = adata_sp.layers["normalized"] adata_sp.obsm["spatial"] = adata_sp.obs[["centroid_x", "centroid_y"]].to_numpy() +sc.pp.pca(adata_sc, n_comps=50) # X is the normalized layer set above + # Define mapping problem mp = MappingProblem(adata_sc=adata_sc, adata_sp=adata_sp) mp = mp.prepare( - sc_attr={"attr": "layers", "key": "normalized"}, + sc_attr={"attr": "obsm", "key": "X_pca"}, # <-- 50-dim, not raw genes xy_callback="local-pca", ) diff --git a/src/methods_transcript_assignment/segger/script.py b/src/methods_transcript_assignment/segger/script.py index 7c97e66c3..fe1c32ece 100644 --- a/src/methods_transcript_assignment/segger/script.py +++ b/src/methods_transcript_assignment/segger/script.py @@ -79,11 +79,15 @@ trans = sd.transformations.Sequence([trans_segm_to_global, trans_global_to_transcripts]) boundaries = sd.transform(boundaries, trans, par['coordinate_system']) -# Xenium boundary schema: one row per polygon vertex (cell_id int64, vertex_x/y float64). +# Xenium boundary schema: one row per polygon vertex (vertex_x/y float64). cell_id is cast +# to STRING (via int64, so labels render as clean "1"/"2") to match the transcripts.parquet +# cell_id written below: segger joins transcript cell_id -> boundary cell_id to map each cell +# to its polygon, and a str-vs-int mismatch makes every row miss -> all-nan entity_index -> +# "Index has duplicate keys: [nan]" in setup_anndata. Real Xenium v2+ uses string ids in both. boundaries.index.name = "cell_id" boundaries_df = boundaries['geometry'].get_coordinates().rename(columns={'x': 'vertex_x', 'y': 'vertex_y'}) boundaries_df = boundaries_df.reset_index() -boundaries_df['cell_id'] = boundaries_df['cell_id'].astype(np.int64) +boundaries_df['cell_id'] = boundaries_df['cell_id'].astype(np.int64).astype(str) boundaries_df['vertex_x'] = boundaries_df['vertex_x'].astype(np.float64) boundaries_df['vertex_y'] = boundaries_df['vertex_y'].astype(np.float64) boundaries_df = boundaries_df[['cell_id', 'vertex_x', 'vertex_y']] From d7afb84b5cfdc3f644f86c6a39732f82fa16edcc Mon Sep 17 00:00:00 2001 From: dariarom94 Date: Thu, 23 Jul 2026 14:59:08 +0200 Subject: [PATCH 26/60] added fastreseg --- .../fastreseg/config.vsh.yaml | 78 ++++++++++++ .../fastreseg/input.py | 115 ++++++++++++++++++ .../fastreseg/orchestrator.sh | 69 +++++++++++ .../fastreseg/output.py | 58 +++++++++ .../fastreseg/script.R | 91 ++++++++++++++ 5 files changed, 411 insertions(+) create mode 100644 src/methods_transcript_assignment/fastreseg/config.vsh.yaml create mode 100644 src/methods_transcript_assignment/fastreseg/input.py create mode 100644 src/methods_transcript_assignment/fastreseg/orchestrator.sh create mode 100644 src/methods_transcript_assignment/fastreseg/output.py create mode 100644 src/methods_transcript_assignment/fastreseg/script.R diff --git a/src/methods_transcript_assignment/fastreseg/config.vsh.yaml b/src/methods_transcript_assignment/fastreseg/config.vsh.yaml new file mode 100644 index 000000000..512828392 --- /dev/null +++ b/src/methods_transcript_assignment/fastreseg/config.vsh.yaml @@ -0,0 +1,78 @@ +__merge__: /src/api/comp_method_transcript_assignment.yaml + +name: fastreseg +label: "fastReseg transcript assignment" +summary: "Spatial segmentation correction using fastReseg method" +description: | + fastReseg is an R package designed to enhance the precision of cell segmentation in spatial transcriptomics by + leveraging transcriptomic data to correct and refine initial image-based segmentation results. +links: + documentation: "https://github.com/openproblems-bio/task_ist_preprocessing" + repository: "https://github.com/openproblems-bio/task_ist_preprocessing" +references: + doi: "10.1038/s41598-025-08733-5" + + +#arguments: +# - name: --transcripts_key +# type: string +# default: "transcripts" +# description: "Key for transcripts in the points layer" +# - name: --coordinate_system +# type: string +# default: "global" +# description: "Coordinate system for the transcripts" + + +resources: + - type: bash_script + path: orchestrator.sh + - type: python_script + path: input.py + - type: r_script + path: script.R + - type: python_script + path: output.py + +engines: + - type: docker + image: openproblems/base_python:1 + setup: + - type: apt + packages: + - libv8-dev + - libudunits2-dev + - libabsl-dev + - gdal-bin + - libgdal-dev + - r-base + - type: r + packages: + - devtools + - terra + - Matrix + - dbscan + - igraph + - matrixStats + - codetools + - data.table + # Add other CRAN packages here + github: + # Add GitHub packages here if needed + - Nanostring-Biostats/FastReseg + #- type: python + # github: + # - openproblems-bio/core#subdirectory=packages/python/openproblems + __merge__: + - /src/base/setup_txsim_partial.yaml + - /src/base/setup_spatialdata_partial.yaml + - type: native + +runners: + - type: executable + - type: nextflow + directives: + label: [ midtime, midcpu, midmem ] + + +##macOS workaround: export DOCKER_DEFAULT_PLATFORM=linux/amd64 \ No newline at end of file diff --git a/src/methods_transcript_assignment/fastreseg/input.py b/src/methods_transcript_assignment/fastreseg/input.py new file mode 100644 index 000000000..1da6f5a48 --- /dev/null +++ b/src/methods_transcript_assignment/fastreseg/input.py @@ -0,0 +1,115 @@ +import spatialdata as sd +import sys +import numpy as np +import pandas as pd +import xarray as xr +import dask +import txsim as tx +import anndata as ad +import argparse + +def parse_arguments(): + parser = argparse.ArgumentParser( + description="Process input files and generate output files for FastReseg input" + ) + parser.add_argument('input_path_ist', help='Path to the input file') + parser.add_argument('input_segmentation_path', help='Path to the input segmentation file') + parser.add_argument('input_sc_reference_path', help='Path to the input single-cell reference file') + parser.add_argument('output_path_counts', help='Path for the output TSV file') + parser.add_argument('output_path_transcripts', help='Path for the output TIF file') + parser.add_argument('output_path_cell_type', help='Output cell type specification') + + return parser.parse_args() + +### parsing arguments +args = parse_arguments() +print("args:") +print(args) +input_path = args.input_path_ist +input_segmentation_path = args.input_segmentation_path +input_sc_reference_path = args.input_sc_reference_path +print("path") +print(input_sc_reference_path) +output_path_counts = args.output_path_counts +output_path_transcripts = args.output_path_transcripts +output_path_cell_type = args.output_path_cell_type + +## potential other parameters (TODO - make configurable) +um_per_pixel = 0.5 +sc_celltype_key = 'cell_type' + +### reading the data in +sdata = sd.read_zarr(input_path) + +### reading in basic segmentation +sdata_segm = sd.read_zarr(input_segmentation_path) +segmentation_coord_systems = sd.transformations.get_transformation(sdata_segm["segmentation"], get_all=True).keys() + +# In case of a translation transformation of the segmentation (e.g. crop of the data), we need to adjust the transcript coordinates +trans = sd.transformations.get_transformation(sdata_segm["segmentation"], get_all=True)['global'].inverse() + +transcripts = sd.transform(sdata['transcripts'], to_coordinate_system='global') +transcripts = sd.transform(transcripts, trans, 'global') + +print('Assigning transcripts to cell ids', flush=True) +y_coords = transcripts.y.compute().to_numpy(dtype=np.int64) +x_coords = transcripts.x.compute().to_numpy(dtype=np.int64) +if isinstance(sdata_segm["segmentation"], xr.DataTree): + label_image = sdata_segm["segmentation"]["scale0"].image.to_numpy() +else: + label_image = sdata_segm["segmentation"].to_numpy() +cell_id_dask_series = dask.dataframe.from_dask_array( + dask.array.from_array( + label_image[y_coords, x_coords], chunks=tuple(sdata['transcripts'].map_partitions(len).compute()) + ), + index=sdata['transcripts'].index +) +sdata['transcripts']["cell_id"] = cell_id_dask_series + +### extracting transcript ids +print('Transforming transcripts coordinates', flush=True) +transcripts = sd.transform(sdata['transcripts'], to_coordinate_system='global') + +transcripts_df = transcripts.compute() +transcripts_df.rename(columns = {'feature_name': 'target', +'transcript_id': 'UMI_transID', 'cell_id': 'UMI_cellID'}, inplace = True) + +transcripts_df = transcripts_df.loc[:, ['target', 'x', 'y', 'z', 'UMI_transID', 'UMI_cellID']] +transcripts_df.to_csv(output_path_transcripts) + + +#### aggregating counts per transcript, based on +df = sdata['transcripts'].compute() +df.feature_name = df.feature_name.astype(str) + +adata_sp = tx.preprocessing.generate_adata(df, cell_id_col='cell_id', gene_col='feature_name') #TODO: x and y refers to a specific coordinate system. Decide which space we want to use here. (probably should be handled in the previous assignment step) +adata_sp.layers['counts'] = adata_sp.layers['raw_counts'] +del adata_sp.layers['raw_counts'] +adata_sp.var["gene_name"] = adata_sp.var_names +print(adata_sp.var_names[1:10]) + +# currently the function also saves the transcripts in the adata object, but this is not necessary here +del adata_sp.uns['spots'] +del adata_sp.uns['pct_noise'] + + +count_df = pd.DataFrame(adata_sp.X.toarray(), + index=adata_sp.obs_names, + columns=adata_sp.var_names) +count_df.to_csv(output_path_counts) + +#### run cell annotation with ssam +adata_sc = ad.read_h5ad(input_sc_reference_path) +adata_sc.X = adata_sc.layers["normalized"] +print(adata_sc.var_names[1:10]) + +shared_genes = [g for g in adata_sc.var_names if g in adata_sp.var_names] +adata_sp = adata_sp[:,shared_genes] + +print('Annotating cell types', flush=True) +adata_sp = tx.preprocessing.run_ssam( + adata_sp, transcripts.compute(), adata_sc, um_p_px=um_per_pixel, + cell_id_col='cell_id', gene_col='feature_name', sc_ct_key=sc_celltype_key +) +cell_type_df = adata_sp.obs["ct_ssam"].astype(str) +cell_type_df.to_csv(output_path_cell_type, header=True) \ No newline at end of file diff --git a/src/methods_transcript_assignment/fastreseg/orchestrator.sh b/src/methods_transcript_assignment/fastreseg/orchestrator.sh new file mode 100644 index 000000000..9529d5b51 --- /dev/null +++ b/src/methods_transcript_assignment/fastreseg/orchestrator.sh @@ -0,0 +1,69 @@ +#!/bin/bash + +## VIASH START +# The following code has been auto-generated by Viash. +par_input_ist='resources_test/task_ist_preprocessing/mouse_brain_combined/raw_ist.zarr' +par_input_segmentation='resources_test/task_ist_preprocessing/mouse_brain_combined/segmentation.zarr' +par_input_scrnaseq='resources_test/task_ist_preprocessing/mouse_brain_combined/scrnaseq_reference.h5ad' +par_sc_cell_type_key='cell_type' +par_output='resources_test/task_ist_preprocessing/mouse_brain_combined/transcript_assignments.zarr' +par_transcripts_key='transcripts' +par_coordinate_system='global' +meta_name='fastreseg' +meta_functionality_name='fastreseg' +meta_resources_dir='/private/tmp/viash_inject_fastreseg18170326436127140412' +meta_executable='/private/tmp/viash_inject_fastreseg18170326436127140412/fastreseg' +meta_config='/private/tmp/viash_inject_fastreseg18170326436127140412/.config.vsh.yaml' +meta_temp_dir='/var/folders/fq/ymt0vml175s4yvqxzbmlmpz80000gn/T/' +meta_cpus='123' +meta_memory_b='123' +meta_memory_kb='123' +meta_memory_mb='123' +meta_memory_gb='123' +meta_memory_tb='123' +meta_memory_pb='123' +meta_memory_kib='123' +meta_memory_mib='123' +meta_memory_gib='123' +meta_memory_tib='123' +meta_memory_pib='123' + +## VIASH END + +par_intermediate_dir=$(mktemp -d -p "$(pwd)" tmp-processing-XXXXXXXX) + +echo "running FastReseg orchestrator" + +# Create intermediate directory +mkdir -p "$par_intermediate_dir" +echo $(date +%T) + +# Step 1: Run Python script to reformat input in the first Python environment +python "$meta_resources_dir/input.py" \ + "$par_input_ist" \ + "$par_input_segmentation" \ + "$par_input_scrnaseq" \ + "$par_intermediate_dir/counts.tsv" \ + "$par_intermediate_dir/transcripts.tsv" \ + "$par_intermediate_dir/cell_types.tsv" + +head $par_intermediate_dir/cell_types.tsv + +# Step 2: RunFastReseg + +##running the R script +Rscript "$meta_resources_dir/script.R" "$par_intermediate_dir/counts.tsv" \ + "$par_intermediate_dir/transcripts.tsv" \ + "$par_intermediate_dir/cell_types.tsv" \ + "$par_intermediate_dir/cell_ids.csv" \ + "$par_intermediate_dir/gene_names.csv" \ + "$par_intermediate_dir/transcripts_out.csv" + +## python output +python "$meta_resources_dir/output.py" \ + "$par_intermediate_dir/cell_ids.csv" \ + "$par_intermediate_dir/gene_names.csv" \ + "$par_intermediate_dir/transcripts_out.csv" \ + "$par_output" + +echo $(date +%T) \ No newline at end of file diff --git a/src/methods_transcript_assignment/fastreseg/output.py b/src/methods_transcript_assignment/fastreseg/output.py new file mode 100644 index 000000000..7849d351e --- /dev/null +++ b/src/methods_transcript_assignment/fastreseg/output.py @@ -0,0 +1,58 @@ +import spatialdata as sd +import sys +import numpy as np +import pandas as pd +import xarray as xr +from scipy.sparse import coo_matrix +from pathlib import Path +import os +import dask +import anndata as ad + +def convert_to_lower_dtype(arr): + max_val = arr.max() + if max_val <= np.iinfo(np.uint8).max: + new_dtype = np.uint8 + elif max_val <= np.iinfo(np.uint16).max: + new_dtype = np.uint16 + elif max_val <= np.iinfo(np.uint32).max: + new_dtype = np.uint32 + else: + new_dtype = np.uint64 + + return arr.astype(new_dtype) + +path_to_cell_ids_out = sys.argv[1] +path_to_gene_names_out = sys.argv[2] +path_to_transcripts_out = sys.argv[3] + +output_zarr_path = sys.argv[4] + +cell_df = pd.read_csv(path_to_cell_ids_out, index_col=0) +var_names = pd.read_csv(path_to_gene_names_out, index_col=0) +df = pd.read_csv(path_to_transcripts_out, index_col=0) +print(df.head()) + +##converting to dask transcript output for spatialdata format +df = df.loc[:,["x", "y", "z", "target", "updated_cellID", "updated_celltype", "UMI_transID"]] +df.rename(columns={"updated_cellID": "cell_id", "target": "feature_name", "UMI_transID": "transcript_id"}, inplace=True) +transcripts_dask = dask.dataframe.from_pandas(df, npartitions = 1) + + +sdata_transcripts_only = sd.SpatialData( + points={ + "transcripts": sd.models.PointsModel.parse(transcripts_dask) + }, + tables={ + "table": ad.AnnData( + # The transcript-assignment output format requires a `cell_id` column in + # the table's obs (see src/api/file_transcript_assignments.yaml); expose + # fastReseg's updated segmentation under the standard column names. + obs=cell_df.loc[:,['updated_cellID', 'updated_celltype']].rename( + columns={'updated_cellID': 'cell_id', 'updated_celltype': 'cell_type'} + ), + var=pd.DataFrame(index=var_names['x']) + ) + } +) +sdata_transcripts_only.write(output_zarr_path) \ No newline at end of file diff --git a/src/methods_transcript_assignment/fastreseg/script.R b/src/methods_transcript_assignment/fastreseg/script.R new file mode 100644 index 000000000..f1e67810e --- /dev/null +++ b/src/methods_transcript_assignment/fastreseg/script.R @@ -0,0 +1,91 @@ +#install.packages("devtools") +#devtools::install_github("Nanostring-Biostats/FastReseg", build_vignettes = FALSE, ref = "main") + +library(FastReseg) +library(data.table) +library(dplyr) + +args <- commandArgs(trailingOnly = TRUE) + +# Check if arguments are provided +if (length(args) == 0) { + stop("No arguments provided") +} + +# Access individual arguments +path_to_counts <- args[1] +path_to_transcripts <- args[2] +path_to_cell_annot <- args[3] + +path_to_cell_ids_out <- args[4] +path_to_gene_names_out <- args[5] +path_to_transcripts_out <- args[6] + + +### reading in data +count_df <- as.matrix(read.csv(path_to_counts, row.names = 1)) +head(count_df) + +cell_df <- read.csv(path_to_cell_annot, row.names = 1) +cell_annot <- cell_df[row.names(count_df),] +cell_annot <- setNames(cell_annot, row.names(count_df)) + +transcriptDF <- read.csv(path_to_transcripts) +head(transcriptDF) + + +##run pipeline +refineAll_res_one_FOC <- fastReseg_full_pipeline( + counts = count_df, + clust = cell_annot, + + # Similar to `runPreprocess()`, one can use `clust = NULL` if providing `refProfiles` + + transcript_df = transcriptDF, # + transDF_fileInfo = NULL, + pixel_size = 0.18, + zstep_size = 0.8, + transID_coln = NULL, + transGene_coln = "target", + cellID_coln = "UMI_cellID", + spatLocs_colns = c("x","y","z"), + extracellular_cellID = c(-1), + + # Similar to `runPreprocess()`, one can set various cutoffs to NULL for automatic calculation from input data + + # distance cutoff for neighborhood searching at molecular and cellular levels, respectively + molecular_distance_cutoff = 2.7, + cellular_distance_cutoff = NULL, + + # cutoffs for transcript scores and number for cells under each cell type + score_baseline = NULL, + lowerCutoff_transNum = NULL, + higherCutoff_transNum= NULL, + imputeFlag_missingCTs = TRUE, + + # Settings for error detection and correction, refer to `runSegRefinement()` for more details + flagCell_lrtest_cutoff = 5, # cutoff to flag for cells with strong spatial dependcy in transcript score profiles + svmClass_score_cutoff = -2, # cutoff of transcript score to separate between high and low score classes + groupTranscripts_method = "dbscan", + spatialMergeCheck_method = "leidenCut", + cutoff_spatialMerge = 0.5, # spatial constraint cutoff for a valid merge event + + path_to_output = "res2_multiFiles", + save_intermediates = TRUE, # flag to return and write intermediate results to disk + return_perCellData = TRUE, # flag to return per cell level outputs from updated segmentation + combine_extra = FALSE # flag to include trimmed and extracellular transcripts in the exported `updated_transDF.csv` files +) + +if(is.null(refineAll_res_one_FOC$updated_transDF_list[[1]])){ + print("no transcripts assigned after refinement") + cell_df$UMI_cellID <- as.integer(row.names(cell_df)) + transcriptDF <- left_join(cell_df, transcriptDF) + names(transcriptDF)[names(transcriptDF) == "UMI_cellID"] <- "updated_cellID" + names(transcriptDF)[names(transcriptDF) == "ct_ssam"] <- "updated_celltype" + write.csv(transcriptDF, file = path_to_transcripts_out) +} else { + write.csv(refineAll_res_one_FOC$updated_transDF_list[[1]], file = path_to_transcripts_out) +} +### export outputs +write.csv(row.names(refineAll_res_one_FOC$updated_perCellExprs), file = path_to_gene_names_out) +write.csv(refineAll_res_one_FOC$updated_perCellDT, file = path_to_cell_ids_out) From f87a1d9243813b3a80956cfeca0919a66112c183 Mon Sep 17 00:00:00 2001 From: dariarom94 Date: Thu, 23 Jul 2026 14:59:54 +0200 Subject: [PATCH 27/60] segger bug new fix --- .../segger/config.vsh.yaml | 11 +++++ .../segger/script.py | 46 +++++++++++++++++-- 2 files changed, 54 insertions(+), 3 deletions(-) diff --git a/src/methods_transcript_assignment/segger/config.vsh.yaml b/src/methods_transcript_assignment/segger/config.vsh.yaml index 377150533..c77e62aaf 100644 --- a/src/methods_transcript_assignment/segger/config.vsh.yaml +++ b/src/methods_transcript_assignment/segger/config.vsh.yaml @@ -35,6 +35,17 @@ arguments: choices: [nucleus, cell, uniform] description: "Which polygon set drives the prediction graph." default: cell + - name: --node_representation_dim + type: integer + required: false + description: >- + Node embedding size for segger's GNN — maps to segger's --node-representation-dim + (its `in_channels` / cells_embedding_size, default 128). segger runs a PCA with this + many components on a genes x genes correlation matrix, so on small panels/crops (where + fewer genes survive segger's internal count filters) the PCA fails if this exceeds the + surviving-gene count. script.py auto-retries once at the ceiling segger reports, so this + is effectively an upper bound; leave at 128 for full panels. + default: 128 resources: - type: python_script diff --git a/src/methods_transcript_assignment/segger/script.py b/src/methods_transcript_assignment/segger/script.py index fe1c32ece..4293760d5 100644 --- a/src/methods_transcript_assignment/segger/script.py +++ b/src/methods_transcript_assignment/segger/script.py @@ -1,4 +1,5 @@ import os +import re import json import shutil import subprocess @@ -184,7 +185,7 @@ ################ SEGGER_OUT_DIR.mkdir(parents=True, exist_ok=True) -cmd = [ +base_cmd = [ "segger", "segment", "-i", str(XENIUM_DIR), "-o", str(SEGGER_OUT_DIR), @@ -198,8 +199,47 @@ env.setdefault("RAPIDS_NO_INITIALIZE", "1") env.setdefault("CUDF_NO_INITIALIZE", "1") env.setdefault("RMM_NO_INITIALIZE", "1") -print("Running segger:", " ".join(cmd), flush=True) -subprocess.run(cmd, check=True, env=env) + + +def run_segger(node_dim): + # --node-representation-dim is segger's `in_channels` (== cells_embedding_size): + # the PCA n_components for the gene/cell embeddings (segger's own default 128). + cmd = base_cmd + ["--node-representation-dim", str(int(node_dim))] + print("Running segger:", " ".join(cmd), flush=True) + # Stream segger's output live AND capture it, so we can recover the max valid + # embedding dim from the PCA error below without hiding the training logs. + proc = subprocess.Popen( + cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, bufsize=1, env=env + ) + captured = [] + for line in proc.stdout: + print(line, end="", flush=True) + captured.append(line) + proc.wait() + return proc.returncode, "".join(captured) + + +# segger runs PCA(n_components=node_dim) on a genes x genes correlation matrix in +# setup_anndata. On small panels/crops fewer genes survive segger's internal count +# filters (cells_min_counts etc.) than node_dim, so the PCA raises e.g. +# "n_components=128 must be between 0 and min(n_samples, n_features)=62". The surviving +# gene count depends on segger's own filtering (not cheaply predictable here), but segger +# reports the exact ceiling in that message, so retry once at that dim. On full panels +# (>=node_dim genes survive) this never triggers. +node_dim = int(par["node_representation_dim"]) +returncode, seg_out = run_segger(node_dim) +if returncode != 0: + m = re.search(r"must be between 0 and min\(n_samples, n_features\)=(\d+)", seg_out) + if m and 0 < int(m.group(1)) < node_dim: + max_dim = int(m.group(1)) + print( + f"segger's gene PCA rejected node_representation_dim={node_dim}: only {max_dim} " + f"genes survived its count filter. Retrying at {max_dim}.", + flush=True, + ) + returncode, seg_out = run_segger(max_dim) + if returncode != 0: + raise subprocess.CalledProcessError(returncode, base_cmd) seg_pq = SEGGER_OUT_DIR / "segger_segmentation.parquet" if not seg_pq.exists(): From 123e112883f8bc9f0794653d02062436534968ba Mon Sep 17 00:00:00 2001 From: dariarom94 Date: Thu, 23 Jul 2026 15:00:19 +0200 Subject: [PATCH 28/60] fastreseg to workflow --- src/workflows/run_benchmark/config.vsh.yaml | 3 ++- src/workflows/run_benchmark/main.nf | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/workflows/run_benchmark/config.vsh.yaml b/src/workflows/run_benchmark/config.vsh.yaml index a61489ffb..638799d3b 100644 --- a/src/workflows/run_benchmark/config.vsh.yaml +++ b/src/workflows/run_benchmark/config.vsh.yaml @@ -68,7 +68,7 @@ argument_groups: A list of transcript assignment methods to run. type: string multiple: true - default: "basic_transcript_assignment:baysor:clustermap:pciseq:comseg:proseg:segger" + default: "basic_transcript_assignment:baysor:clustermap:pciseq:comseg:proseg:segger:fastreseg" - name: "--count_aggregation_methods" description: | A list of count aggregation methods to run. @@ -167,6 +167,7 @@ dependencies: - name: methods_transcript_assignment/comseg - name: methods_transcript_assignment/proseg - name: methods_transcript_assignment/segger + - name: methods_transcript_assignment/fastreseg - name: methods_count_aggregation/basic_count_aggregation - name: methods_qc_filter/basic_qc_filter - name: methods_calculate_cell_volume/alpha_shapes diff --git a/src/workflows/run_benchmark/main.nf b/src/workflows/run_benchmark/main.nf index f1725397b..63a1b0e6c 100644 --- a/src/workflows/run_benchmark/main.nf +++ b/src/workflows/run_benchmark/main.nf @@ -139,7 +139,8 @@ workflow run_wf { pciseq, comseg, proseg, - segger + segger, + fastreseg ] segm_ass_ch = segm_ch From 7549589ff566f4f213ab5628f912cf922723ff36 Mon Sep 17 00:00:00 2001 From: dariarom94 Date: Thu, 23 Jul 2026 15:00:40 +0200 Subject: [PATCH 29/60] add fastreseg test --- scripts/run_benchmark/run_test_segger_nebius.sh | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/scripts/run_benchmark/run_test_segger_nebius.sh b/scripts/run_benchmark/run_test_segger_nebius.sh index 22d1916ba..53e9488cb 100644 --- a/scripts/run_benchmark/run_test_segger_nebius.sh +++ b/scripts/run_benchmark/run_test_segger_nebius.sh @@ -1,10 +1,11 @@ #!/bin/bash -# Test run for the segger transcript-assignment method. +# Test run for the segger and fastreseg transcript-assignment methods. # Runs on the small S3 test resources, using the DEFAULT method at every stage -# plus segger added to the transcript-assignment stage. Segger requires a GPU, -# so this runs on Nebius (the gpu labels in labels_nebius.config pin it to the -# GPU node group). +# plus segger and fastreseg added to the transcript-assignment stage. Segger +# requires a GPU, so this runs on Nebius (the gpu labels in labels_nebius.config +# pin it to the GPU node group). fastreseg is CPU-based (midcpu/midmem labels) +# and schedules on a regular node within the same run. # get the root of the directory REPO_ROOT=$(git rev-parse --show-toplevel) @@ -32,6 +33,7 @@ segmentation_methods: transcript_assignment_methods: - basic_transcript_assignment - segger + - fastreseg count_aggregation_methods: - basic_count_aggregation qc_filtering_methods: @@ -68,4 +70,4 @@ tw launch https://github.com/openproblems-bio/task_ist_preprocessing.git \ --params-file /tmp/params_segger.yaml \ --entry-name auto \ --config src/base/labels_nebius.config \ - --labels task_ist_preprocessing,test,segger + --labels task_ist_preprocessing,test,segger,fastreseg From ff044673d01ecc7ec0dbdcfdebe962eaaec32d07 Mon Sep 17 00:00:00 2001 From: dariarom94 Date: Thu, 23 Jul 2026 22:21:57 +0200 Subject: [PATCH 30/60] optimized fastreseg build --- .../fastreseg/config.vsh.yaml | 57 +++++++++++----- .../fastreseg/input.py | 65 ++++++++++++++++++- .../fastreseg/orchestrator.sh | 5 ++ 3 files changed, 106 insertions(+), 21 deletions(-) diff --git a/src/methods_transcript_assignment/fastreseg/config.vsh.yaml b/src/methods_transcript_assignment/fastreseg/config.vsh.yaml index 512828392..d5c9985ec 100644 --- a/src/methods_transcript_assignment/fastreseg/config.vsh.yaml +++ b/src/methods_transcript_assignment/fastreseg/config.vsh.yaml @@ -46,24 +46,45 @@ engines: - gdal-bin - libgdal-dev - r-base - - type: r - packages: - - devtools - - terra - - Matrix - - dbscan - - igraph - - matrixStats - - codetools - - data.table - # Add other CRAN packages here - github: - # Add GitHub packages here if needed - - Nanostring-Biostats/FastReseg - #- type: python - # github: - # - openproblems-bio/core#subdirectory=packages/python/openproblems - __merge__: + # FastReseg's R dependencies as prebuilt Debian binaries. Installing + # these via apt avoids compiling terra/igraph/sf/spatstat/stringi/etc. + # from source (extremely slow, especially under amd64 emulation on + # arm64 hosts). Only concaveman + GiottoUtils/GiottoClass + FastReseg + # itself still build from source below; their heavy deps are already + # present as binaries here. + - r-cran-remotes + - r-cran-terra + - r-cran-igraph + - r-cran-sf + - r-cran-matrix + - r-cran-mass + - r-cran-e1071 + - r-cran-dbscan + - r-cran-data.table + - r-cran-dplyr + - r-cran-ggplot2 + - r-cran-reshape2 + - r-cran-stringr + - r-cran-fs + - r-cran-lmtest + - r-cran-matrixstats + - r-cran-geometry + - r-cran-spatstat.geom + - type: docker + # Install FastReseg from GitHub. upgrade = "never" so remotes reuses the + # apt-installed binary deps above instead of re-fetching/recompiling them + # from CRAN; only the unpackaged deps (concaveman, GiottoUtils, + # GiottoClass) and FastReseg itself are compiled from source. + run: + - Rscript -e 'options(warn = 2); remotes::install_github("Nanostring-Biostats/FastReseg", upgrade = "never", repos = "https://cran.rstudio.com")' + - type: python + # `plankton` (pulled in by txsim.run_ssam for ssam cell annotation in + # input.py) still does `from matplotlib.cm import get_cmap`, removed in + # matplotlib 3.9. Pin < 3.9 so run_ssam imports. This is a local setup + # step, so it runs AFTER the __merge__'d spatialdata/txsim partials (which + # otherwise pull matplotlib >= 3.9) and wins. Mirrors methods_cell_type_annotation/ssam. + pypi: [planktonspace, "matplotlib<3.9"] + __merge__: - /src/base/setup_txsim_partial.yaml - /src/base/setup_spatialdata_partial.yaml - type: native diff --git a/src/methods_transcript_assignment/fastreseg/input.py b/src/methods_transcript_assignment/fastreseg/input.py index 1da6f5a48..9eabfe0ab 100644 --- a/src/methods_transcript_assignment/fastreseg/input.py +++ b/src/methods_transcript_assignment/fastreseg/input.py @@ -7,6 +7,61 @@ import txsim as tx import anndata as ad import argparse +from scipy.sparse import csr_matrix + + +def generate_adata(input_spots, cell_id_col="cell", gene_col="Gene"): + """Aggregate per-transcript assignments into a cell x gene AnnData. + + Reimplementation of txsim.preprocessing.generate_adata: txsim's version is + incompatible with anndata>=0.12 (it passes the removed `dtype=` argument to + AnnData() and uses per-row item assignment `adata[cell, :] = ...`, which + anndata no longer supports). This fills the count matrix directly and + produces an identical output structure (X, layers['raw_counts'], + obs/var stats, uns['spots'/'pct_noise']). Kept in sync with the copy in + src/methods_count_aggregation/basic_count_aggregation/script.py. + """ + spots = input_spots.copy() + pct_noise = sum(spots[cell_id_col] <= 0) / len(spots[cell_id_col]) + spots_raw = spots.copy() # kept in uns['spots']; 0 (background) -> None + spots_raw.loc[spots_raw[cell_id_col] == 0, cell_id_col] = None + spots = spots[spots[cell_id_col] > 0] + + cell_ids = pd.unique(spots[cell_id_col]) + genes = pd.unique(spots[gene_col]) + cell_pos = {c: i for i, c in enumerate(cell_ids)} + + # Populate the count matrix + centroids per cell (no AnnData item assignment). + # feature_name may be categorical, so value_counts() can return unobserved + # categories; reindex to `genes` (fill 0) to align to the var order, mirroring + # txsim's `.reindex(var_names, fill_value=0)`. + X = np.zeros((len(cell_ids), len(genes)), dtype=np.float32) + centroid_x = np.zeros(len(cell_ids)) + centroid_y = np.zeros(len(cell_ids)) + for cell_id, grp in spots.groupby(cell_id_col, sort=False): + row = cell_pos[cell_id] + cts = grp[gene_col].value_counts().reindex(genes, fill_value=0) + X[row, :] = cts.values + centroid_x[row] = grp["x"].mean() + centroid_y[row] = grp["y"].mean() + + adata = ad.AnnData(csr_matrix(X)) + adata.obs["cell_id"] = cell_ids + adata.obs_names = [f"{i:d}" for i in cell_ids] + adata.var_names = genes + adata.obs["centroid_x"] = centroid_x + adata.obs["centroid_y"] = centroid_y + + adata.uns["spots"] = spots_raw + adata.uns["pct_noise"] = pct_noise + adata.layers["raw_counts"] = adata.X.copy() + + adata.obs["n_counts"] = np.ravel(adata.layers["raw_counts"].sum(axis=1)) + adata.obs["n_genes"] = adata.layers["raw_counts"].getnnz(axis=1) + adata.var["n_counts"] = np.ravel(adata.layers["raw_counts"].sum(axis=0)) + adata.var["n_cells"] = adata.layers["raw_counts"].getnnz(axis=0) + return adata + def parse_arguments(): parser = argparse.ArgumentParser( @@ -70,8 +125,12 @@ def parse_arguments(): print('Transforming transcripts coordinates', flush=True) transcripts = sd.transform(sdata['transcripts'], to_coordinate_system='global') -transcripts_df = transcripts.compute() -transcripts_df.rename(columns = {'feature_name': 'target', +# .copy() is required: for a single-partition points object, transcripts.compute() +# returns a reference to the dask graph's backing frame, so an in-place rename here +# would corrupt it and the later transcripts.compute() (passed to run_ssam) would +# yield renamed columns that no longer match the dask _meta -> "Metadata mismatch". +transcripts_df = transcripts.compute().copy() +transcripts_df.rename(columns = {'feature_name': 'target', 'transcript_id': 'UMI_transID', 'cell_id': 'UMI_cellID'}, inplace = True) transcripts_df = transcripts_df.loc[:, ['target', 'x', 'y', 'z', 'UMI_transID', 'UMI_cellID']] @@ -82,7 +141,7 @@ def parse_arguments(): df = sdata['transcripts'].compute() df.feature_name = df.feature_name.astype(str) -adata_sp = tx.preprocessing.generate_adata(df, cell_id_col='cell_id', gene_col='feature_name') #TODO: x and y refers to a specific coordinate system. Decide which space we want to use here. (probably should be handled in the previous assignment step) +adata_sp = generate_adata(df, cell_id_col='cell_id', gene_col='feature_name') #TODO: x and y refers to a specific coordinate system. Decide which space we want to use here. (probably should be handled in the previous assignment step) adata_sp.layers['counts'] = adata_sp.layers['raw_counts'] del adata_sp.layers['raw_counts'] adata_sp.var["gene_name"] = adata_sp.var_names diff --git a/src/methods_transcript_assignment/fastreseg/orchestrator.sh b/src/methods_transcript_assignment/fastreseg/orchestrator.sh index 9529d5b51..34d1eff74 100644 --- a/src/methods_transcript_assignment/fastreseg/orchestrator.sh +++ b/src/methods_transcript_assignment/fastreseg/orchestrator.sh @@ -1,5 +1,10 @@ #!/bin/bash +# Fail loudly: without this, a crash in any sub-step (input.py / script.R / +# output.py) is swallowed and Viash only reports the generic "required output +# file is missing", hiding the real Python/R traceback. +set -eo pipefail + ## VIASH START # The following code has been auto-generated by Viash. par_input_ist='resources_test/task_ist_preprocessing/mouse_brain_combined/raw_ist.zarr' From a7404d899e12351edf1b2379326e39f8ced5a6f3 Mon Sep 17 00:00:00 2001 From: dariarom94 Date: Thu, 23 Jul 2026 23:28:45 +0200 Subject: [PATCH 31/60] add s3 paths --- .../create_resources/spatial/process_bruker_cosmx_nebius.sh | 6 +++--- src/methods_cell_type_annotation/moscot/script.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/create_resources/spatial/process_bruker_cosmx_nebius.sh b/scripts/create_resources/spatial/process_bruker_cosmx_nebius.sh index 9c00dfcb7..c12a28b1b 100644 --- a/scripts/create_resources/spatial/process_bruker_cosmx_nebius.sh +++ b/scripts/create_resources/spatial/process_bruker_cosmx_nebius.sh @@ -14,8 +14,8 @@ cat > /tmp/params.yaml << HERE param_list: - id: "bruker_cosmx/bruker_mouse_brain_cosmx/rep1" - input_raw: "https://smi-public.objects.liquidweb.services/HalfBrain.zip" - input_flat_files: "https://smi-public.objects.liquidweb.services/Half%20%20Brain%20simple%20%20files%20.zip" + input_raw: "s3://openproblems-data/resources/raw_data/bruker_cosmx/HalfBrain.zip" + input_flat_files: "s3://openproblems-data/resources/raw_data/bruker_cosmx/Half Brain simple files.zip" dataset_name: "Bruker CosMx Mouse Brain" dataset_url: "https://nanostring.com/products/cosmx-spatial-molecular-imager/ffpe-dataset/cosmx-smi-mouse-brain-ffpe-dataset/" dataset_summary: "Bruker CosMx Mouse Brain dataset on FFPE covering a full hemisphere of a mouse brain." @@ -24,7 +24,7 @@ param_list: segmentation_id: ["cell"] - id: "bruker_cosmx/bruker_human_liver_cosmx" - input_raw: "https://smi-public.objects.liquidweb.services/NormalLiverFiles.zip" + input_raw: "s3://openproblems-data/resources/raw_data/bruker_cosmx/NormalLiverFiles.zip" dataset_name: "Bruker CosMx Human Liver" dataset_url: "https://nanostring.com/products/cosmx-spatial-molecular-imager/ffpe-dataset/human-liver-rna-ffpe-dataset/" dataset_summary: "Bruker CosMx Human Liver dataset on FFPE." diff --git a/src/methods_cell_type_annotation/moscot/script.py b/src/methods_cell_type_annotation/moscot/script.py index ac0b7343b..26be68189 100644 --- a/src/methods_cell_type_annotation/moscot/script.py +++ b/src/methods_cell_type_annotation/moscot/script.py @@ -75,13 +75,13 @@ adata_sp.X = adata_sp.layers["normalized"] adata_sp.obsm["spatial"] = adata_sp.obs[["centroid_x", "centroid_y"]].to_numpy() -sc.pp.pca(adata_sc, n_comps=50) # X is the normalized layer set above +sc.pp.pca(adata_sc, n_comps=30) # X is the normalized layer set above # Define mapping problem mp = MappingProblem(adata_sc=adata_sc, adata_sp=adata_sp) mp = mp.prepare( - sc_attr={"attr": "obsm", "key": "X_pca"}, # <-- 50-dim, not raw genes + sc_attr={"attr": "obsm", "key": "X_pca"}, # <-- 30-dim, not raw genes xy_callback="local-pca", ) From 19e5f834e02789280fdb980beb6fdf0711377a50 Mon Sep 17 00:00:00 2001 From: dariarom94 Date: Fri, 24 Jul 2026 13:08:41 +0200 Subject: [PATCH 32/60] troubleshoot comseg/segger --- .../spatial/process_bruker_cosmx.sh | 6 +-- .../comseg/config.vsh.yaml | 24 +++++++++ .../comseg/script.py | 51 +++++++++++++++++-- .../fastreseg/output.py | 6 +++ .../segger/script.py | 17 +++++++ 5 files changed, 98 insertions(+), 6 deletions(-) diff --git a/scripts/create_resources/spatial/process_bruker_cosmx.sh b/scripts/create_resources/spatial/process_bruker_cosmx.sh index 7e4a7ea29..a303b0997 100644 --- a/scripts/create_resources/spatial/process_bruker_cosmx.sh +++ b/scripts/create_resources/spatial/process_bruker_cosmx.sh @@ -14,8 +14,8 @@ cat > /tmp/params.yaml << HERE param_list: - id: "bruker_cosmx/bruker_mouse_brain_cosmx/rep1" - input_raw: "https://smi-public.objects.liquidweb.services/HalfBrain.zip" - input_flat_files: "https://smi-public.objects.liquidweb.services/Half%20%20Brain%20simple%20%20files%20.zip" + input_raw: "s3://openproblems-data/resources/raw_data/bruker_cosmx/HalfBrain.zip" + input_flat_files: "s3://openproblems-data/resources/raw_data/bruker_cosmx/Half Brain simple files.zip" dataset_name: "Bruker CosMx Mouse Brain" dataset_url: "https://nanostring.com/products/cosmx-spatial-molecular-imager/ffpe-dataset/cosmx-smi-mouse-brain-ffpe-dataset/" dataset_summary: "Bruker CosMx Mouse Brain dataset on FFPE covering a full hemisphere of a mouse brain." @@ -24,7 +24,7 @@ param_list: segmentation_id: ["cell"] - id: "bruker_cosmx/bruker_human_liver_cosmx" - input_raw: "https://smi-public.objects.liquidweb.services/NormalLiverFiles.zip" + input_raw: "s3://openproblems-data/resources/raw_data/bruker_cosmx/NormalLiverFiles.zip" dataset_name: "Bruker CosMx Human Liver" dataset_url: "https://nanostring.com/products/cosmx-spatial-molecular-imager/ffpe-dataset/human-liver-rna-ffpe-dataset/" dataset_summary: "Bruker CosMx Human Liver dataset on FFPE." diff --git a/src/methods_transcript_assignment/comseg/config.vsh.yaml b/src/methods_transcript_assignment/comseg/config.vsh.yaml index 000e363f8..a92907f43 100644 --- a/src/methods_transcript_assignment/comseg/config.vsh.yaml +++ b/src/methods_transcript_assignment/comseg/config.vsh.yaml @@ -63,6 +63,30 @@ arguments: type: boolean default: true description: "Allow disconnected polygons in segmentation" + - name: --n_workers + type: integer + default: 4 + description: | + Maximum number of dask worker processes used to run ComSeg patches in + parallel. ComSeg builds an in-memory graph per patch, so peak memory + scales with the number of concurrent workers; the value is clamped to + (cpus - 1). Lower this if the job is OOM-killed (exit 137); raise it + (up to the CPU count) to run faster when memory allows. + - name: --worker_memory_limit + type: string + default: "0" + description: | + Per-worker memory limit for the dask LocalCluster. Default "0" (also + "none"/"off"/"disabled") neutralises dask's per-worker memory monitor by + setting each worker's limit to the FULL detected container memory. ComSeg's + per-patch memory is dominated by unmanaged native allocations + (numba/shapely/graphs) that dask cannot spill, so a tighter limit does not + buy spilling — it only makes the nanny kill otherwise-healthy workers + (KilledWorker) once they cross ~95% of the limit. Total memory is instead + bounded by n_workers, and a genuine OOM is caught by the container cgroup + + Nextflow retry. Set an explicit value (e.g. "20GB") only if you want dask + to police each worker; "auto" is discouraged because it divides detected + RAM by CPU count and is easily smaller than a single patch's working set. resources: diff --git a/src/methods_transcript_assignment/comseg/script.py b/src/methods_transcript_assignment/comseg/script.py index 2829b873f..1905fe29b 100644 --- a/src/methods_transcript_assignment/comseg/script.py +++ b/src/methods_transcript_assignment/comseg/script.py @@ -4,6 +4,7 @@ import anndata as ad import pandas as pd import numpy as np +import dask import dask.dataframe as dd ## VIASH START @@ -24,6 +25,8 @@ "gene_column": "feature_name", "norm_vector": False, "allow_disconnected_polygon": True, + "n_workers": 4, + "worker_memory_limit": "0", } meta = { "name": "comseg", @@ -79,12 +82,54 @@ # ComSeg processes each transcript patch independently and is pure-Python, so it # runs single-threaded unless a parallelization backend is enabled. Use the dask -# backend to spread patches across the allocated CPUs (see midcpu/highmem labels). -n_workers = max((meta["cpus"] or os.cpu_count() or 1) - 1, 1) +# backend to spread patches across workers, but CAP concurrency: comseg builds an +# in-memory graph per patch, so peak memory scales with the number of concurrent +# workers. Running one worker per CPU (14 on midcpu) multiplied peak RAM ~14x and +# got the job OOM-killed (exit 137 = cgroup SIGKILL). Bounding the worker count is +# the real memory safeguard (the per-worker memory monitor is neutralised below, +# see the note there). +cpu_cap = max((meta["cpus"] or os.cpu_count() or 1) - 1, 1) +n_workers = max(min(par["n_workers"], cpu_cap), 1) + +# `distributed` defaults worker startup to "spawn", which re-imports this script +# in every worker. As a plain viash script it has no `if __name__ == "__main__"` +# guard, so the re-import re-runs the module-level code (creating yet another +# cluster) and Python's spawn bootstrap check aborts with "An attempt has been +# made to start a new process before ... bootstrapping". Force "fork" so workers +# inherit the already-initialised interpreter instead of re-importing it; this is +# the Linux-native default and the mode comseg ran under before the deps bumped. +dask.config.set({"distributed.worker.multiprocessing-method": "fork"}) + sopa.settings.parallelization_backend = "dask" sopa.settings.dask_client_kwargs["n_workers"] = n_workers sopa.settings.dask_client_kwargs["threads_per_worker"] = 1 # CPU-bound work, avoid GIL contention -print(f"Running ComSeg with dask backend, n_workers={n_workers}", flush=True) + +# ComSeg's per-patch memory is mostly UNMANAGED (native numba/shapely/graph +# allocations, plus the forked interpreter's copy-on-write pages), which dask +# cannot spill. A per-worker memory_limit therefore never spills here; it only +# lets the nanny kill workers that cross ~95% of the limit -> KilledWorker, even +# though nothing is leaking. dask's "auto" is worse: it divides detected RAM by +# the CPU count (not n_workers), so on an 8-core box each worker gets ~1/8 of RAM +# and a normal ~1 GB patch already trips 95%. We therefore neutralise the monitor +# by default, but sopa reads worker.memory_manager.memory_limit and compares it to +# 4 GiB, so the value must be a real number (0/None makes sopa crash). Setting the +# limit to the FULL detected container memory does both: sopa sees a number, and +# the nanny only ever fires near 95% of the whole cgroup — where the OS + Nextflow +# retry already take over — never prematurely for a single patch. Total memory is +# bounded by n_workers instead. An explicit value (e.g. "20GB") is honoured as-is. +_mem_limit_arg = (par["worker_memory_limit"] or "").strip().lower() +if _mem_limit_arg in ("", "0", "none", "off", "disabled"): + from distributed.system import MEMORY_LIMIT as _CONTAINER_MEMORY + worker_memory_limit = _CONTAINER_MEMORY +else: + worker_memory_limit = par["worker_memory_limit"] +sopa.settings.dask_client_kwargs["memory_limit"] = worker_memory_limit + +print( + f"Running ComSeg with dask backend, n_workers={n_workers} " + f"(cap {cpu_cap}), memory_limit={worker_memory_limit!r}", + flush=True, +) sopa.segmentation.comseg(sdata, config) diff --git a/src/methods_transcript_assignment/fastreseg/output.py b/src/methods_transcript_assignment/fastreseg/output.py index 7849d351e..dbca407fb 100644 --- a/src/methods_transcript_assignment/fastreseg/output.py +++ b/src/methods_transcript_assignment/fastreseg/output.py @@ -36,6 +36,12 @@ def convert_to_lower_dtype(arr): ##converting to dask transcript output for spatialdata format df = df.loc[:,["x", "y", "z", "target", "updated_cellID", "updated_celltype", "UMI_transID"]] df.rename(columns={"updated_cellID": "cell_id", "target": "feature_name", "UMI_transID": "transcript_id"}, inplace=True) +# Match the raw_ist convention where feature_name is categorical. Rebuilt from +# R's CSV it would otherwise be a pyarrow-backed nullable string that survives +# the zarr round-trip; the downstream count-aggregation then builds var_names +# from it and write_h5ad refuses to serialize the nullable-string index +# ("anndata.settings.allow_write_nullable_strings is None"). +df["feature_name"] = df["feature_name"].astype(str).astype("category") transcripts_dask = dask.dataframe.from_pandas(df, npartitions = 1) diff --git a/src/methods_transcript_assignment/segger/script.py b/src/methods_transcript_assignment/segger/script.py index 4293760d5..610fc203c 100644 --- a/src/methods_transcript_assignment/segger/script.py +++ b/src/methods_transcript_assignment/segger/script.py @@ -34,6 +34,23 @@ } ## VIASH END +# The Nextflow/k8s GPU task hands us an EMPTY CUDA_VISIBLE_DEVICES. An empty (or +# whitespace) value masks every GPU from the CUDA *driver* API, so cudf/cuspatial and +# numba enumerate zero devices and segger dies deep in tiling with +# "cudaErrorNoDevice: no CUDA-capable device is detected" / numba's +# "IndexError: list index out of range" (self.gpus[devnum] on an empty device list). +# torch's is_available() is NVML-based and ignores CVD, so the gate below still passes +# and the failure only surfaces inside the `segger segment` subprocess. Drop an empty +# value so the allocated device (exposed via NVIDIA_VISIBLE_DEVICES) is visible again; +# run_segger's os.environ.copy() then inherits the fix for the subprocess. +if os.environ.get("CUDA_VISIBLE_DEVICES", "x").strip() == "": + print( + "CUDA_VISIBLE_DEVICES was empty; unsetting so the allocated GPU is visible " + "to cudf/numba (torch's NVML check hides this).", + flush=True, + ) + del os.environ["CUDA_VISIBLE_DEVICES"] + # segger runs its tiling / GNN training / prediction end-to-end on the GPU # (cudf, cuspatial, torch kernels). Fail fast with a clear message rather than # deep inside the subprocess when no CUDA device is present. From aaca1515bd7a482aba8535aa658c2a79b08fcaf1 Mon Sep 17 00:00:00 2001 From: dariarom94 Date: Sat, 25 Jul 2026 23:32:44 +0200 Subject: [PATCH 33/60] segger update --- .../segger/config.vsh.yaml | 11 +++++++- .../segger/script.py | 25 ++++++++++++------- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/src/methods_transcript_assignment/segger/config.vsh.yaml b/src/methods_transcript_assignment/segger/config.vsh.yaml index c77e62aaf..889e143b9 100644 --- a/src/methods_transcript_assignment/segger/config.vsh.yaml +++ b/src/methods_transcript_assignment/segger/config.vsh.yaml @@ -107,12 +107,21 @@ engines: # crashes under pandas 2.2.3. Hold anndata/scanpy at their last pandas-2.2 # majors (anndata 0.12.x: pandas>=2.1; scanpy 1.11.x: pandas>=1.5), and # re-assert spatialdata<0.8 + dask==2025.2.0 in case a dep nudged them. + # - PIN numba<0.61 + numpy<2.1. scanpy's umap-learn/pynndescent (no numba upper + # bound) otherwise drag numba to 0.66 and numpy to 2.4.x, which violate the + # RAPIDS 25.04 stack: cudf/cuml/cugraph require numba<0.61 and cupy requires + # numpy<2.3. numba 0.60 pulls numpy<2.2; cap at <2.1 for the base's 2.0.2. The + # numba drift did NOT itself crash the GPU op (the NUMBA_CUDA_USE_NVIDIA_BINDING + # env var in script.py is the actual fix), but running cuml/cugraph out of their + # supported numba/numpy range is a latent risk for segger's phenograph step. + # Validated set: numba 0.60.0 / numpy 2.0.2 / llvmlite 0.43.0 (cudf .values, + # cuml kNN, cugraph louvain, torch, spatialdata/anndata all import + run). # - swap opencv-python -> opencv-python-headless: segger pulls opencv-python, # whose cv2 needs libGL.so.1 (a system lib we cannot apt-install as non-root); # the headless build provides the same cv2 without it. See NOTES.md. - type: docker run: - - pip install --no-cache-dir "pandas>=2.0,<2.2.4" "anndata>=0.12,<0.13" "scanpy>=1.11,<1.12" "spatialdata>=0.7,<0.8" "dask==2025.2.0" "distributed==2025.2.0" + - pip install --no-cache-dir "pandas>=2.0,<2.2.4" "anndata>=0.12,<0.13" "scanpy>=1.11,<1.12" "spatialdata>=0.7,<0.8" "dask==2025.2.0" "distributed==2025.2.0" "numba>=0.59.1,<0.61" "numpy>=2.0,<2.1" - pip uninstall -y opencv-python || true - pip install --no-cache-dir opencv-python-headless - type: native diff --git a/src/methods_transcript_assignment/segger/script.py b/src/methods_transcript_assignment/segger/script.py index 610fc203c..040417644 100644 --- a/src/methods_transcript_assignment/segger/script.py +++ b/src/methods_transcript_assignment/segger/script.py @@ -34,15 +34,12 @@ } ## VIASH END -# The Nextflow/k8s GPU task hands us an EMPTY CUDA_VISIBLE_DEVICES. An empty (or -# whitespace) value masks every GPU from the CUDA *driver* API, so cudf/cuspatial and -# numba enumerate zero devices and segger dies deep in tiling with -# "cudaErrorNoDevice: no CUDA-capable device is detected" / numba's -# "IndexError: list index out of range" (self.gpus[devnum] on an empty device list). -# torch's is_available() is NVML-based and ignores CVD, so the gate below still passes -# and the failure only surfaces inside the `segger segment` subprocess. Drop an empty -# value so the allocated device (exposed via NVIDIA_VISIBLE_DEVICES) is visible again; -# run_segger's os.environ.copy() then inherits the fix for the subprocess. +# Defensive (NOT the observed root cause — see the NUMBA_CUDA_USE_NVIDIA_BINDING fix in +# run_segger). A set-but-EMPTY CUDA_VISIBLE_DEVICES masks every GPU from the CUDA driver +# API (cudf/numba would then enumerate zero devices → "cudaErrorNoDevice") while torch's +# NVML-based is_available() still passes the gate below. The real Nextflow task runs with +# CVD *unset* (verified), so this branch does not fire there — but an empty value is a real +# failure mode in other launchers, so drop it to expose the allocated device. if os.environ.get("CUDA_VISIBLE_DEVICES", "x").strip() == "": print( "CUDA_VISIBLE_DEVICES was empty; unsetting so the allocated GPU is visible " @@ -216,6 +213,16 @@ env.setdefault("RAPIDS_NO_INITIALIZE", "1") env.setdefault("CUDF_NO_INITIALIZE", "1") env.setdefault("RMM_NO_INITIALIZE", "1") +# THE load-bearing GPU fix. RAPIDS (cudf/cuml/cugraph) builds its CUDA context via the +# cuda-python (NVIDIA) driver binding. numba MUST use the same binding, otherwise numba's +# own driver enumerates ZERO devices inside cudf's `.values`/`as_cuda_array` path and dies +# with `numba_cuda ... IndexError: list index out of range` (self.gpus empty) deep in +# segger's `setup_anndata`/`phenograph_rapids` — even though torch, `cudf.sum()` and cupy +# all see the GPU fine. Reproduced + fixed live on a GPU pod: without this the op crashes; +# with it, cudf `.values.get()` + cuml kNN + cugraph louvain all pass. (A benign +# "cuDriverGetVersion() takes no arguments / Not patching Numba" warning is printed and can +# be ignored.) Force it (not setdefault) so an inherited "0" can't re-break it. +env["NUMBA_CUDA_USE_NVIDIA_BINDING"] = "1" def run_segger(node_dim): From c9bdb9175d4c16d62085fd653ac4024390d42ab9 Mon Sep 17 00:00:00 2001 From: dariarom94 Date: Sat, 25 Jul 2026 23:39:09 +0200 Subject: [PATCH 34/60] data loader bug --- src/datasets/loaders/bruker_cosmx/config.vsh.yaml | 2 +- src/datasets/loaders/ganier_human_skin_sc/script.py | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/datasets/loaders/bruker_cosmx/config.vsh.yaml b/src/datasets/loaders/bruker_cosmx/config.vsh.yaml index 0ed1a6bc5..f51200615 100644 --- a/src/datasets/loaders/bruker_cosmx/config.vsh.yaml +++ b/src/datasets/loaders/bruker_cosmx/config.vsh.yaml @@ -74,4 +74,4 @@ runners: - type: executable - type: nextflow directives: - label: [highmem, midcpu, hightime] + label: [veryhighmem, midcpu, hightime] diff --git a/src/datasets/loaders/ganier_human_skin_sc/script.py b/src/datasets/loaders/ganier_human_skin_sc/script.py index e258e1ec2..30e894e5a 100644 --- a/src/datasets/loaders/ganier_human_skin_sc/script.py +++ b/src/datasets/loaders/ganier_human_skin_sc/script.py @@ -40,11 +40,15 @@ # Filter out bcc samples adata = adata[~adata.obs["01_sample"].str.startswith("bcc")] -# NOTE: only log-normalized counts are available, in the workflow we'll skip the log-normalization step -# This is not optimal, layers "counts" should be the raw counts -adata.layers["counts"] = adata.X +# The download stores log-normalized values in .X and the raw integer counts in +# .raw.X (full transcriptome, a superset of adata's genes). Count-based methods +# (RCTD/SPLIT) need integer counts, so source the `counts` layer from .raw aligned +# to this object's genes; keep the log-normalized values as `normalized` (the +# process workflow skips its own log-normalization step for this dataset). +adata.layers["counts"] = adata.raw[:, adata.var_names].X adata.layers["normalized"] = adata.X del adata.X +del adata.raw # Rename fields rename_obs_keys = { From 4467d3275a9ac3c1e36370715be6948abd550a98 Mon Sep 17 00:00:00 2001 From: dariarom94 Date: Sun, 26 Jul 2026 14:57:34 +0200 Subject: [PATCH 35/60] rctd adjustment (raw counts) --- src/methods_cell_type_annotation/rctd/script.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/methods_cell_type_annotation/rctd/script.R b/src/methods_cell_type_annotation/rctd/script.R index 371f1556e..1bdc3e6c4 100644 --- a/src/methods_cell_type_annotation/rctd/script.R +++ b/src/methods_cell_type_annotation/rctd/script.R @@ -18,7 +18,7 @@ par <- list( ) meta <- list( - 'cpus': 4, + cpus = 4 ) ## VIASH END From 58912e464f88bb615bd89432171b61cbf05711f3 Mon Sep 17 00:00:00 2001 From: dariarom94 Date: Sun, 26 Jul 2026 15:08:35 +0200 Subject: [PATCH 36/60] fix segger and comseg --- .../comseg/script.py | 27 ++++++++++++++----- .../segger/config.vsh.yaml | 13 ++++++++- 2 files changed, 33 insertions(+), 7 deletions(-) diff --git a/src/methods_transcript_assignment/comseg/script.py b/src/methods_transcript_assignment/comseg/script.py index 1905fe29b..84ed9a438 100644 --- a/src/methods_transcript_assignment/comseg/script.py +++ b/src/methods_transcript_assignment/comseg/script.py @@ -37,22 +37,37 @@ # Read input files print('Reading input files', flush=True) -sdata = sd.read_zarr(par['input_ist']) +sdata_src = sd.read_zarr(par['input_ist']) sdata_segm = sd.read_zarr(par['input_segmentation']) # Multi-partition parquet restarts its index at 0 per partition, producing duplicate # index labels that break sopa's cell_id assignment and the final write with # "cannot reindex on an axis with duplicate labels". Rebuild the transcripts as a # single-partition frame with a clean RangeIndex before any sopa op touches them. -_tx = sdata[par['transcripts_key']] +_tx = sdata_src[par['transcripts_key']] _tx_reset = dd.from_pandas(_tx.compute().reset_index(drop=True), npartitions=1) _tx_reset.attrs.update(_tx.attrs) -del sdata[par['transcripts_key']] -sdata[par['transcripts_key']] = _tx_reset # Convert the prior segmentation to polygons -sdata["segmentation_boundaries"] = sd.to_polygons(sdata_segm["segmentation"]) -del sdata["segmentation_boundaries"]["label"] # make_transcript_patches will create a new label column and fails if one exists. +segmentation_boundaries = sd.to_polygons(sdata_segm["segmentation"]) +del segmentation_boundaries["label"] # make_transcript_patches will create a new label column and fails if one exists. + +# Run sopa/ComSeg on a FRESH, in-memory SpatialData. sd.read_zarr() leaves `sdata_src` +# backed by the shared source zarr (sdata_src.path -> input_ist), and sopa's +# make_image_patches / make_transcript_patches / segmentation.comseg all write their new +# elements (image_patches, transcripts_patches, comseg_boundaries) and a .sopa_cache back +# through that path -- mutating the supposed-to-be-immutable source dataset in place. If a +# concurrent run or an OOM kill interrupts one of those writes, it leaves a half-written +# element (e.g. shapes/image_patches with no shapes.parquet) that then breaks EVERY +# downstream reader of the dataset. baysor and proseg avoid this by running sopa on a fresh +# in-memory object; do the same here (carrying the image for make_image_patches and the +# prior boundaries for the transcript patches) so all sopa writes land in the ephemeral +# work dir, never the source zarr. +sdata = sd.SpatialData( + images={"image": sdata_src["image"]}, + points={par['transcripts_key']: _tx_reset}, + shapes={"segmentation_boundaries": segmentation_boundaries}, +) # Make patches sopa.make_image_patches(sdata, image_key="image", patch_width=par["patch_width"], patch_overlap=par["patch_overlap"]) diff --git a/src/methods_transcript_assignment/segger/config.vsh.yaml b/src/methods_transcript_assignment/segger/config.vsh.yaml index 889e143b9..f83d61b0e 100644 --- a/src/methods_transcript_assignment/segger/config.vsh.yaml +++ b/src/methods_transcript_assignment/segger/config.vsh.yaml @@ -116,14 +116,25 @@ engines: # supported numba/numpy range is a latent risk for segger's phenograph step. # Validated set: numba 0.60.0 / numpy 2.0.2 / llvmlite 0.43.0 (cudf .values, # cuml kNN, cugraph louvain, torch, spatialdata/anndata all import + run). + # - PIN polars>=1.24,<1.26 (== what cudf-polars 25.4 requires). segger's github + # install leaves polars unpinned; this keeps it at the RAPIDS-compatible 1.24. + # See the writer.py from_torch shim below for why we must stay <1.26. # - swap opencv-python -> opencv-python-headless: segger pulls opencv-python, # whose cv2 needs libGL.so.1 (a system lib we cannot apt-install as non-root); # the headless build provides the same cv2 without it. See NOTES.md. - type: docker run: - - pip install --no-cache-dir "pandas>=2.0,<2.2.4" "anndata>=0.12,<0.13" "scanpy>=1.11,<1.12" "spatialdata>=0.7,<0.8" "dask==2025.2.0" "distributed==2025.2.0" "numba>=0.59.1,<0.61" "numpy>=2.0,<2.1" + - pip install --no-cache-dir "pandas>=2.0,<2.2.4" "anndata>=0.12,<0.13" "scanpy>=1.11,<1.12" "spatialdata>=0.7,<0.8" "dask==2025.2.0" "distributed==2025.2.0" "numba>=0.59.1,<0.61" "numpy>=2.0,<2.1" "polars>=1.24,<1.26" - pip uninstall -y opencv-python || true - pip install --no-cache-dir opencv-python-headless + # segger's data/writer.py calls pl.from_torch to turn the prediction tensors into a + # polars frame, but polars only added from_torch AFTER 1.27 (PR pola-rs/polars#22177) + # -- and cudf-polars 25.4 caps polars at <1.26, so we cannot get the real one without + # breaking the RAPIDS stack. Inject the missing function (torch tensor -> from_numpy, + # honoring segger's list/dict schema) right after writer.py's `import torch`. getattr + # preserves a real from_torch if a future polars provides one. Path is fixed by the + # py3.12 base image; a build failure here (missing file) is a loud signal it moved. + - "sed -i '/^import torch$/a pl.from_torch = getattr(pl, \"from_torch\", lambda data, schema=None, **k: pl.from_numpy(data.detach().cpu().numpy(), schema=schema))' /opt/conda/lib/python3.12/site-packages/segger/data/writer.py" - type: native runners: From 0a5aa995d7a5bc0e6a568c36cb526805f5d11662 Mon Sep 17 00:00:00 2001 From: dariarom94 Date: Sun, 26 Jul 2026 15:57:26 +0200 Subject: [PATCH 37/60] optimize cosmx --- src/datasets/loaders/bruker_cosmx/script.py | 53 +++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/src/datasets/loaders/bruker_cosmx/script.py b/src/datasets/loaders/bruker_cosmx/script.py index 00b2e3f64..883913f29 100644 --- a/src/datasets/loaders/bruker_cosmx/script.py +++ b/src/datasets/loaders/bruker_cosmx/script.py @@ -199,6 +199,59 @@ def extract_zip(input_zip: Path, output_dir: Path, strip_root: bool = False): else: raise AssertionError(f"No CellLabels folder and no per-FOV CellLabels tifs found in {DATA_DIR}") +############################################# +# Memory optimization: lean transcript read # +############################################# +# sopa's CosMx reader loads the ENTIRE `*_tx_file.csv` into a pandas DataFrame in +# one `pd.read_csv` (sopa/io/reader/cosmx.py::_CosMXReader.read_transcripts). For +# the mouse-brain half-hemisphere that's hundreds of millions of rows, and the +# string columns ('target', 'CellComp', 'cell', ...) come in as object dtype +# (~60 B/row) — this is the peak that blows past even the 480 GB veryhighmem cap. +# +# We wrap the `read_csv` used inside sopa's cosmx module so that, *only for the +# transcripts file*, we (a) read the 'target' feature column as categorical and +# (b) keep just the columns sopa's stitched-FOV path (read_transcripts + +# _get_global_cell_id) and the downstream transcript schema actually use. Every +# other sopa read (fov positions, metadata, counts, polygons) is left untouched, +# and all of sopa's coordinate/stitching math runs unchanged on the leaner frame. +from sopa.io.reader import cosmx as _cosmx_mod + +_real_pd = _cosmx_mod.pd +_real_read_csv = _real_pd.read_csv + +# Columns used by read_transcripts (fov=None) + _get_global_cell_id, plus 'z' +# which the downstream transcript schema allows. Everything else in the tx file +# (x_local_px, y_local_px, cell, CellComp, ...) is unused downstream. +_TX_KEEP = ["fov", "cell_ID", "target", "x_global_px", "y_global_px", "z"] +_TX_REQUIRED = {"target", "x_global_px", "y_global_px", "fov", "cell_ID"} + +def _lean_read_csv(filepath_or_buffer, *args, **kwargs): + name = str(filepath_or_buffer) + if "_tx_file.csv" in name and "usecols" not in kwargs: + header = _real_read_csv( + filepath_or_buffer, nrows=0, + compression=kwargs.get("compression", "infer"), + ) + cols = set(header.columns) + if _TX_REQUIRED.issubset(cols): + kwargs["usecols"] = [c for c in _TX_KEEP if c in cols] + dtype = dict(kwargs.get("dtype") or {}) + dtype.setdefault("target", "category") + kwargs["dtype"] = dtype + log(f"Lean transcript read of {Path(name).name}: usecols={kwargs['usecols']}, target=category") + return _real_read_csv(filepath_or_buffer, *args, **kwargs) + +class _PandasReadCsvProxy: + """Forwards every attribute to pandas, but leans the transcripts read_csv.""" + def __init__(self, real): + self.__dict__["_real"] = real + def read_csv(self, *args, **kwargs): + return _lean_read_csv(*args, **kwargs) + def __getattr__(self, item): + return getattr(self._real, item) + +_cosmx_mod.pd = _PandasReadCsvProxy(_real_pd) + ######################################### # Convert raw files to spatialdata zarr # ######################################### From c921937d8686d151127c9dbcfc99fc90b674f386 Mon Sep 17 00:00:00 2001 From: dariarom94 Date: Mon, 27 Jul 2026 00:53:43 +0200 Subject: [PATCH 38/60] parameter test for cellpose4 --- scripts/run_benchmark/cellposev4_params.yaml | 47 +++++++++++++++++++ .../loaders/bruker_cosmx/config.vsh.yaml | 12 +++-- .../process_bruker_cosmx/config.vsh.yaml | 4 +- .../cellposev4/config.vsh.yaml | 10 +++- src/methods_segmentation/cellposev4/script.py | 3 +- 5 files changed, 69 insertions(+), 7 deletions(-) create mode 100644 scripts/run_benchmark/cellposev4_params.yaml diff --git a/scripts/run_benchmark/cellposev4_params.yaml b/scripts/run_benchmark/cellposev4_params.yaml new file mode 100644 index 000000000..6681b0f89 --- /dev/null +++ b/scripts/run_benchmark/cellposev4_params.yaml @@ -0,0 +1,47 @@ +# Parameter sweep for the cellposev4 (Cellpose 4 / Cellpose-SAM) segmentation method. +# Shared source of truth for both run_test_cellposev4_local.sh (read as a local file) +# and run_test_cellposev4_nebius.sh (read from GitHub via a raw URL, since the Nebius +# compute env pulls the repo but cannot see the launch host's local files). +# +# Consumed by the run_benchmark workflow via the `method_parameters_yaml` setting +# (src/workflows/run_benchmark/main.nf). For every method the workflow builds: +# * one "default" variant using the `default:` args below, and +# * one extra variant per value in each `sweep:` list, with that ONE arg overridden. +# The benchmark varies a SINGLE parameter at a time (a "star" around the default, not +# a full grid), so total cellposev4 variants = 1 default + sum(sweep list lengths) = 19. +# +# See src/methods_segmentation/cellposev4/NOTES.md ("Optimization / tuning") for the +# rationale, tiers, and defaults. Trim or widen the lists to taste. +parameters: + cellposev4: + # Baseline == the component's shipped (deliberately speed-tuned) defaults. + default: + diameter: 30.0 + flow_threshold: 0.0 + cellprob_threshold: 0.0 + niter: 10 + min_size: -1 + resample: false + sweep: + # ---- Tier 1: highest impact on segmentation quality ---- + # diameter: cellpose rescales so objects land near ~30 px (training mean), + # so 30.0 == "no rescaling". Xenium morphology (~0.2125 um/px) => nuclei + # ~35-40 px, whole cells ~60-70 px. Span small nuclei -> whole cells. + diameter: [15.0, 20.0, 40.0, 60.0] + # cellprob_threshold: recall<->precision dial (network logit ~ -6..+6, def 0). + # Lower -> recover more/dimmer cells; higher -> drop dim-region detections. + cellprob_threshold: [-3.0, -1.0, 1.0, 3.0] + # flow_threshold: 0.0 (our default) disables flow-shape QC (keeps ill-shaped + # masks); >0 re-enables it, more permissive as it rises. 0.4 = Cellpose default. + flow_threshold: [0.4, 0.6, 0.8] + # ---- Tier 2: quality/speed trade-offs ---- + # niter: flow-dynamics iterations; 10 (default) is aggressively low and may + # under-converge large/elongated cells. Cellpose's own default is ~diameter- + # proportional (often ~200). + niter: [50, 100, 200] + # resample: BOOLEAN, default false -> the only meaningful non-default value is + # true (run dynamics at full resolution: smoother boundaries, slower). + resample: [true] + # min_size: -1 (default) disables small-mask removal; positive values drop + # progressively larger specks/debris. + min_size: [15, 30, 50] diff --git a/src/datasets/loaders/bruker_cosmx/config.vsh.yaml b/src/datasets/loaders/bruker_cosmx/config.vsh.yaml index f51200615..901efe036 100644 --- a/src/datasets/loaders/bruker_cosmx/config.vsh.yaml +++ b/src/datasets/loaders/bruker_cosmx/config.vsh.yaml @@ -4,10 +4,15 @@ namespace: datasets/loaders argument_groups: - name: Inputs arguments: - - type: file + - type: string name: --input_raw example: "https://smi-public.objects.liquidweb.services/HalfBrain.zip" - description: "Download file url for the raw data" + description: | + URL/path to the raw data zip. Passed as a string (not a staged file) on + purpose: the script streams it in place and extracts only the ~15 GB sopa + needs (Morphology2D + CellLabels), skipping the ~190 GB of Morphology3D / + AnalysisResults. Staging the full zip (175 GiB) alongside the extraction + overruns the node's scratch and OOMs. Supports s3:// (via s3fs) or a local path. - type: file name: --input_flat_files example: "https://smi-public.objects.liquidweb.services/Half%20%20Brain%20simple%20%20files%20.zip" @@ -65,9 +70,10 @@ engines: __merge__: - /src/base/setup_spatialdata_partial.yaml setup: - - type: python + - type: python pypi: - sopa + - s3fs # stream the raw zip from s3:// and extract only what sopa needs (see --input_raw) - type: native runners: diff --git a/src/datasets/workflows/process_bruker_cosmx/config.vsh.yaml b/src/datasets/workflows/process_bruker_cosmx/config.vsh.yaml index 3c256c603..195b96788 100644 --- a/src/datasets/workflows/process_bruker_cosmx/config.vsh.yaml +++ b/src/datasets/workflows/process_bruker_cosmx/config.vsh.yaml @@ -4,10 +4,10 @@ namespace: datasets/workflows argument_groups: - name: Inputs arguments: - - type: file + - type: string name: --input_raw example: "https://smi-public.objects.liquidweb.services/HalfBrain.zip" - description: "Download file url for the raw data" + description: "URL/path to the raw data zip. A string (not a staged file) so the loader can stream it and extract only what sopa needs — see the loader's --input_raw." - type: file name: --input_flat_files example: "https://smi-public.objects.liquidweb.services/Half%20%20Brain%20simple%20%20files%20.zip" diff --git a/src/methods_segmentation/cellposev4/config.vsh.yaml b/src/methods_segmentation/cellposev4/config.vsh.yaml index 25d20d264..7b994cf7d 100644 --- a/src/methods_segmentation/cellposev4/config.vsh.yaml +++ b/src/methods_segmentation/cellposev4/config.vsh.yaml @@ -12,7 +12,11 @@ links: documentation: "https://cellpose.readthedocs.io/en/latest/" repository: "https://github.com/mouseland/cellpose" references: - doi: "10.1038/s41592-020-01018-x" + doi: + # Cellpose-SAM (cpsam): the model this component actually runs + - "10.1101/2025.04.28.651001" + # Original Cellpose: the flow-dynamics framework it builds on + - "10.1038/s41592-020-01018-x" __merge__: /src/api/comp_method_segmentation.yaml @@ -25,6 +29,10 @@ arguments: type: double default: 0.0 description: "Flow error threshold. Set to 0 to skip flow quality check for faster execution." + - name: --cellprob_threshold + type: double + default: 0.0 + description: "Cell probability threshold (network output ~-6 to 6). Pixels above it are used to seed masks. Lower to recover more/dimmer cells; raise to drop detections in dim/low-contrast regions." - name: --niter type: integer default: 10 diff --git a/src/methods_segmentation/cellposev4/script.py b/src/methods_segmentation/cellposev4/script.py index 1c5e51a7f..159584419 100644 --- a/src/methods_segmentation/cellposev4/script.py +++ b/src/methods_segmentation/cellposev4/script.py @@ -18,6 +18,7 @@ "output": "segmentation.zarr", "diameter": 30.0, "flow_threshold": 0.0, + "cellprob_threshold": 0.0, "niter": 10, "min_size": -1, "resample": False, @@ -50,7 +51,7 @@ def convert_to_lower_dtype(arr): print('Initializing Cellpose model', flush=True) model = CellposeModel(gpu=torch.cuda.is_available()) -eval_params = {k: par[k] for k in ("diameter", "flow_threshold", "niter", "min_size", "resample") if par.get(k) is not None} +eval_params = {k: par[k] for k in ("diameter", "flow_threshold", "cellprob_threshold", "niter", "min_size", "resample") if par.get(k) is not None} print('Running Cellpose segmentation with parameters:', eval_params, flush=True) masks, _, _ = model.eval(image[0], progress=True, **eval_params) From 57c2d791b0192264c208771f9b868528a6208cbe Mon Sep 17 00:00:00 2001 From: dariarom94 Date: Mon, 27 Jul 2026 00:54:08 +0200 Subject: [PATCH 39/60] add atera --- .../combine/process_datasets_xenium_nebius.sh | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/scripts/create_resources/combine/process_datasets_xenium_nebius.sh b/scripts/create_resources/combine/process_datasets_xenium_nebius.sh index a630053f1..259208fd5 100644 --- a/scripts/create_resources/combine/process_datasets_xenium_nebius.sh +++ b/scripts/create_resources/combine/process_datasets_xenium_nebius.sh @@ -129,16 +129,6 @@ param_list: dataset_description: "Xenium V1 FFPE Human Breast IDC + 2021 Wu scRNAseq" dataset_organism: "homo_sapiens" - - id: "2026_10x_human_breast_cancer_atera_combined" - input_sp: "$input_dir/10x_atera/2026_10x_human_breast_cancer_atera/dataset.zarr" - input_sc: "$input_dir/wu_human_breast_cancer_sc/2021Wu_human_breast_cancer_sc/dataset.h5ad" - dataset_id: "2026_10x_human_breast_cancer_atera_combined" - dataset_name: "Human breast cancer combined 2026 10x Atera WTA 2021 Wu scRNAseq" - dataset_url: "https://www.10xgenomics.com/datasets/atera-wta-ffpe-human-breast-cancer" - dataset_reference: "https://doi.org/10.1038/s41588-021-00911-1" - dataset_summary: "Atera WTA FFPE Human Breast Cancer (DCIS Grade 3) + 2021 Wu scRNAseq" - dataset_description: "Atera WTA FFPE Human Breast Cancer (DCIS Grade 3) + 2021 Wu scRNAseq" - dataset_organism: "homo_sapiens" output_sc: "\$id/output_sc.h5ad" output_sp: "\$id/output_sp.zarr" From cf67e092e1d1888c8f10216224a529e9393e03f7 Mon Sep 17 00:00:00 2001 From: dariarom94 Date: Mon, 27 Jul 2026 21:51:43 +0200 Subject: [PATCH 40/60] add a test in pciseq and dynamic memory for bruker --- src/datasets/loaders/bruker_cosmx/script.py | 68 ++++++++++++++----- .../pciseq/script.py | 17 ++++- 2 files changed, 67 insertions(+), 18 deletions(-) diff --git a/src/datasets/loaders/bruker_cosmx/script.py b/src/datasets/loaders/bruker_cosmx/script.py index 883913f29..902ccdc4b 100644 --- a/src/datasets/loaders/bruker_cosmx/script.py +++ b/src/datasets/loaders/bruker_cosmx/script.py @@ -118,39 +118,73 @@ def log(msg): TMP_DIR = Path(meta["temp_dir"] or "/tmp") TMP_DIR.mkdir(parents=True, exist_ok=True) -def extract_zip(input_zip: Path, output_dir: Path, strip_root: bool = False): +# CosMx export directories that sopa's cosmx reader never touches. Skipping them on +# extraction is what makes the mouse-brain hemisphere fit in scratch: the raw zip is +# ~208 GB, but ~190 GB of that is Morphology3D (per-z-plane 3D stacks — sopa uses the +# 2D Morphology2D composite) plus AnalysisResults (decoding CSVs) and RunSummary (logs). +# What's left (Morphology2D + FOV*/CellLabels + the flat CSVs) is ~15 GB. Extracting the +# whole thing next to the staged zip overruns the node's ~220 GB scratch and OOMs (137). +SKIP_DIRS = {"Morphology3D", "AnalysisResults", "RunSummary"} + +def _member_wanted(name: str) -> bool: + parts = name.split("/") + if name.startswith("__MACOSX/") or parts[-1] == ".DS_Store": + return False + return not any(part in SKIP_DIRS for part in parts) + +def _open_zip_source(src): + """Seekable binary handle for a local path or an s3:// URL (streamed, never fully downloaded).""" + s = str(src) + if s.startswith("s3://"): + import s3fs + # openproblems-data is public; anonymous read is deterministic and needs no creds. + fs = s3fs.S3FileSystem(anon=True, default_block_size=32 * 2**20) + return fs.open(s, "rb") + return open(s, "rb") + +def extract_zip(input_zip, output_dir: Path, strip_root: bool = False): """ - Extracts the input zip file to the specified output directory. + Stream a zip (local path or s3:// URL) into output_dir, skipping the CosMx + directories sopa never reads (see SKIP_DIRS). Only the wanted members are + transferred, so a remote zip streams just the bytes we keep — it is never + downloaded in full first. Arguments: - - - input_zip: Path to the input zip file. - - output_dir: Path to the directory where the contents of the zip file will be extracted - - strip_root: If True, and if all entries in the zip file share exactly one top-level directory, then this top-level directory will be stripped when extracting. + + - input_zip: local path or s3:// URL of the input zip. + - output_dir: directory where the (filtered) contents are written. + - strip_root: if True and all entries share exactly one top-level directory, strip it. """ output_dir = Path(output_dir) - with zipfile.ZipFile(input_zip, 'r') as zip_ref: + kept = skipped = 0 + kept_bytes = 0 + with _open_zip_source(input_zip) as fh, zipfile.ZipFile(fh, "r") as zip_ref: members = zip_ref.infolist() # only strip when all entries share exactly one top-level directory roots = {Path(m.filename).parts[0] for m in members if m.filename.strip("/") and not m.filename.startswith("__MACOSX/")} - if not (strip_root and len(roots) == 1): - zip_ref.extractall(output_dir) - return + do_strip = strip_root and len(roots) == 1 for member in members: - if member.filename.startswith("__MACOSX/"): - continue # skip macOS resource-fork entries, don't recreate them - parts = Path(member.filename).parts[1:] + if not _member_wanted(member.filename): + skipped += 1 + continue + parts = Path(member.filename).parts + if do_strip: + parts = parts[1:] if not parts: continue # the wrapper directory entry itself target = output_dir.joinpath(*parts) if member.is_dir(): target.mkdir(parents=True, exist_ok=True) - else: - target.parent.mkdir(parents=True, exist_ok=True) - with zip_ref.open(member) as src, open(target, "wb") as dst: - shutil.copyfileobj(src, dst) + continue + target.parent.mkdir(parents=True, exist_ok=True) + with zip_ref.open(member) as src, open(target, "wb") as dst: + shutil.copyfileobj(src, dst) + kept += 1 + kept_bytes += member.file_size + log(f"Extracted {kept} files ({kept_bytes / 1e9:.1f} GB); skipped {skipped} members " + f"(dirs {sorted(SKIP_DIRS)} + macOS cruft)") # Extract zip files log("Extract zip of raw files") diff --git a/src/methods_transcript_assignment/pciseq/script.py b/src/methods_transcript_assignment/pciseq/script.py index f3001003c..1156a833d 100644 --- a/src/methods_transcript_assignment/pciseq/script.py +++ b/src/methods_transcript_assignment/pciseq/script.py @@ -131,10 +131,25 @@ def eta_update_no_assert(self): #same as before if isinstance(sdata_segm["segmentation"], xr.DataTree): - label_image = sdata_segm["segmentation"]["scale0"].image.to_numpy() + label_image = sdata_segm["segmentation"]["scale0"].image.to_numpy() else: label_image = sdata_segm["segmentation"].to_numpy() +# Guard against an empty segmentation. pciSeq builds `coo = coo_matrix(label_image)` and +# then calls `coo.data.max()` in stage_data; when the label image has no cells (all zeros) +# `coo.data` is a zero-size array and numpy raises the cryptic +# "zero-size array to reduction operation maximum which has no identity". +# Fail fast with an actionable message instead. This happens when the upstream `cell_labels` +# copied by custom_segmentation is empty (observed for the vizgen lung-cancer merscope dataset, +# whose polygon->label rasterization produced an all-zero image). +if not (label_image > 0).any(): + raise ValueError( + f"Segmentation '{par['input_segmentation']}' contains no cells: the label image " + f"(shape={label_image.shape}, dtype={label_image.dtype}) is entirely zero. pciSeq " + f"cannot assign transcripts without any cell labels. Check the upstream segmentation " + f"(for custom_segmentation this means the dataset's `cell_labels` is empty)." + ) + # pciSeq internally drops spots that fall OUTSIDE the label image, and returns # per-spot assignments only for the kept spots; txsim.run_pciSeq then does # `spots['cell'] = assignments`, which raises "Length of values ... does not match From 9f71692e948a4d2747a5943df4a340316551562c Mon Sep 17 00:00:00 2001 From: dariarom94 Date: Tue, 28 Jul 2026 23:39:34 +0200 Subject: [PATCH 41/60] add test to vizgen data --- .../loaders/vizgen_merscope/script.py | 136 +++++++++++------- .../pciseq/script.py | 2 +- 2 files changed, 84 insertions(+), 54 deletions(-) diff --git a/src/datasets/loaders/vizgen_merscope/script.py b/src/datasets/loaders/vizgen_merscope/script.py index 5bf8e8182..d3251cb95 100644 --- a/src/datasets/loaders/vizgen_merscope/script.py +++ b/src/datasets/loaders/vizgen_merscope/script.py @@ -4,8 +4,10 @@ from datetime import datetime from pathlib import Path +import dask.array as da import geopandas as gpd import h5py +import numpy as np import pandas as pd import spatialdata as sd import spatialdata_io as sdio @@ -44,47 +46,50 @@ ) +def _cell_polygon(cell_grp): + """Boundary for one cell: prefer zIndex_3, else any available z-plane. + + MERSCOPE stores a cell's boundary only at the z-planes where it appears, so keying on a + single hardcoded z-index (the old ``["zIndex_3"]``) drops -- or KeyErrors on -- every cell + not present at that plane. Returns None if the cell has no usable boundary. + """ + z_keys = list(cell_grp.keys()) + z = "zIndex_3" if "zIndex_3" in z_keys else (z_keys[0] if z_keys else None) + if z is None or "p_0" not in cell_grp[z] or "coordinates" not in cell_grp[z]["p_0"]: + return None + return MultiPolygon([Polygon(cell_grp[z]["p_0"]["coordinates"][()][0])]) + + def read_boundary_hdf5(folder): print(datetime.now() - t0, "Convert boundary hdf5 to parquet", flush=True) - all_boundaries = {} - boundaries = None - hdf5_files = os.listdir(folder + "/cell_boundaries/") + hdf5_files = [ + f for f in os.listdir(folder + "/cell_boundaries/") if f.endswith(".hdf5") + ] n_files = len(hdf5_files) - incr = n_files // 15 - for _, i in enumerate(hdf5_files): - if (_ % incr) == 0: - print(datetime.now() - t0, f"\tProcessed {_}/{n_files}", flush=True) - with h5py.File(folder + "/cell_boundaries/" + i, "r") as f: - for key in f["featuredata"].keys(): - if boundaries is not None: - boundaries.loc[key] = MultiPolygon( - [ - Polygon( - f["featuredata"][key]["zIndex_3"]["p_0"]["coordinates"][ - () - ][0] - ) - ] - ) # doesn't matter which zIndex we use, MultiPolygon to work with read function in spatialdata-io - else: - # boundaries = gpd.GeoDataFrame(index=[key], geometry=MultiPolygon([Polygon(f['featuredata'][key]['zIndex_3']['p_0']['coordinates'][()][0])])) - boundaries = gpd.GeoDataFrame( - geometry=gpd.GeoSeries( - MultiPolygon( - [ - Polygon( - f["featuredata"][key]["zIndex_3"]["p_0"][ - "coordinates" - ][()][0] - ) - ] - ), - index=[key], - ) - ) - all_boundaries[i] = boundaries - boundaries = None - all_concat = pd.concat(list(all_boundaries.values())) + incr = max(1, n_files // 15) # guard against ZeroDivision when <15 files + geoms, index, n_skipped = [], [], 0 + for n, fname in enumerate(hdf5_files): + if n % incr == 0: + print(datetime.now() - t0, f"\tProcessed {n}/{n_files}", flush=True) + with h5py.File(folder + "/cell_boundaries/" + fname, "r") as f: + featuredata = f["featuredata"] + for key in featuredata.keys(): + poly = _cell_polygon(featuredata[key]) + if poly is None: + n_skipped += 1 + continue + geoms.append(poly) + index.append(key) + # n_skipped is logged (never silently dropped) so a re-run reveals whether low boundary + # coverage is a reader problem or a genuine property of the raw hdf5. + print( + datetime.now() - t0, + f"Read {len(geoms)} cell boundaries from {n_files} files " + f"({n_skipped} cells had no usable z-plane boundary)", + flush=True, + ) + # Build the GeoDataFrame in one pass; the old row-by-row `.loc` growth was O(n^2) over ~10^5 cells. + all_concat = gpd.GeoDataFrame(geometry=gpd.GeoSeries(geoms, index=index)) all_concat = all_concat[ ~all_concat.index.duplicated(keep="first") ] # hdf5 can contain duplicates with same cell_id and position, removing those @@ -241,23 +246,48 @@ def read_boundary_hdf5(folder): "return_regions_as_labels": True, } -for i in range(n_iter): - print( - datetime.now() - t0, - f"Rasterizing iteration {i + 1}/{n_iter} (cells {i * N}..{min((i + 1) * N, n_cells)})", - flush=True, +if n_iter == 1: + # Common case (every current dataset has <65535 cells): a single rasterize call. + # Keep the lazy DataArray exactly as before. + print(datetime.now() - t0, "Rasterizing all cells in a single pass", flush=True) + labels_image = sd.rasterize( + sdata["cell_boundaries"], ["x", "y"], **rasterize_args ) - labels_image_ = sd.rasterize( - sdata["cell_boundaries"].iloc[i * N : min((i + 1) * N, n_cells)], - ["x", "y"], - **rasterize_args, +else: + # >65535 cells: rasterize in chunks and merge. Two subtleties make the naive in-place + # merge wrong (it was silently broken before; this path had never run on real data since + # no dataset exceeds 65535 cells): + # 1. sd.rasterize returns a *lazy* (dask-backed) DataArray whose `.values` is a computed + # copy, so the old `labels_image.values[mask] = ...` write never persisted. We + # materialise each chunk and merge in a plain numpy array instead. + # 2. return_regions_as_labels labels each chunk 1..len(chunk) *positionally*, so chunks + # would collide on the same label. We offset each chunk by its global start index + # (and use uint32, since labels exceed uint16 past 65535 cells). + combined = None + template = None + for i in range(n_iter): + start = i * N + end = min((i + 1) * N, n_cells) + print( + datetime.now() - t0, + f"Rasterizing iteration {i + 1}/{n_iter} (cells {start}..{end})", + flush=True, + ) + chunk = sd.rasterize( + sdata["cell_boundaries"].iloc[start:end], ["x", "y"], **rasterize_args + ) + chunk_np = np.asarray(chunk.data) + if combined is None: + combined = chunk_np.astype("uint32") + template = chunk + else: + mask = chunk_np > 0 + combined[mask] = chunk_np[mask].astype("uint32") + start + # Rebuild a dask-backed labels element (spatialdata rejects a numpy-backed one) while + # preserving the template's transform and attrs (incl. label_index_to_category). + labels_image = template.copy( + data=da.from_array(combined, chunks=template.data.chunksize) ) - if i == 0: - labels_image = labels_image_ - else: - labels_image.values[labels_image_.values > 0] = labels_image_.values[ - labels_image_.values > 0 - ] print(datetime.now() - t0, "Rasterization finished", flush=True) try: diff --git a/src/methods_transcript_assignment/pciseq/script.py b/src/methods_transcript_assignment/pciseq/script.py index 1156a833d..264c61718 100644 --- a/src/methods_transcript_assignment/pciseq/script.py +++ b/src/methods_transcript_assignment/pciseq/script.py @@ -142,7 +142,7 @@ def eta_update_no_assert(self): # Fail fast with an actionable message instead. This happens when the upstream `cell_labels` # copied by custom_segmentation is empty (observed for the vizgen lung-cancer merscope dataset, # whose polygon->label rasterization produced an all-zero image). -if not (label_image > 0).any(): +if label_image.max() == 0: raise ValueError( f"Segmentation '{par['input_segmentation']}' contains no cells: the label image " f"(shape={label_image.shape}, dtype={label_image.dtype}) is entirely zero. pciSeq " From fefaadc77bec2f169e82081254982d6fc3adc9fa Mon Sep 17 00:00:00 2001 From: dariarom94 Date: Tue, 28 Jul 2026 23:53:46 +0200 Subject: [PATCH 42/60] param sweep --- .../param_sweep/binning_params.yaml | 39 +++++++ .../param_sweep/cellpose_params.yaml | 67 +++++++++++ .../param_sweep/cellposev4_params.yaml | 47 ++++++++ .../param_sweep/run_test_binning_local.sh | 88 ++++++++++++++ .../param_sweep/run_test_binning_nebius.sh | 109 ++++++++++++++++++ .../param_sweep/run_test_cellpose_local.sh | 86 ++++++++++++++ .../param_sweep/run_test_cellpose_nebius.sh | 107 +++++++++++++++++ .../param_sweep/run_test_cellposev4_local.sh | 84 ++++++++++++++ .../param_sweep/run_test_cellposev4_nebius.sh | 105 +++++++++++++++++ .../param_sweep/run_test_stardist_local.sh | 85 ++++++++++++++ .../param_sweep/run_test_stardist_nebius.sh | 105 +++++++++++++++++ .../param_sweep/run_test_watershed_local.sh | 85 ++++++++++++++ .../param_sweep/run_test_watershed_nebius.sh | 105 +++++++++++++++++ .../param_sweep/stardist_params.yaml | 43 +++++++ .../param_sweep/watershed_params.yaml | 36 ++++++ 15 files changed, 1191 insertions(+) create mode 100644 scripts/run_benchmark/param_sweep/binning_params.yaml create mode 100644 scripts/run_benchmark/param_sweep/cellpose_params.yaml create mode 100644 scripts/run_benchmark/param_sweep/cellposev4_params.yaml create mode 100644 scripts/run_benchmark/param_sweep/run_test_binning_local.sh create mode 100644 scripts/run_benchmark/param_sweep/run_test_binning_nebius.sh create mode 100644 scripts/run_benchmark/param_sweep/run_test_cellpose_local.sh create mode 100644 scripts/run_benchmark/param_sweep/run_test_cellpose_nebius.sh create mode 100644 scripts/run_benchmark/param_sweep/run_test_cellposev4_local.sh create mode 100644 scripts/run_benchmark/param_sweep/run_test_cellposev4_nebius.sh create mode 100644 scripts/run_benchmark/param_sweep/run_test_stardist_local.sh create mode 100644 scripts/run_benchmark/param_sweep/run_test_stardist_nebius.sh create mode 100644 scripts/run_benchmark/param_sweep/run_test_watershed_local.sh create mode 100644 scripts/run_benchmark/param_sweep/run_test_watershed_nebius.sh create mode 100644 scripts/run_benchmark/param_sweep/stardist_params.yaml create mode 100644 scripts/run_benchmark/param_sweep/watershed_params.yaml diff --git a/scripts/run_benchmark/param_sweep/binning_params.yaml b/scripts/run_benchmark/param_sweep/binning_params.yaml new file mode 100644 index 000000000..7ca9ec45e --- /dev/null +++ b/scripts/run_benchmark/param_sweep/binning_params.yaml @@ -0,0 +1,39 @@ +# Parameter sweep for the binning segmentation method (the "poor segmentation" +# baseline: a fixed square grid of pseudo-cells, no image content used). +# +# Shared source of truth for both run_test_binning_local.sh (read as a local file) +# and run_test_binning_nebius.sh (read from GitHub via a raw URL, since the Nebius +# compute env pulls the repo but cannot see the launch host's local files). +# +# Consumed by the run_benchmark workflow via the `method_parameters_yaml` setting +# (src/workflows/run_benchmark/main.nf). For every method the workflow builds: +# * one "default" variant using the `default:` args below, and +# * one extra variant per value in each `sweep:` list, with that ONE arg overridden. +# The benchmark varies a SINGLE parameter at a time (a "star" around the default, not +# a full grid), so total binning variants = 1 default + sum(sweep list lengths) = 6. +# +# See src/methods_segmentation/binning/NOTES.md ("Optimization / tuning") for the +# rationale. The ONLY knob is the bin edge length. We sweep it in MICRONS +# (--bin_size_um), the physically meaningful unit, rather than raw pixels: the +# component converts um -> pixels from the image's coordinate transform. On this +# benchmark's standardized raw_ist grid (~1 um/pixel, target_unit_to_pixels=1) the +# two coincide, so a value in um is ~ the same value in px. +# +# NOTE: --bin_size_um is a NEWLY EXPOSED argument. It needs `viash ns build` + +# a binning container rebuild before it takes effect (see check-component). Until +# then, swap `bin_size_um` for the pre-existing `bin_size` (pixels) below. +parameters: + binning: + # Baseline == the shipped 30 px default, expressed in microns (~30 um on the + # 1 um/px grid): bins ~2-3 cell diameters across -> coarse pseudo-cells. + default: + bin_size_um: 30.0 + sweep: + # bin_size_um: bin edge length in microns. Grounded in the ~1 um/px grid and + # brain/Xenium cell scale (nuclei ~5-8 um, whole cells ~10-15 um): + # 10 -> sub-cell / nucleus scale, many bins per cell (over-segmentation) + # 15 -> ~one whole cell per bin (best case for a fixed grid) + # 20 -> slightly coarse + # (30 = default, omitted here to avoid a duplicate variant) + # 40, 50 -> several cells per bin (strong under-segmentation) + bin_size_um: [10.0, 15.0, 20.0, 40.0, 50.0] diff --git a/scripts/run_benchmark/param_sweep/cellpose_params.yaml b/scripts/run_benchmark/param_sweep/cellpose_params.yaml new file mode 100644 index 000000000..3887fb1e7 --- /dev/null +++ b/scripts/run_benchmark/param_sweep/cellpose_params.yaml @@ -0,0 +1,67 @@ +# Parameter sweep for the cellpose (Cellpose v3, cyto/nuclei CNN) segmentation method. +# Shared source of truth for both run_test_cellpose_local.sh (read as a local file) +# and run_test_cellpose_nebius.sh (read from GitHub via a raw URL, since the Nebius +# compute env pulls the repo but cannot see the launch host's local files). +# +# Consumed by the run_benchmark workflow via the `method_parameters_yaml` setting +# (src/workflows/run_benchmark/main.nf). For every method the workflow builds: +# * one "default" variant using the `default:` args below, and +# * one extra variant per value in each `sweep:` list, with that ONE arg overridden. +# The benchmark varies a SINGLE parameter at a time (a "star" around the default, not +# a full grid), so total cellpose variants = 1 default + sum(sweep list lengths) = 20. +# +# See src/methods_segmentation/cellpose/NOTES.md ("Optimization / tuning") for the +# rationale, tiers, and defaults. Unlike cellposev4, this component's shipped defaults +# already equal Cellpose's own library defaults (quality-oriented) EXCEPT model_type +# (cyto vs library cyto3), so this sweep explores model choice + object scale + +# recall/precision dials rather than "walking back" speed-tuned defaults. +# +# NOTE: as of txsim@dev the v3 path runs on CPU (the wrapper never passes gpu=), so +# these 20 variants x the full downstream pipeline are slow; niter: 50 probes the +# speed lever. Trim the lists if the budget is tight. +parameters: + cellpose: + # Baseline == the component's shipped defaults (== Cellpose library defaults, + # except model_type). Applied to every variant; the "default variant" is exactly + # this point, so none of these values are repeated in the sweep lists below. + default: + model_type: cyto + diameter: 30.0 + flow_threshold: 0.4 + cellprob_threshold: 0.0 + min_size: 15 + resample: true + normalize: true + niter: 0 + sweep: + # ---- Tier 1: highest impact on segmentation quality ---- + # model_type: the most distinctive v3 knob (v4 has no model choice). On a single + # grayscale morphology plane the model matters: nuclei for a nuclear stain, cyto2 + # (Cellpose 2.0) and cyto3 (Cellpose3 generalist) as improved generalists over + # the 2021 cyto default. All four are pre-cached in the image (config docker.run). + model_type: [nuclei, cyto2, cyto3] + # diameter: cellpose rescales so objects land near diam_mean (~30 px cyto). + # Xenium morphology (~0.2125 um/px) => nuclei ~38 px, whole cells ~60-70 px. + # 0.0 = auto-estimate via the SizeModel (slower). 30.0 = default (omitted). + diameter: [0.0, 40.0, 60.0] + # cellprob_threshold: recall<->precision dial (per-pixel logit ~ -6..+6, def 0), + # no speed cost. Lower -> recover more/dimmer cells; higher -> drop dim detections. + cellprob_threshold: [-2.0, -1.0, 1.0, 2.0] + # flow_threshold: flow-error QC (def 0.4). 0.2 = stricter shape filter; + # 0.6/0.8 = more permissive (keep cells with higher flow error -> more cells). + flow_threshold: [0.2, 0.6, 0.8] + # ---- Tier 2: quality/speed trade-offs ---- + # min_size: def 15. 0 keeps small specks (recall on tiny cells); 50 drops debris. + min_size: [0, 50] + # resample: BOOLEAN, default true (the quality setting). Only non-default value + # is false -> dynamics on the downsampled grid (faster, coarser boundaries). + resample: [false] + # augment: BOOLEAN, default false. Only non-default is true -> test-time + # augmentation (accuracy ceiling at ~4-8x cost). + augment: [true] + # normalize: BOOLEAN, default true. Only non-default is false -> no percentile + # normalization (probes intensity sensitivity; usually worse for iST). + normalize: [false] + # niter: def 0 = auto (~200 at resample). 50 = fewer dynamics iterations -> + # faster (meaningful because the v3 path is CPU-bound). + niter: [50] diff --git a/scripts/run_benchmark/param_sweep/cellposev4_params.yaml b/scripts/run_benchmark/param_sweep/cellposev4_params.yaml new file mode 100644 index 000000000..6681b0f89 --- /dev/null +++ b/scripts/run_benchmark/param_sweep/cellposev4_params.yaml @@ -0,0 +1,47 @@ +# Parameter sweep for the cellposev4 (Cellpose 4 / Cellpose-SAM) segmentation method. +# Shared source of truth for both run_test_cellposev4_local.sh (read as a local file) +# and run_test_cellposev4_nebius.sh (read from GitHub via a raw URL, since the Nebius +# compute env pulls the repo but cannot see the launch host's local files). +# +# Consumed by the run_benchmark workflow via the `method_parameters_yaml` setting +# (src/workflows/run_benchmark/main.nf). For every method the workflow builds: +# * one "default" variant using the `default:` args below, and +# * one extra variant per value in each `sweep:` list, with that ONE arg overridden. +# The benchmark varies a SINGLE parameter at a time (a "star" around the default, not +# a full grid), so total cellposev4 variants = 1 default + sum(sweep list lengths) = 19. +# +# See src/methods_segmentation/cellposev4/NOTES.md ("Optimization / tuning") for the +# rationale, tiers, and defaults. Trim or widen the lists to taste. +parameters: + cellposev4: + # Baseline == the component's shipped (deliberately speed-tuned) defaults. + default: + diameter: 30.0 + flow_threshold: 0.0 + cellprob_threshold: 0.0 + niter: 10 + min_size: -1 + resample: false + sweep: + # ---- Tier 1: highest impact on segmentation quality ---- + # diameter: cellpose rescales so objects land near ~30 px (training mean), + # so 30.0 == "no rescaling". Xenium morphology (~0.2125 um/px) => nuclei + # ~35-40 px, whole cells ~60-70 px. Span small nuclei -> whole cells. + diameter: [15.0, 20.0, 40.0, 60.0] + # cellprob_threshold: recall<->precision dial (network logit ~ -6..+6, def 0). + # Lower -> recover more/dimmer cells; higher -> drop dim-region detections. + cellprob_threshold: [-3.0, -1.0, 1.0, 3.0] + # flow_threshold: 0.0 (our default) disables flow-shape QC (keeps ill-shaped + # masks); >0 re-enables it, more permissive as it rises. 0.4 = Cellpose default. + flow_threshold: [0.4, 0.6, 0.8] + # ---- Tier 2: quality/speed trade-offs ---- + # niter: flow-dynamics iterations; 10 (default) is aggressively low and may + # under-converge large/elongated cells. Cellpose's own default is ~diameter- + # proportional (often ~200). + niter: [50, 100, 200] + # resample: BOOLEAN, default false -> the only meaningful non-default value is + # true (run dynamics at full resolution: smoother boundaries, slower). + resample: [true] + # min_size: -1 (default) disables small-mask removal; positive values drop + # progressively larger specks/debris. + min_size: [15, 30, 50] diff --git a/scripts/run_benchmark/param_sweep/run_test_binning_local.sh b/scripts/run_benchmark/param_sweep/run_test_binning_local.sh new file mode 100644 index 000000000..3afe8980e --- /dev/null +++ b/scripts/run_benchmark/param_sweep/run_test_binning_local.sh @@ -0,0 +1,88 @@ +#!/bin/bash + +# Test run: all default methods + binning segmentation, with a parameter sweep +# over the one binning knob (bin edge length, in microns). +# See src/methods_segmentation/binning/NOTES.md ("Optimization / tuning"). +# +# NOTE: binning is a light CPU-only method (a numpy grid, no model, no image +# content used), so it runs comfortably on any Docker host. It is the deliberately +# "poor" segmentation baseline. +# +# NOTE: the sweep uses --bin_size_um, a NEWLY EXPOSED argument. Run +# 'viash ns build --setup cachedbuild' (or scripts/project/build_all_docker_containers.sh) +# so the regenerated binning container knows the arg before launching. + +# get the root of the directory +REPO_ROOT=$(git rev-parse --show-toplevel) + +# ensure that the command below is run from the root of the repository +cd "$REPO_ROOT" + +set -e + +echo "Running benchmark on test data (defaults + binning sweep)" +echo " Make sure to run 'scripts/project/build_all_docker_containers.sh'!" + +# generate a unique id +RUN_ID="testrun_binning_$(date +%Y-%m-%d_%H-%M-%S)" +publish_dir="temp/results/${RUN_ID}" + +cat > /tmp/params_settings.yaml << HERE +default_methods: + - custom_segmentation + - basic_transcript_assignment + - basic_count_aggregation + - basic_qc_filter + - alpha_shapes + - normalize_by_volume + - tacco + - no_correction +segmentation_methods: + - custom_segmentation + - binning + # - cellpose + # - cellposev4 + # - stardist + # - watershed +transcript_assignment_methods: + - basic_transcript_assignment +count_aggregation_methods: + - basic_count_aggregation +qc_filtering_methods: + - basic_qc_filter +volume_calculation_methods: + - alpha_shapes +normalization_methods: + - normalize_by_volume +celltype_annotation_methods: + - ssam +expression_correction_methods: + - no_correction +gene_efficiency_correction_methods: + - no_correction +method_parameters_yaml: $REPO_ROOT/scripts/run_benchmark/binning_params.yaml +HERE + +# Write the parameters to file (input_states version, NOTE: enable `-entry auto` for this) +cat > /tmp/params.yaml << HERE +input_states: resources_test/task_ist_preprocessing/**/state.yaml +rename_keys: 'input_sc:output_sc;input_sp:output_sp' +save_spatial_data: false +settings: '$(yq -o json /tmp/params_settings.yaml | jq -c .)' +output_state: "state.yaml" +publish_dir: "$publish_dir" +HERE + +# The binning parameter sweep lives in a committed file (single source of truth, +# shared with run_test_binning_nebius.sh): scripts/run_benchmark/binning_params.yaml +# It defines a `default:` variant + one-arg-at-a-time `sweep:` variants (a "star" +# around the default, not a grid) => 6 binning variants. Edit that file to change +# the sweep. Referenced via $REPO_ROOT above so local Nextflow reads it directly. + +nextflow run . \ + -main-script target/nextflow/workflows/run_benchmark/main.nf \ + -profile docker \ + -resume \ + -entry auto \ + -c common/nextflow_helpers/labels_ci.config \ + -params-file /tmp/params.yaml diff --git a/scripts/run_benchmark/param_sweep/run_test_binning_nebius.sh b/scripts/run_benchmark/param_sweep/run_test_binning_nebius.sh new file mode 100644 index 000000000..647e73f33 --- /dev/null +++ b/scripts/run_benchmark/param_sweep/run_test_binning_nebius.sh @@ -0,0 +1,109 @@ +#!/bin/bash + +# Nebius test run: all default methods + binning segmentation, with a parameter +# sweep over the one binning knob (bin edge length, in microns). +# See src/methods_segmentation/binning/NOTES.md ("Optimization / tuning"). +# Local sibling: run_test_binning_local.sh +# +# binning is a light CPU-only method (a numpy grid, no model), so it runs on the +# standard (non-GPU) compute env with no `gpu` label. +# +# NOTE: the sweep uses --bin_size_um, a NEWLY EXPOSED argument. The image on +# ghcr must be rebuilt (viash ns build + container rebuild, revision build/main) +# so the binning container knows the arg before launching (see check-component). +# +# PARAMS-FILE CAVEAT (why this differs from the local script): +# `tw launch --params-file` is read client-side, but `method_parameters_yaml` +# is a path the WORKFLOW opens at runtime on the cloud (readYaml -> Nextflow +# file()). A local /tmp path does not exist there, and /scratch (where results +# publish) is READ-ONLY from the launch host — which is why the binning +# method_params block is commented out in run_test_nebius.sh. file() does stage +# http(s):// though, and this repo is public, so we keep the sweep in a COMMITTED +# file (scripts/run_benchmark/binning_params.yaml) and read it from GitHub via +# its raw URL. => the params file must be committed AND PUSHED to $params_branch +# before launching (edit the file there, not here, to change the sweep). + +# get the root of the directory +REPO_ROOT=$(git rev-parse --show-toplevel) + +# ensure that the command below is run from the root of the repository +cd "$REPO_ROOT" + +set -e + +resources_test_s3=s3://openproblems-data/resources_test/task_ist_preprocessing +# Results publish to /scratch — created and written by the cloud compute env, so +# the launcher does NOT create it here (it is read-only from the launch host). +publish_dir="/scratch/results/runs/$(date +%Y-%m-%d_%H-%M-%S)_binning" + +# The sweep lives in a committed file, read from GitHub at runtime. $params_branch +# defaults to the branch you are on; the file must be pushed there on GitHub. (This +# is independent of --revision below, which selects the pipeline CODE to run.) +params_repo="openproblems-bio/task_ist_preprocessing" +params_branch="$(git rev-parse --abbrev-ref HEAD)" +params_url="https://raw.githubusercontent.com/${params_repo}/${params_branch}/scripts/run_benchmark/binning_params.yaml" + +cat > /tmp/params_settings.yaml << HERE +default_methods: + - custom_segmentation + - basic_transcript_assignment + - basic_count_aggregation + - basic_qc_filter + - alpha_shapes + - normalize_by_volume + - tacco + - no_correction +segmentation_methods: + - custom_segmentation + - binning +# - cellpose +# - cellposev4 +# - stardist +# - watershed +transcript_assignment_methods: + - basic_transcript_assignment +count_aggregation_methods: + - basic_count_aggregation +qc_filtering_methods: + - basic_qc_filter +volume_calculation_methods: + - alpha_shapes +normalization_methods: + - normalize_by_volume +celltype_annotation_methods: + - tacco +expression_correction_methods: + - no_correction +gene_efficiency_correction_methods: + - no_correction +method_parameters_yaml: $params_url +HERE + +# Write the parameters to file (input_states version, NOTE: enable `-entry_name auto` for this) +cat > /tmp/params.yaml << HERE +input_states: $resources_test_s3/**/state.yaml +rename_keys: 'input_sc:output_sc;input_sp:output_sp' +save_spatial_data: false +settings: '$(yq -o json /tmp/params_settings.yaml | jq -c .)' +output_state: "state.yaml" +publish_dir: "$publish_dir" +HERE + +# Fail early with a clear message if the params file isn't reachable on GitHub yet. +if ! curl -fsSL -o /dev/null "$params_url"; then + echo "ERROR: params file not reachable at:" >&2 + echo " $params_url" >&2 + echo "Commit and push scripts/run_benchmark/binning_params.yaml to '$params_branch' first." >&2 + exit 1 +fi + +tw launch https://github.com/openproblems-bio/task_ist_preprocessing.git \ + --revision build/main \ + --pull-latest \ + --main-script target/nextflow/workflows/run_benchmark/main.nf \ + --workspace 167877437119966 \ + --compute-env 5hfmdCBxMRd4nHZaJKYEQZ \ + --params-file /tmp/params.yaml \ + --entry-name auto \ + --config src/base/labels_nebius.config \ + --labels task_ist_preprocessing,test,binning diff --git a/scripts/run_benchmark/param_sweep/run_test_cellpose_local.sh b/scripts/run_benchmark/param_sweep/run_test_cellpose_local.sh new file mode 100644 index 000000000..19ed5de22 --- /dev/null +++ b/scripts/run_benchmark/param_sweep/run_test_cellpose_local.sh @@ -0,0 +1,86 @@ +#!/bin/bash + +# Test run: all default methods + Cellpose v3 (cellpose, cyto/nuclei CNN) +# segmentation, with a parameter sweep over the optimization levers identified +# for Cellpose v3. See src/methods_segmentation/cellpose/NOTES.md +# ("Optimization / tuning"). +# +# NOTE: despite the component's gpuhighmem label, the v3 path runs on CPU (the +# txsim wrapper never passes gpu= -> models.Cellpose defaults to gpu=False). So +# these 20 variants x the full downstream pipeline are CPU-bound and slow; the +# sweep includes niter: 50 as a speed probe. See the GPU gotcha in NOTES.md. + +# get the root of the directory +REPO_ROOT=$(git rev-parse --show-toplevel) + +# ensure that the command below is run from the root of the repository +cd "$REPO_ROOT" + +set -e + +echo "Running benchmark on test data (defaults + cellpose v3 sweep)" +echo " Make sure to run 'scripts/project/build_all_docker_containers.sh'!" + +# generate a unique id +RUN_ID="testrun_cellpose_$(date +%Y-%m-%d_%H-%M-%S)" +publish_dir="temp/results/${RUN_ID}" + +cat > /tmp/params_settings.yaml << HERE +default_methods: + - custom_segmentation + - basic_transcript_assignment + - basic_count_aggregation + - basic_qc_filter + - alpha_shapes + - normalize_by_volume + - tacco + - no_correction +segmentation_methods: + - custom_segmentation + - cellpose + # - cellposev4 + # - binning + # - stardist + # - watershed +transcript_assignment_methods: + - basic_transcript_assignment +count_aggregation_methods: + - basic_count_aggregation +qc_filtering_methods: + - basic_qc_filter +volume_calculation_methods: + - alpha_shapes +normalization_methods: + - normalize_by_volume +celltype_annotation_methods: + - ssam +expression_correction_methods: + - no_correction +gene_efficiency_correction_methods: + - no_correction +method_parameters_yaml: $REPO_ROOT/scripts/run_benchmark/cellpose_params.yaml +HERE + +# Write the parameters to file (input_states version, NOTE: enable `-entry auto` for this) +cat > /tmp/params.yaml << HERE +input_states: resources_test/task_ist_preprocessing/**/state.yaml +rename_keys: 'input_sc:output_sc;input_sp:output_sp' +save_spatial_data: false +settings: '$(yq -o json /tmp/params_settings.yaml | jq -c .)' +output_state: "state.yaml" +publish_dir: "$publish_dir" +HERE + +# The cellpose parameter sweep lives in a committed file (single source of truth, +# shared with run_test_cellpose_nebius.sh): scripts/run_benchmark/cellpose_params.yaml +# It defines a `default:` variant + one-arg-at-a-time `sweep:` variants (a "star" +# around the default, not a grid) => 20 cellpose variants. Edit that file to +# change the sweep. Referenced via $REPO_ROOT above so local Nextflow reads it directly. + +nextflow run . \ + -main-script target/nextflow/workflows/run_benchmark/main.nf \ + -profile docker \ + -resume \ + -entry auto \ + -c common/nextflow_helpers/labels_ci.config \ + -params-file /tmp/params.yaml diff --git a/scripts/run_benchmark/param_sweep/run_test_cellpose_nebius.sh b/scripts/run_benchmark/param_sweep/run_test_cellpose_nebius.sh new file mode 100644 index 000000000..781da777b --- /dev/null +++ b/scripts/run_benchmark/param_sweep/run_test_cellpose_nebius.sh @@ -0,0 +1,107 @@ +#!/bin/bash + +# Nebius test run: all default methods + Cellpose v3 (cellpose, cyto/nuclei CNN) +# segmentation, with a parameter sweep over the optimization levers identified +# for Cellpose v3. See src/methods_segmentation/cellpose/NOTES.md +# ("Optimization / tuning"). Local sibling: run_test_cellpose_local.sh +# +# NOTE: the component carries a gpuhighmem label (so it is scheduled onto a GPU +# node), but as of txsim@dev the v3 code path runs on CPU (the wrapper never +# passes gpu=). The `gpu` run label below matches the config's scheduling; it +# does NOT make the model GPU-accelerated. See the GPU gotcha in NOTES.md. +# +# PARAMS-FILE CAVEAT (why this differs from the local script): +# `tw launch --params-file` is read client-side, but `method_parameters_yaml` +# is a path the WORKFLOW opens at runtime on the cloud (readYaml -> Nextflow +# file()). A local /tmp path does not exist there, and /scratch (where results +# publish) is READ-ONLY from the launch host — which is why the binning +# method_params block is commented out in run_test_nebius.sh. file() does stage +# http(s):// though, and this repo is public, so we keep the sweep in a COMMITTED +# file (scripts/run_benchmark/cellpose_params.yaml) and read it from GitHub via +# its raw URL. => the params file must be committed AND PUSHED to $params_branch +# before launching (edit the file there, not here, to change the sweep). + +# get the root of the directory +REPO_ROOT=$(git rev-parse --show-toplevel) + +# ensure that the command below is run from the root of the repository +cd "$REPO_ROOT" + +set -e + +resources_test_s3=s3://openproblems-data/resources_test/task_ist_preprocessing +# Results publish to /scratch — created and written by the cloud compute env, so +# the launcher does NOT create it here (it is read-only from the launch host). +publish_dir="/scratch/results/runs/$(date +%Y-%m-%d_%H-%M-%S)_cellpose" + +# The sweep lives in a committed file, read from GitHub at runtime. $params_branch +# defaults to the branch you are on; the file must be pushed there on GitHub. (This +# is independent of --revision below, which selects the pipeline CODE to run.) +params_repo="openproblems-bio/task_ist_preprocessing" +params_branch="$(git rev-parse --abbrev-ref HEAD)" +params_url="https://raw.githubusercontent.com/${params_repo}/${params_branch}/scripts/run_benchmark/cellpose_params.yaml" + +cat > /tmp/params_settings.yaml << HERE +default_methods: + - custom_segmentation + - basic_transcript_assignment + - basic_count_aggregation + - basic_qc_filter + - alpha_shapes + - normalize_by_volume + - tacco + - no_correction +segmentation_methods: + - custom_segmentation + - cellpose +# - cellposev4 +# - binning +# - stardist +# - watershed +transcript_assignment_methods: + - basic_transcript_assignment +count_aggregation_methods: + - basic_count_aggregation +qc_filtering_methods: + - basic_qc_filter +volume_calculation_methods: + - alpha_shapes +normalization_methods: + - normalize_by_volume +celltype_annotation_methods: + - tacco +expression_correction_methods: + - no_correction +gene_efficiency_correction_methods: + - no_correction +method_parameters_yaml: $params_url +HERE + +# Write the parameters to file (input_states version, NOTE: enable `-entry_name auto` for this) +cat > /tmp/params.yaml << HERE +input_states: $resources_test_s3/**/state.yaml +rename_keys: 'input_sc:output_sc;input_sp:output_sp' +save_spatial_data: false +settings: '$(yq -o json /tmp/params_settings.yaml | jq -c .)' +output_state: "state.yaml" +publish_dir: "$publish_dir" +HERE + +# Fail early with a clear message if the params file isn't reachable on GitHub yet. +if ! curl -fsSL -o /dev/null "$params_url"; then + echo "ERROR: params file not reachable at:" >&2 + echo " $params_url" >&2 + echo "Commit and push scripts/run_benchmark/cellpose_params.yaml to '$params_branch' first." >&2 + exit 1 +fi + +tw launch https://github.com/openproblems-bio/task_ist_preprocessing.git \ + --revision build/main \ + --pull-latest \ + --main-script target/nextflow/workflows/run_benchmark/main.nf \ + --workspace 167877437119966 \ + --compute-env 5hfmdCBxMRd4nHZaJKYEQZ \ + --params-file /tmp/params.yaml \ + --entry-name auto \ + --config src/base/labels_nebius.config \ + --labels task_ist_preprocessing,test,cellpose,gpu diff --git a/scripts/run_benchmark/param_sweep/run_test_cellposev4_local.sh b/scripts/run_benchmark/param_sweep/run_test_cellposev4_local.sh new file mode 100644 index 000000000..9f2f923f0 --- /dev/null +++ b/scripts/run_benchmark/param_sweep/run_test_cellposev4_local.sh @@ -0,0 +1,84 @@ +#!/bin/bash + +# Test run: all default methods + Cellpose 4 (cellposev4) segmentation, with a +# parameter sweep over the optimization levers identified for Cellpose-SAM. +# See src/methods_segmentation/cellposev4/NOTES.md ("Optimization / tuning"). +# +# NOTE: cellposev4 (cpsam / SAM ViT) is GPU-heavy. Run on a CUDA-capable Docker +# host; on CPU it is very slow (the documented reason it is off the standard CI +# test path). + +# get the root of the directory +REPO_ROOT=$(git rev-parse --show-toplevel) + +# ensure that the command below is run from the root of the repository +cd "$REPO_ROOT" + +set -e + +echo "Running benchmark on test data (defaults + cellposev4 sweep)" +echo " Make sure to run 'scripts/project/build_all_docker_containers.sh'!" + +# generate a unique id +RUN_ID="testrun_cellposev4_$(date +%Y-%m-%d_%H-%M-%S)" +publish_dir="temp/results/${RUN_ID}" + +cat > /tmp/params_settings.yaml << HERE +default_methods: + - custom_segmentation + - basic_transcript_assignment + - basic_count_aggregation + - basic_qc_filter + - alpha_shapes + - normalize_by_volume + - tacco + - no_correction +segmentation_methods: + - custom_segmentation + - cellposev4 + # - cellpose + # - binning + # - stardist + # - watershed +transcript_assignment_methods: + - basic_transcript_assignment +count_aggregation_methods: + - basic_count_aggregation +qc_filtering_methods: + - basic_qc_filter +volume_calculation_methods: + - alpha_shapes +normalization_methods: + - normalize_by_volume +celltype_annotation_methods: + - ssam +expression_correction_methods: + - no_correction +gene_efficiency_correction_methods: + - no_correction +method_parameters_yaml: $REPO_ROOT/scripts/run_benchmark/cellposev4_params.yaml +HERE + +# Write the parameters to file (input_states version, NOTE: enable `-entry auto` for this) +cat > /tmp/params.yaml << HERE +input_states: resources_test/task_ist_preprocessing/**/state.yaml +rename_keys: 'input_sc:output_sc;input_sp:output_sp' +save_spatial_data: false +settings: '$(yq -o json /tmp/params_settings.yaml | jq -c .)' +output_state: "state.yaml" +publish_dir: "$publish_dir" +HERE + +# The cellposev4 parameter sweep lives in a committed file (single source of truth, +# shared with run_test_cellposev4_nebius.sh): scripts/run_benchmark/cellposev4_params.yaml +# It defines a `default:` variant + one-arg-at-a-time `sweep:` variants (a "star" +# around the default, not a grid) => 19 cellposev4 variants. Edit that file to +# change the sweep. Referenced via $REPO_ROOT above so local Nextflow reads it directly. + +nextflow run . \ + -main-script target/nextflow/workflows/run_benchmark/main.nf \ + -profile docker \ + -resume \ + -entry auto \ + -c common/nextflow_helpers/labels_ci.config \ + -params-file /tmp/params.yaml diff --git a/scripts/run_benchmark/param_sweep/run_test_cellposev4_nebius.sh b/scripts/run_benchmark/param_sweep/run_test_cellposev4_nebius.sh new file mode 100644 index 000000000..5c092195e --- /dev/null +++ b/scripts/run_benchmark/param_sweep/run_test_cellposev4_nebius.sh @@ -0,0 +1,105 @@ +#!/bin/bash + +# Nebius test run: all default methods + Cellpose 4 (cellposev4) segmentation, +# with a parameter sweep over the optimization levers identified for Cellpose-SAM. +# See src/methods_segmentation/cellposev4/NOTES.md ("Optimization / tuning"). +# Local sibling: run_test_cellposev4_local.sh +# +# cellposev4 (cpsam / SAM ViT) is GPU-heavy, hence the `gpu` label and the Nebius +# GPU compute env. +# +# PARAMS-FILE CAVEAT (why this differs from the local script): +# `tw launch --params-file` is read client-side, but `method_parameters_yaml` +# is a path the WORKFLOW opens at runtime on the cloud (readYaml -> Nextflow +# file()). A local /tmp path does not exist there, and /scratch (where results +# publish) is READ-ONLY from the launch host — which is why the binning +# method_params block is commented out in run_test_nebius.sh. file() does stage +# http(s):// though, and this repo is public, so we keep the sweep in a COMMITTED +# file (scripts/run_benchmark/cellposev4_params.yaml) and read it from GitHub via +# its raw URL. => the params file must be committed AND PUSHED to $params_branch +# before launching (edit the file there, not here, to change the sweep). + +# get the root of the directory +REPO_ROOT=$(git rev-parse --show-toplevel) + +# ensure that the command below is run from the root of the repository +cd "$REPO_ROOT" + +set -e + +resources_test_s3=s3://openproblems-data/resources_test/task_ist_preprocessing +# Results publish to /scratch — created and written by the cloud compute env, so +# the launcher does NOT create it here (it is read-only from the launch host). +publish_dir="/scratch/results/runs/$(date +%Y-%m-%d_%H-%M-%S)_cellposev4" + +# The sweep lives in a committed file, read from GitHub at runtime. $params_branch +# defaults to the branch you are on; the file must be pushed there on GitHub. (This +# is independent of --revision below, which selects the pipeline CODE to run.) +params_repo="openproblems-bio/task_ist_preprocessing" +params_branch="$(git rev-parse --abbrev-ref HEAD)" +params_url="https://raw.githubusercontent.com/${params_repo}/${params_branch}/scripts/run_benchmark/cellposev4_params.yaml" + +cat > /tmp/params_settings.yaml << HERE +default_methods: + - custom_segmentation + - basic_transcript_assignment + - basic_count_aggregation + - basic_qc_filter + - alpha_shapes + - normalize_by_volume + - tacco + - no_correction +segmentation_methods: + - custom_segmentation + - cellposev4 +# - cellpose +# - binning +# - stardist +# - watershed +transcript_assignment_methods: + - basic_transcript_assignment +count_aggregation_methods: + - basic_count_aggregation +qc_filtering_methods: + - basic_qc_filter +volume_calculation_methods: + - alpha_shapes +normalization_methods: + - normalize_by_volume +celltype_annotation_methods: + - tacco +expression_correction_methods: + - no_correction +gene_efficiency_correction_methods: + - no_correction +method_parameters_yaml: $params_url +HERE + +# Write the parameters to file (input_states version, NOTE: enable `-entry_name auto` for this) +cat > /tmp/params.yaml << HERE +input_states: $resources_test_s3/**/state.yaml +rename_keys: 'input_sc:output_sc;input_sp:output_sp' +save_spatial_data: false +settings: '$(yq -o json /tmp/params_settings.yaml | jq -c .)' +output_state: "state.yaml" +publish_dir: "$publish_dir" +HERE + +# Fail early with a clear message if the params file isn't reachable on GitHub yet. +if ! curl -fsSL -o /dev/null "$params_url"; then + echo "ERROR: params file not reachable at:" >&2 + echo " $params_url" >&2 + echo "Commit and push scripts/run_benchmark/cellposev4_params.yaml to '$params_branch' first." >&2 + exit 1 +fi + +tw launch https://github.com/openproblems-bio/task_ist_preprocessing.git \ + --revision build/main \ + --pull-latest \ + --main-script target/nextflow/workflows/run_benchmark/main.nf \ + --workspace 167877437119966 \ + --compute-env 5hfmdCBxMRd4nHZaJKYEQZ \ + --params-file /tmp/params.yaml \ + --entry-name auto \ + --config src/base/labels_nebius.config \ + --labels task_ist_preprocessing,test,cellposev4,gpu diff --git a/scripts/run_benchmark/param_sweep/run_test_stardist_local.sh b/scripts/run_benchmark/param_sweep/run_test_stardist_local.sh new file mode 100644 index 000000000..e6b168151 --- /dev/null +++ b/scripts/run_benchmark/param_sweep/run_test_stardist_local.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +# Test run: all default methods + StarDist2D (stardist) segmentation, with a +# parameter sweep over the optimization levers identified for StarDist. +# See src/methods_segmentation/stardist/NOTES.md ("Optimization / tuning"). +# +# NOTE: stardist runs on TensorFlow via a GPU-capable base image +# (openproblems/base_tensorflow_nvidia). It falls back to CPU if no GPU is present +# (slower, but far lighter than cellposev4's SAM ViT). Run on a CUDA-capable Docker +# host for the full sweep. + +# get the root of the directory +REPO_ROOT=$(git rev-parse --show-toplevel) + +# ensure that the command below is run from the root of the repository +cd "$REPO_ROOT" + +set -e + +echo "Running benchmark on test data (defaults + stardist sweep)" +echo " Make sure to run 'scripts/project/build_all_docker_containers.sh'!" + +# generate a unique id +RUN_ID="testrun_stardist_$(date +%Y-%m-%d_%H-%M-%S)" +publish_dir="temp/results/${RUN_ID}" + +cat > /tmp/params_settings.yaml << HERE +default_methods: + - custom_segmentation + - basic_transcript_assignment + - basic_count_aggregation + - basic_qc_filter + - alpha_shapes + - normalize_by_volume + - tacco + - no_correction +segmentation_methods: + - custom_segmentation + - stardist + # - cellpose + # - cellposev4 + # - binning + # - watershed +transcript_assignment_methods: + - basic_transcript_assignment +count_aggregation_methods: + - basic_count_aggregation +qc_filtering_methods: + - basic_qc_filter +volume_calculation_methods: + - alpha_shapes +normalization_methods: + - normalize_by_volume +celltype_annotation_methods: + - ssam +expression_correction_methods: + - no_correction +gene_efficiency_correction_methods: + - no_correction +method_parameters_yaml: $REPO_ROOT/scripts/run_benchmark/stardist_params.yaml +HERE + +# Write the parameters to file (input_states version, NOTE: enable `-entry auto` for this) +cat > /tmp/params.yaml << HERE +input_states: resources_test/task_ist_preprocessing/**/state.yaml +rename_keys: 'input_sc:output_sc;input_sp:output_sp' +save_spatial_data: false +settings: '$(yq -o json /tmp/params_settings.yaml | jq -c .)' +output_state: "state.yaml" +publish_dir: "$publish_dir" +HERE + +# The stardist parameter sweep lives in a committed file (single source of truth, +# shared with run_test_stardist_nebius.sh): scripts/run_benchmark/stardist_params.yaml +# It defines a `default:` variant + one-arg-at-a-time `sweep:` variants (a "star" +# around the default, not a grid) => 13 stardist variants. Edit that file to change +# the sweep. Referenced via $REPO_ROOT above so local Nextflow reads it directly. + +nextflow run . \ + -main-script target/nextflow/workflows/run_benchmark/main.nf \ + -profile docker \ + -resume \ + -entry auto \ + -c common/nextflow_helpers/labels_ci.config \ + -params-file /tmp/params.yaml diff --git a/scripts/run_benchmark/param_sweep/run_test_stardist_nebius.sh b/scripts/run_benchmark/param_sweep/run_test_stardist_nebius.sh new file mode 100644 index 000000000..fe58738e3 --- /dev/null +++ b/scripts/run_benchmark/param_sweep/run_test_stardist_nebius.sh @@ -0,0 +1,105 @@ +#!/bin/bash + +# Nebius test run: all default methods + StarDist2D (stardist) segmentation, with a +# parameter sweep over the optimization levers identified for StarDist. +# See src/methods_segmentation/stardist/NOTES.md ("Optimization / tuning"). +# Local sibling: run_test_stardist_local.sh +# +# stardist runs on TensorFlow and is scheduled with the `gpu` label (its config uses +# a GPU-capable base image), hence the Nebius GPU compute env. +# +# PARAMS-FILE CAVEAT (why this differs from the local script): +# `tw launch --params-file` is read client-side, but `method_parameters_yaml` +# is a path the WORKFLOW opens at runtime on the cloud (readYaml -> Nextflow +# file()). A local /tmp path does not exist there, and /scratch (where results +# publish) is READ-ONLY from the launch host — which is why the binning +# method_params block is commented out in run_test_nebius.sh. file() does stage +# http(s):// though, and this repo is public, so we keep the sweep in a COMMITTED +# file (scripts/run_benchmark/stardist_params.yaml) and read it from GitHub via +# its raw URL. => the params file must be committed AND PUSHED to $params_branch +# before launching (edit the file there, not here, to change the sweep). + +# get the root of the directory +REPO_ROOT=$(git rev-parse --show-toplevel) + +# ensure that the command below is run from the root of the repository +cd "$REPO_ROOT" + +set -e + +resources_test_s3=s3://openproblems-data/resources_test/task_ist_preprocessing +# Results publish to /scratch — created and written by the cloud compute env, so +# the launcher does NOT create it here (it is read-only from the launch host). +publish_dir="/scratch/results/runs/$(date +%Y-%m-%d_%H-%M-%S)_stardist" + +# The sweep lives in a committed file, read from GitHub at runtime. $params_branch +# defaults to the branch you are on; the file must be pushed there on GitHub. (This +# is independent of --revision below, which selects the pipeline CODE to run.) +params_repo="openproblems-bio/task_ist_preprocessing" +params_branch="$(git rev-parse --abbrev-ref HEAD)" +params_url="https://raw.githubusercontent.com/${params_repo}/${params_branch}/scripts/run_benchmark/stardist_params.yaml" + +cat > /tmp/params_settings.yaml << HERE +default_methods: + - custom_segmentation + - basic_transcript_assignment + - basic_count_aggregation + - basic_qc_filter + - alpha_shapes + - normalize_by_volume + - tacco + - no_correction +segmentation_methods: + - custom_segmentation + - stardist +# - cellpose +# - cellposev4 +# - binning +# - watershed +transcript_assignment_methods: + - basic_transcript_assignment +count_aggregation_methods: + - basic_count_aggregation +qc_filtering_methods: + - basic_qc_filter +volume_calculation_methods: + - alpha_shapes +normalization_methods: + - normalize_by_volume +celltype_annotation_methods: + - tacco +expression_correction_methods: + - no_correction +gene_efficiency_correction_methods: + - no_correction +method_parameters_yaml: $params_url +HERE + +# Write the parameters to file (input_states version, NOTE: enable `-entry_name auto` for this) +cat > /tmp/params.yaml << HERE +input_states: $resources_test_s3/**/state.yaml +rename_keys: 'input_sc:output_sc;input_sp:output_sp' +save_spatial_data: false +settings: '$(yq -o json /tmp/params_settings.yaml | jq -c .)' +output_state: "state.yaml" +publish_dir: "$publish_dir" +HERE + +# Fail early with a clear message if the params file isn't reachable on GitHub yet. +if ! curl -fsSL -o /dev/null "$params_url"; then + echo "ERROR: params file not reachable at:" >&2 + echo " $params_url" >&2 + echo "Commit and push scripts/run_benchmark/stardist_params.yaml to '$params_branch' first." >&2 + exit 1 +fi + +tw launch https://github.com/openproblems-bio/task_ist_preprocessing.git \ + --revision build/main \ + --pull-latest \ + --main-script target/nextflow/workflows/run_benchmark/main.nf \ + --workspace 167877437119966 \ + --compute-env 5hfmdCBxMRd4nHZaJKYEQZ \ + --params-file /tmp/params.yaml \ + --entry-name auto \ + --config src/base/labels_nebius.config \ + --labels task_ist_preprocessing,test,stardist,gpu diff --git a/scripts/run_benchmark/param_sweep/run_test_watershed_local.sh b/scripts/run_benchmark/param_sweep/run_test_watershed_local.sh new file mode 100644 index 000000000..7ea207252 --- /dev/null +++ b/scripts/run_benchmark/param_sweep/run_test_watershed_local.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +# Test run: all default methods + watershed segmentation, with a parameter sweep +# over the Tier-1 optimization levers identified for classic marker-controlled +# watershed. See src/methods_segmentation/watershed/NOTES.md ("Optimization / +# tuning"). +# +# watershed is a CPU method (no GPU needed): the txsim classic pipeline +# normalize -> contrast -> blur -> threshold -> post-process -> distance transform +# -> local-maxima markers -> watershed -> background filter. + +# get the root of the directory +REPO_ROOT=$(git rev-parse --show-toplevel) + +# ensure that the command below is run from the root of the repository +cd "$REPO_ROOT" + +set -e + +echo "Running benchmark on test data (defaults + watershed sweep)" +echo " Make sure to run 'scripts/project/build_all_docker_containers.sh'!" + +# generate a unique id +RUN_ID="testrun_watershed_$(date +%Y-%m-%d_%H-%M-%S)" +publish_dir="temp/results/${RUN_ID}" + +cat > /tmp/params_settings.yaml << HERE +default_methods: + - custom_segmentation + - basic_transcript_assignment + - basic_count_aggregation + - basic_qc_filter + - alpha_shapes + - normalize_by_volume + - tacco + - no_correction +segmentation_methods: + - custom_segmentation + - watershed + # - cellpose + # - cellposev4 + # - binning + # - stardist +transcript_assignment_methods: + - basic_transcript_assignment +count_aggregation_methods: + - basic_count_aggregation +qc_filtering_methods: + - basic_qc_filter +volume_calculation_methods: + - alpha_shapes +normalization_methods: + - normalize_by_volume +celltype_annotation_methods: + - ssam +expression_correction_methods: + - no_correction +gene_efficiency_correction_methods: + - no_correction +method_parameters_yaml: $REPO_ROOT/scripts/run_benchmark/watershed_params.yaml +HERE + +# Write the parameters to file (input_states version, NOTE: enable `-entry auto` for this) +cat > /tmp/params.yaml << HERE +input_states: resources_test/task_ist_preprocessing/**/state.yaml +rename_keys: 'input_sc:output_sc;input_sp:output_sp' +save_spatial_data: false +settings: '$(yq -o json /tmp/params_settings.yaml | jq -c .)' +output_state: "state.yaml" +publish_dir: "$publish_dir" +HERE + +# The watershed parameter sweep lives in a committed file (single source of truth, +# shared with run_test_watershed_nebius.sh): scripts/run_benchmark/watershed_params.yaml +# It defines a `default:` variant + one-arg-at-a-time `sweep:` variants (a "star" +# around the default, not a grid) => 12 watershed variants. Edit that file to +# change the sweep. Referenced via $REPO_ROOT above so local Nextflow reads it directly. + +nextflow run . \ + -main-script target/nextflow/workflows/run_benchmark/main.nf \ + -profile docker \ + -resume \ + -entry auto \ + -c common/nextflow_helpers/labels_ci.config \ + -params-file /tmp/params.yaml diff --git a/scripts/run_benchmark/param_sweep/run_test_watershed_nebius.sh b/scripts/run_benchmark/param_sweep/run_test_watershed_nebius.sh new file mode 100644 index 000000000..697767dfb --- /dev/null +++ b/scripts/run_benchmark/param_sweep/run_test_watershed_nebius.sh @@ -0,0 +1,105 @@ +#!/bin/bash + +# Nebius test run: all default methods + watershed segmentation, with a parameter +# sweep over the Tier-1 optimization levers identified for classic marker-controlled +# watershed. See src/methods_segmentation/watershed/NOTES.md ("Optimization / tuning"). +# Local sibling: run_test_watershed_local.sh +# +# watershed is a CPU method (midcpu/midmem/hightime) — NO gpu label, and it reuses +# the standard (non-GPU) Nebius test compute env. +# +# PARAMS-FILE CAVEAT (why this differs from the local script): +# `tw launch --params-file` is read client-side, but `method_parameters_yaml` +# is a path the WORKFLOW opens at runtime on the cloud (readYaml -> Nextflow +# file()). A local /tmp path does not exist there, and /scratch (where results +# publish) is READ-ONLY from the launch host — which is why the binning +# method_params block is commented out in run_test_nebius.sh. file() does stage +# http(s):// though, and this repo is public, so we keep the sweep in a COMMITTED +# file (scripts/run_benchmark/watershed_params.yaml) and read it from GitHub via +# its raw URL. => the params file must be committed AND PUSHED to $params_branch +# before launching (edit the file there, not here, to change the sweep). + +# get the root of the directory +REPO_ROOT=$(git rev-parse --show-toplevel) + +# ensure that the command below is run from the root of the repository +cd "$REPO_ROOT" + +set -e + +resources_test_s3=s3://openproblems-data/resources_test/task_ist_preprocessing +# Results publish to /scratch — created and written by the cloud compute env, so +# the launcher does NOT create it here (it is read-only from the launch host). +publish_dir="/scratch/results/runs/$(date +%Y-%m-%d_%H-%M-%S)_watershed" + +# The sweep lives in a committed file, read from GitHub at runtime. $params_branch +# defaults to the branch you are on; the file must be pushed there on GitHub. (This +# is independent of --revision below, which selects the pipeline CODE to run.) +params_repo="openproblems-bio/task_ist_preprocessing" +params_branch="$(git rev-parse --abbrev-ref HEAD)" +params_url="https://raw.githubusercontent.com/${params_repo}/${params_branch}/scripts/run_benchmark/watershed_params.yaml" + +cat > /tmp/params_settings.yaml << HERE +default_methods: + - custom_segmentation + - basic_transcript_assignment + - basic_count_aggregation + - basic_qc_filter + - alpha_shapes + - normalize_by_volume + - tacco + - no_correction +segmentation_methods: + - custom_segmentation + - watershed +# - cellpose +# - cellposev4 +# - binning +# - stardist +transcript_assignment_methods: + - basic_transcript_assignment +count_aggregation_methods: + - basic_count_aggregation +qc_filtering_methods: + - basic_qc_filter +volume_calculation_methods: + - alpha_shapes +normalization_methods: + - normalize_by_volume +celltype_annotation_methods: + - tacco +expression_correction_methods: + - no_correction +gene_efficiency_correction_methods: + - no_correction +method_parameters_yaml: $params_url +HERE + +# Write the parameters to file (input_states version, NOTE: enable `-entry_name auto` for this) +cat > /tmp/params.yaml << HERE +input_states: $resources_test_s3/**/state.yaml +rename_keys: 'input_sc:output_sc;input_sp:output_sp' +save_spatial_data: false +settings: '$(yq -o json /tmp/params_settings.yaml | jq -c .)' +output_state: "state.yaml" +publish_dir: "$publish_dir" +HERE + +# Fail early with a clear message if the params file isn't reachable on GitHub yet. +if ! curl -fsSL -o /dev/null "$params_url"; then + echo "ERROR: params file not reachable at:" >&2 + echo " $params_url" >&2 + echo "Commit and push scripts/run_benchmark/watershed_params.yaml to '$params_branch' first." >&2 + exit 1 +fi + +tw launch https://github.com/openproblems-bio/task_ist_preprocessing.git \ + --revision build/main \ + --pull-latest \ + --main-script target/nextflow/workflows/run_benchmark/main.nf \ + --workspace 167877437119966 \ + --compute-env 5hfmdCBxMRd4nHZaJKYEQZ \ + --params-file /tmp/params.yaml \ + --entry-name auto \ + --config src/base/labels_nebius.config \ + --labels task_ist_preprocessing,test,watershed diff --git a/scripts/run_benchmark/param_sweep/stardist_params.yaml b/scripts/run_benchmark/param_sweep/stardist_params.yaml new file mode 100644 index 000000000..434c251a1 --- /dev/null +++ b/scripts/run_benchmark/param_sweep/stardist_params.yaml @@ -0,0 +1,43 @@ +# Parameter sweep for the stardist (StarDist2D) segmentation method. +# Shared source of truth for both run_test_stardist_local.sh (read as a local file) +# and run_test_stardist_nebius.sh (read from GitHub via a raw URL, since the Nebius +# compute env pulls the repo but cannot see the launch host's local files). +# +# Consumed by the run_benchmark workflow via the `method_parameters_yaml` setting +# (src/workflows/run_benchmark/main.nf). For every method the workflow builds: +# * one "default" variant using the `default:` args below, and +# * one extra variant per value in each `sweep:` list, with that ONE arg overridden. +# The benchmark varies a SINGLE parameter at a time (a "star" around the default, not +# a full grid), so total stardist variants = 1 default + sum(sweep list lengths) = 13. +# +# See src/methods_segmentation/stardist/NOTES.md ("Optimization / tuning") for the +# rationale, tiers, and defaults. prob_thresh / nms_thresh / scale are the newly +# exposed StarDist knobs (need `viash ns build` + a container rebuild to take effect). +parameters: + stardist: + # Baseline. Only `model` is set explicitly; prob_thresh / nms_thresh / scale are + # intentionally LEFT UNSET so the default variant uses the model's own optimized + # thresholds (0.479071 / 0.3 for 2D_versatile_fluo) and no rescaling — i.e. exactly + # today's production behaviour. Setting them to a fixed number would be wrong for a + # different --model, whose optimized thresholds differ. + default: + model: "2D_versatile_fluo" + sweep: + # ---- Tier 1: highest impact on segmentation quality ---- + # prob_thresh: object-probability threshold (default ~0.479 for 2D_versatile_fluo). + # Pure recall<->precision dial. LOWER -> recover more / dimmer nuclei; HIGHER -> + # keep only confident detections. Span both sides of the default. + prob_thresh: [0.3, 0.4, 0.6, 0.7] + # scale: isotropic rescale before prediction (default = none / 1.0). StarDist's + # analogue of Cellpose's diameter — matches nucleus size to the model's training + # size. Xenium morphology (~0.2125 um/px) => ~8 um nucleus ~= 38 px; sweep down + # (downscale large nuclei) and up (upscale small nuclei). + scale: [0.5, 0.75, 1.5, 2.0] + # nms_thresh: non-max-suppression IoU for overlapping polygons (default 0.3). + # LOWER -> more aggressive suppression (merge/drop touching nuclei); HIGHER -> + # keep more overlapping detections in crowded tissue. + nms_thresh: [0.2, 0.4, 0.5] + # ---- Tier 1: model choice (already exposed) ---- + # 2D_paper_dsb2018 is the other fluorescence/nuclear pretrained model. 2D_versatile_he + # is H&E RGB (wrong for single-channel fluorescence) so it is NOT swept. + model: ["2D_paper_dsb2018"] diff --git a/scripts/run_benchmark/param_sweep/watershed_params.yaml b/scripts/run_benchmark/param_sweep/watershed_params.yaml new file mode 100644 index 000000000..65ac30343 --- /dev/null +++ b/scripts/run_benchmark/param_sweep/watershed_params.yaml @@ -0,0 +1,36 @@ +# Parameter sweep for the `watershed` segmentation method. +# Single source of truth for BOTH run_test_watershed_local.sh (read directly via +# $REPO_ROOT) and run_test_watershed_nebius.sh (read from GitHub via raw URL, so it +# must be committed AND pushed before a Nebius launch). +# +# Expansion is a STAR around `default:`, NOT a grid: one extra variant per sweep +# value, that one arg overridden, everything else = default. +# total variants = 1 (default) + Σ len(sweep list) = 1 + 2 + 3 + 3 + 3 = 12 +# +# The four swept knobs are the Tier-1 levers for classic watershed (see +# src/methods_segmentation/watershed/NOTES.md "Optimization / tuning"). Ranges are +# grounded in Xenium morphology (~0.2125 µm/px => an ~8 µm nucleus ≈ 37 px diameter, +# radius ≈ 18 px, area ≈ 1000 px²). The other ~40 classic-pipeline args stay at +# their component defaults and are intentionally NOT swept. +parameters: + watershed: + default: + threshold_func: local_otsu # adaptive local Otsu (rank.otsu, footprint 50 px) + blur_sigma: 1 # gaussian pre-smoothing sigma (px) + local_maxima_min_distance: 5 # min px between watershed seed markers + post_processing_min_size_1: 64 # remove_small_objects: min mask area (px) + sweep: + # foreground/background mask decision. local_otsu (default) is adaptive; the + # two alternatives use a single global threshold (triangle suits a dominant + # background peak, typical of fluorescence). + threshold_func: [otsu, triangle] + # heavier pre-smoothing suppresses spurious local maxima (fewer over-split + # cells) at the cost of blurring small/dim nuclei. 2-4 px stays sub-nucleus. + blur_sigma: [2, 3, 4] + # merge<->split dial: markers must be >= this many px apart. default 5 is far + # below the nucleus radius (~18 px) and over-splits single nuclei; walking up + # toward the radius merges fragments back into one cell. + local_maxima_min_distance: [10, 15, 20] + # precision dial: drop small foreground blobs before watershed. Stays well + # below a full nucleus area (~1000 px²) so real nuclei survive. + post_processing_min_size_1: [128, 256, 512] From 4dc08d3f03d5e21ff436c0f687034187e83763a3 Mon Sep 17 00:00:00 2001 From: dariarom94 Date: Tue, 28 Jul 2026 23:55:31 +0200 Subject: [PATCH 43/60] add params to segmentation --- .../spatial/process_bruker_cosmx_nebius.sh | 4 +- scripts/run_benchmark/cellposev4_params.yaml | 47 ------------------- .../binning/config.vsh.yaml | 14 ++++++ src/methods_segmentation/binning/script.py | 42 ++++++++++++++++- .../cellpose/config.vsh.yaml | 21 +++++++-- .../stardist/config.vsh.yaml | 32 +++++++++++++ src/methods_segmentation/stardist/script.py | 19 +++++++- .../watershed/config.vsh.yaml | 4 ++ src/methods_segmentation/watershed/script.py | 8 ++++ 9 files changed, 134 insertions(+), 57 deletions(-) delete mode 100644 scripts/run_benchmark/cellposev4_params.yaml diff --git a/scripts/create_resources/spatial/process_bruker_cosmx_nebius.sh b/scripts/create_resources/spatial/process_bruker_cosmx_nebius.sh index c12a28b1b..dd3232ef4 100644 --- a/scripts/create_resources/spatial/process_bruker_cosmx_nebius.sh +++ b/scripts/create_resources/spatial/process_bruker_cosmx_nebius.sh @@ -8,7 +8,9 @@ cd "$REPO_ROOT" set -e -publish_dir="s3://openproblems-data/resources/datasets" +# store the loader output locally, mirroring the process_datasets layout ($id/) +# under the sibling raw/ folder (same as the other *_nebius.sh spatial scripts) +publish_dir="/scratch/task_ist_preprocessing/raw" cat > /tmp/params.yaml << HERE param_list: diff --git a/scripts/run_benchmark/cellposev4_params.yaml b/scripts/run_benchmark/cellposev4_params.yaml deleted file mode 100644 index 6681b0f89..000000000 --- a/scripts/run_benchmark/cellposev4_params.yaml +++ /dev/null @@ -1,47 +0,0 @@ -# Parameter sweep for the cellposev4 (Cellpose 4 / Cellpose-SAM) segmentation method. -# Shared source of truth for both run_test_cellposev4_local.sh (read as a local file) -# and run_test_cellposev4_nebius.sh (read from GitHub via a raw URL, since the Nebius -# compute env pulls the repo but cannot see the launch host's local files). -# -# Consumed by the run_benchmark workflow via the `method_parameters_yaml` setting -# (src/workflows/run_benchmark/main.nf). For every method the workflow builds: -# * one "default" variant using the `default:` args below, and -# * one extra variant per value in each `sweep:` list, with that ONE arg overridden. -# The benchmark varies a SINGLE parameter at a time (a "star" around the default, not -# a full grid), so total cellposev4 variants = 1 default + sum(sweep list lengths) = 19. -# -# See src/methods_segmentation/cellposev4/NOTES.md ("Optimization / tuning") for the -# rationale, tiers, and defaults. Trim or widen the lists to taste. -parameters: - cellposev4: - # Baseline == the component's shipped (deliberately speed-tuned) defaults. - default: - diameter: 30.0 - flow_threshold: 0.0 - cellprob_threshold: 0.0 - niter: 10 - min_size: -1 - resample: false - sweep: - # ---- Tier 1: highest impact on segmentation quality ---- - # diameter: cellpose rescales so objects land near ~30 px (training mean), - # so 30.0 == "no rescaling". Xenium morphology (~0.2125 um/px) => nuclei - # ~35-40 px, whole cells ~60-70 px. Span small nuclei -> whole cells. - diameter: [15.0, 20.0, 40.0, 60.0] - # cellprob_threshold: recall<->precision dial (network logit ~ -6..+6, def 0). - # Lower -> recover more/dimmer cells; higher -> drop dim-region detections. - cellprob_threshold: [-3.0, -1.0, 1.0, 3.0] - # flow_threshold: 0.0 (our default) disables flow-shape QC (keeps ill-shaped - # masks); >0 re-enables it, more permissive as it rises. 0.4 = Cellpose default. - flow_threshold: [0.4, 0.6, 0.8] - # ---- Tier 2: quality/speed trade-offs ---- - # niter: flow-dynamics iterations; 10 (default) is aggressively low and may - # under-converge large/elongated cells. Cellpose's own default is ~diameter- - # proportional (often ~200). - niter: [50, 100, 200] - # resample: BOOLEAN, default false -> the only meaningful non-default value is - # true (run dynamics at full resolution: smoother boundaries, slower). - resample: [true] - # min_size: -1 (default) disables small-mask removal; positive values drop - # progressively larger specks/debris. - min_size: [15, 30, 50] diff --git a/src/methods_segmentation/binning/config.vsh.yaml b/src/methods_segmentation/binning/config.vsh.yaml index c9474b430..5bc586957 100644 --- a/src/methods_segmentation/binning/config.vsh.yaml +++ b/src/methods_segmentation/binning/config.vsh.yaml @@ -14,6 +14,20 @@ arguments: - name: --bin_size type: integer default: 30 + description: | + Edge length of each square pseudo-cell bin, in PIXELS of the input `image` + element. On this benchmark's standardized raw_ist grid (images are rasterized + at target_unit_to_pixels=1, i.e. ~1 um/pixel) 1 px is ~1 um, so 30 px is + ~30 um bins. Used only when --bin_size_um is not set. + - name: --bin_size_um + type: double + required: false + description: | + Edge length of each square bin in MICRONS. When set, it OVERRIDES --bin_size: + the script reads the input image's coordinate transform (um per pixel) and + converts to a pixel bin size, so the physical bin size is comparable across + datasets even if they are gridded at different pixel sizes. Leave unset to use + --bin_size (raw pixels). resources: - type: python_script diff --git a/src/methods_segmentation/binning/script.py b/src/methods_segmentation/binning/script.py index 49ead7e0b..ec8255600 100644 --- a/src/methods_segmentation/binning/script.py +++ b/src/methods_segmentation/binning/script.py @@ -24,10 +24,36 @@ def convert_to_lower_dtype(arr): return arr.astype(new_dtype) + +def pixel_size_um(image_element, coordinate_system="global"): + """Microns-per-pixel of an image element, read from its coordinate transform. + + On this benchmark's standardized raw_ist grid the images are rasterized at + target_unit_to_pixels=1 (~1 um/pixel), so this is ~1.0 for current datasets; + reading it keeps --bin_size_um correct if a dataset is ever gridded at a + different resolution. Falls back to 1.0 um/px (the standardized grid) if the + transform cannot be read, so a micron sweep still runs on current data. + """ + try: + transforms = image_element.transform # dict: coord_system -> transformation + trans = transforms.get(coordinate_system) or next(iter(transforms.values())) + aff = trans.to_affine_matrix(input_axes=("y", "x"), output_axes=("y", "x")) + sy, sx = abs(float(aff[0, 0])), abs(float(aff[1, 1])) + px = (sy + sx) / 2.0 + if not np.isfinite(px) or px <= 0: + raise ValueError(f"non-positive pixel size {px}") + return px + except Exception as e: + print(f"WARNING: could not read pixel size from transform ({e}); " + f"assuming 1.0 um/pixel (the standardized raw_ist grid).", flush=True) + return 1.0 + ## VIASH START par = { "input": "../task_ist_preprocessing/resources_test/common/2023_10x_mouse_brain_xenium/dataset.zarr", - "output": "segmentation.zarr" + "output": "segmentation.zarr", + "bin_size": 30, + "bin_size_um": None } ## VIASH END @@ -43,7 +69,19 @@ def convert_to_lower_dtype(arr): sd_output = sd.SpatialData() image = sdata['image']['scale0'].image.compute().to_numpy() transformation = sdata['image']['scale0'].image.transform.copy() -img_arr = tx.preprocessing.segment_binning(image[0], hyperparameters['bin_size']) ### TOdo find the optimal bin_size + +# bin_size is in PIXELS of the image grid. If bin_size_um is given, it overrides +# bin_size: convert the physical (micron) bin edge to pixels via the image's +# um-per-pixel scale, so the bin size is comparable across datasets. +if hyperparameters.get('bin_size_um') is not None: + px_um = pixel_size_um(sdata['image']['scale0'].image) + bin_size = max(1, int(round(float(hyperparameters['bin_size_um']) / px_um))) + print(f"bin_size_um={hyperparameters['bin_size_um']} um / {px_um} um/px " + f"-> bin_size={bin_size} px", flush=True) +else: + bin_size = int(hyperparameters['bin_size']) + +img_arr = tx.preprocessing.segment_binning(image[0], bin_size) image = convert_to_lower_dtype(img_arr) data_array = xr.DataArray(image, name=f'segmentation', dims=('y', 'x')) parsed_data = Labels2DModel.parse(data_array, transformations=transformation) diff --git a/src/methods_segmentation/cellpose/config.vsh.yaml b/src/methods_segmentation/cellpose/config.vsh.yaml index 4f982f3ed..57b6516ea 100644 --- a/src/methods_segmentation/cellpose/config.vsh.yaml +++ b/src/methods_segmentation/cellpose/config.vsh.yaml @@ -69,6 +69,14 @@ arguments: - name: --min_size type: integer default: 15 + - name: --niter + type: integer + default: 0 + description: | + Number of flow-dynamics iterations. 0 (Cellpose treats 0 or None the same) + lets Cellpose set it proportional to the diameter (~200 at resample=true). + Lower it (e.g. 50) to speed up the CPU path; raise it for large/elongated + cells that under-converge. Forwarded verbatim to model.eval. - name: --stitch_threshold type: double default: 0.0 @@ -84,13 +92,16 @@ engines: setup: - type: python pypi: cellpose<4.0.0 - # Pre-download the pretrained cyto weights into the image (~/.cellpose) - # so runtime tasks never hit the cellpose model server (avoids the - # intermittent "HTTP Error 504: Gateway Time-out" seen when several - # segmentation tasks fetch the weights concurrently). + # Pre-download the pretrained weights + size models into the image + # (~/.cellpose) so runtime tasks never hit the cellpose model server + # (avoids the intermittent "HTTP Error 504: Gateway Time-out" seen when + # several segmentation tasks fetch the weights concurrently). Instantiating + # Cellpose(model_type=m) caches both the mask model and its size model. + # All four builtins are cached because the --model_type sweep selects among + # them (see scripts/run_benchmark/cellpose_params.yaml). - type: docker run: - - "python -c \"from cellpose import models; models.Cellpose(gpu=False, model_type='cyto')\"" + - "python -c \"from cellpose import models; [models.Cellpose(gpu=False, model_type=m) for m in ['cyto','nuclei','cyto2','cyto3']]\"" __merge__: - /src/base/setup_txsim_partial.yaml - /src/base/setup_spatialdata_partial.yaml diff --git a/src/methods_segmentation/stardist/config.vsh.yaml b/src/methods_segmentation/stardist/config.vsh.yaml index 0330b270a..cba9c2272 100644 --- a/src/methods_segmentation/stardist/config.vsh.yaml +++ b/src/methods_segmentation/stardist/config.vsh.yaml @@ -15,6 +15,38 @@ arguments: - name: --model type: string default: "2D_versatile_fluo" + description: >- + Pretrained StarDist2D model loaded via StarDist2D.from_pretrained. + Fluorescence/nuclear models: "2D_versatile_fluo" (default, trained on a DSB2018 + subset) and "2D_paper_dsb2018". "2D_versatile_he" is a brightfield H&E RGB model + and is NOT appropriate for the single-channel fluorescence morphology images here. + # --- StarDist predict_instances knobs, forwarded through predict_instances_big --- + # All optional: when unset the model's own optimized values are used (thresholds.json + # for prob/nms, no rescaling for scale) — i.e. exactly the pre-tuning behaviour. + - name: --prob_thresh + type: double + required: false + description: >- + Object probability threshold; candidate pixels below it are discarded. If unset, + the model's optimized value is used (0.479071 for 2D_versatile_fluo). LOWER => more / + dimmer nuclei detected (higher recall); HIGHER => fewer, more confident detections + (higher precision). Valid range ~0..1. + - name: --nms_thresh + type: double + required: false + description: >- + Non-maximum-suppression IoU threshold for overlapping polygon candidates. If unset, + the model's optimized value is used (0.3 for 2D_versatile_fluo). LOWER => more + aggressive suppression (touching nuclei more likely merged/dropped); HIGHER => keeps + more overlapping detections. Valid range ~0..1. + - name: --scale + type: double + required: false + description: >- + Isotropic factor the image is rescaled by before prediction and undone afterwards. + If unset, no rescaling. Use it to match nucleus size to the model's training size: + >1 upscales (small nuclei appear larger), <1 downscales. StarDist's analogue of + Cellpose's diameter. resources: - type: python_script diff --git a/src/methods_segmentation/stardist/script.py b/src/methods_segmentation/stardist/script.py index 82d72ec4f..4bf502c61 100644 --- a/src/methods_segmentation/stardist/script.py +++ b/src/methods_segmentation/stardist/script.py @@ -39,7 +39,10 @@ def convert_to_lower_dtype(arr): par = { "input": "resources_test/task_ist_preprocessing/mouse_brain_combined/raw_ist.zarr", "output": "temp/stardist/segmentation.zarr", - "model": "2D_versatile_fluo" + "model": "2D_versatile_fluo", + "prob_thresh": None, + "nms_thresh": None, + "scale": None, } ## VIASH END @@ -75,8 +78,20 @@ def do_after(self): block_size = min(image.shape[1] // 3, 4096) offset = min(block_size // 5.5, 128) +# Tunable knobs forwarded through predict_instances_big -> predict_instances. +# A value left as None (i.e. omitted from par) means "use the model's own optimized +# value": thresholds.json for prob_thresh/nms_thresh, no rescaling for scale. This +# keeps the default (no-args) call identical to the pre-tuning behaviour. +eval_params = { + k: par[k] + for k in ("prob_thresh", "nms_thresh", "scale") + if par.get(k) is not None +} +print(f"predict_instances_big overrides: {eval_params}", flush=True) + labels, _ = model.predict_instances_big( - image[0,:,:], axes='YX', block_size=block_size, min_overlap=offset, context=offset, normalizer=normalizer#, n_tiles=(4,4) + image[0,:,:], axes='YX', block_size=block_size, min_overlap=offset, + context=offset, normalizer=normalizer, **eval_params # n_tiles left to block_size ) diff --git a/src/methods_segmentation/watershed/config.vsh.yaml b/src/methods_segmentation/watershed/config.vsh.yaml index cd554a435..54b25f622 100644 --- a/src/methods_segmentation/watershed/config.vsh.yaml +++ b/src/methods_segmentation/watershed/config.vsh.yaml @@ -156,6 +156,10 @@ arguments: - name: --bg_intensity_filter_bg_size type: integer default: 2000 +- name: --watershed_compactness + type: double + default: 0.0 + description: "Compactness passed to skimage.segmentation.watershed. 0.0 = standard marker-controlled watershed (unchanged default). Higher values (try 1e-3 to 1e-1) enable compact watershed (Neubert & Protzel), biasing basins toward rounder, more regular cell shapes. Forwarded via txsim's watershed_params dict (see script.py)." resources: - type: python_script diff --git a/src/methods_segmentation/watershed/script.py b/src/methods_segmentation/watershed/script.py index 198e30b3a..f52fbf05c 100644 --- a/src/methods_segmentation/watershed/script.py +++ b/src/methods_segmentation/watershed/script.py @@ -37,6 +37,14 @@ def convert_to_lower_dtype(arr): del hyperparameters['input'] del hyperparameters['output'] +# skimage.segmentation.watershed's `compactness` knob is only reachable through the +# nested `watershed_params` dict that txsim.segment_watershed forwards (it reads +# hyperparams.get("watershed_params", {})). Lift the exposed --watershed_compactness +# arg into that dict so it actually takes effect. compactness=0.0 == standard +# marker-controlled watershed, i.e. unchanged default behaviour. +compactness = hyperparameters.pop("watershed_compactness", 0.0) +hyperparameters["watershed_params"] = {"compactness": compactness} + sdata = sd.read_zarr(par["input"]) sd_output = sd.SpatialData() From e9505f13ac341e88af1b717c27f073e191a37985 Mon Sep 17 00:00:00 2001 From: dariarom94 Date: Wed, 29 Jul 2026 19:57:30 +0200 Subject: [PATCH 44/60] adjust segger mem --- .../param_sweep/run_test_binning_nebius.sh | 2 +- .../param_sweep/run_test_cellpose_nebius.sh | 4 +- .../param_sweep/run_test_stardist_nebius.sh | 2 +- .../param_sweep/run_test_watershed_nebius.sh | 2 +- src/datasets/loaders/bruker_cosmx/script.py | 253 ++++++++++++++++-- .../segger/config.vsh.yaml | 2 +- 6 files changed, 233 insertions(+), 32 deletions(-) diff --git a/scripts/run_benchmark/param_sweep/run_test_binning_nebius.sh b/scripts/run_benchmark/param_sweep/run_test_binning_nebius.sh index 647e73f33..527e0d3d6 100644 --- a/scripts/run_benchmark/param_sweep/run_test_binning_nebius.sh +++ b/scripts/run_benchmark/param_sweep/run_test_binning_nebius.sh @@ -41,7 +41,7 @@ publish_dir="/scratch/results/runs/$(date +%Y-%m-%d_%H-%M-%S)_binning" # is independent of --revision below, which selects the pipeline CODE to run.) params_repo="openproblems-bio/task_ist_preprocessing" params_branch="$(git rev-parse --abbrev-ref HEAD)" -params_url="https://raw.githubusercontent.com/${params_repo}/${params_branch}/scripts/run_benchmark/binning_params.yaml" +params_url="https://raw.githubusercontent.com/${params_repo}/${params_branch}/scripts/run_benchmark/param_sweep/binning_params.yaml" cat > /tmp/params_settings.yaml << HERE default_methods: diff --git a/scripts/run_benchmark/param_sweep/run_test_cellpose_nebius.sh b/scripts/run_benchmark/param_sweep/run_test_cellpose_nebius.sh index 781da777b..e3458e9a6 100644 --- a/scripts/run_benchmark/param_sweep/run_test_cellpose_nebius.sh +++ b/scripts/run_benchmark/param_sweep/run_test_cellpose_nebius.sh @@ -39,7 +39,7 @@ publish_dir="/scratch/results/runs/$(date +%Y-%m-%d_%H-%M-%S)_cellpose" # is independent of --revision below, which selects the pipeline CODE to run.) params_repo="openproblems-bio/task_ist_preprocessing" params_branch="$(git rev-parse --abbrev-ref HEAD)" -params_url="https://raw.githubusercontent.com/${params_repo}/${params_branch}/scripts/run_benchmark/cellpose_params.yaml" +params_url="https://raw.githubusercontent.com/${params_repo}/${params_branch}/scripts/run_benchmark/param_sweep/cellpose_params.yaml" cat > /tmp/params_settings.yaml << HERE default_methods: @@ -91,7 +91,7 @@ HERE if ! curl -fsSL -o /dev/null "$params_url"; then echo "ERROR: params file not reachable at:" >&2 echo " $params_url" >&2 - echo "Commit and push scripts/run_benchmark/cellpose_params.yaml to '$params_branch' first." >&2 + echo "Commit and push scripts/run_benchmark/param_sweep/cellpose_params.yaml to '$params_branch' first." >&2 exit 1 fi diff --git a/scripts/run_benchmark/param_sweep/run_test_stardist_nebius.sh b/scripts/run_benchmark/param_sweep/run_test_stardist_nebius.sh index fe58738e3..21d22c6bc 100644 --- a/scripts/run_benchmark/param_sweep/run_test_stardist_nebius.sh +++ b/scripts/run_benchmark/param_sweep/run_test_stardist_nebius.sh @@ -37,7 +37,7 @@ publish_dir="/scratch/results/runs/$(date +%Y-%m-%d_%H-%M-%S)_stardist" # is independent of --revision below, which selects the pipeline CODE to run.) params_repo="openproblems-bio/task_ist_preprocessing" params_branch="$(git rev-parse --abbrev-ref HEAD)" -params_url="https://raw.githubusercontent.com/${params_repo}/${params_branch}/scripts/run_benchmark/stardist_params.yaml" +params_url="https://raw.githubusercontent.com/${params_repo}/${params_branch}/scripts/run_benchmark/param_sweep/stardist_params.yaml" cat > /tmp/params_settings.yaml << HERE default_methods: diff --git a/scripts/run_benchmark/param_sweep/run_test_watershed_nebius.sh b/scripts/run_benchmark/param_sweep/run_test_watershed_nebius.sh index 697767dfb..657635223 100644 --- a/scripts/run_benchmark/param_sweep/run_test_watershed_nebius.sh +++ b/scripts/run_benchmark/param_sweep/run_test_watershed_nebius.sh @@ -37,7 +37,7 @@ publish_dir="/scratch/results/runs/$(date +%Y-%m-%d_%H-%M-%S)_watershed" # is independent of --revision below, which selects the pipeline CODE to run.) params_repo="openproblems-bio/task_ist_preprocessing" params_branch="$(git rev-parse --abbrev-ref HEAD)" -params_url="https://raw.githubusercontent.com/${params_repo}/${params_branch}/scripts/run_benchmark/watershed_params.yaml" +params_url="https://raw.githubusercontent.com/${params_repo}/${params_branch}/scripts/run_benchmark/param_sweep/watershed_params.yaml" cat > /tmp/params_settings.yaml << HERE default_methods: diff --git a/src/datasets/loaders/bruker_cosmx/script.py b/src/datasets/loaders/bruker_cosmx/script.py index 902ccdc4b..9317fbe8a 100644 --- a/src/datasets/loaders/bruker_cosmx/script.py +++ b/src/datasets/loaders/bruker_cosmx/script.py @@ -44,7 +44,7 @@ │ └── -polygons.csv ├── (AnalysisResults) └── (RunSummary) ---> the CellLabels folder is not present, but the CellLabels_FXXX.tif files are in the FOV folders. +--> the CellLabels folder is not present, but the CellLabels_FXXX.tif files are in the FOV folders. They are moved to a newly created CellLabels folder. ### Version 2 (Example: CosMx Mouse brain) ### @@ -65,21 +65,30 @@ ├── _tx_file.csv └── -polygons.csv --> the flat files are in a separate zip, they need to be moved to CellStatsDir ( = `DATA_DIR`) ---> as in version 1, the CellLabels folder is not present, but the CellLabels_FXXX.tif files are in the FOV folders. - +--> as in version 1, the CellLabels folder is not present, but the CellLabels_FXXX.tif files are in the FOV folders. ### Version 3 (Example: CosMx lung cancer) ### -this has subdirectories for each sample and an extra sub directories for morphology images. Also, +this has subdirectories for each sample and an extra sub directories for morphology images. Also, the images are given for each z plane. This dataset is covered with the bruker_cosmx_nsclc dataloader. - - - +### Version 4 (Example: CosMx *Normal Liver*, "Raw Data Files" release) ### +NanoString/Bruker never published the AtoMx "flat files" for the public liver FFPE dataset — only the raw +`NormalLiverFiles.zip` (images + segmentation) plus TileDB/Seurat exports. So the raw zip has NO +`_tx_file.csv` / `_fov_positions_file.csv` etc. — `sopa.io.cosmx` would fail at `_infer_dataset_id`. +The raw material to rebuild the flat files IS present, though: + - `AnalysisResults/**/FOV###__complete_code_cell_target_call_coord.csv` — per-transcript rows with local + pixel coords (`x`,`y`), `z`, `target` (gene), `CellId` (per-FOV cell, 0 = unassigned), `CellComp`. + - `RunSummary/latest.fovs.csv` — per-FOV stage position (columns: ..., x_mm, y_mm, ..., fov). + - `CellStatsDir/FOV###/*Cell_Stats_F###.csv` — per-cell `CellId, Area, CenterX, CenterY` (local px). + - `CellStatsDir/FOV###/CellLabels_F###.tif` + `CellStatsDir/Morphology2D/*_F###.TIF` — labels + image. +When the flat files are missing we RECONSTRUCT them (`reconstruct_flat_files`, see below) and then hand the +result to the *same* `sopa.io.cosmx` call, so all the stitching/coordinate math stays sopa's job. """ import os +import re import shutil import zipfile from pathlib import Path @@ -92,6 +101,9 @@ "input_raw": "temp/datasets/bruker_cosmx/HalfBrain.zip", # downloaded from https://smi-public.objects.liquidweb.services/Half%20%20Brain%20simple%20%20files%20.zip "input_flat_files": "temp/datasets/bruker_cosmx/Half Brain simple files .zip", + # For the liver "Raw Data Files" release (no flat files), point input_raw at + # https://smi-public.objects.liquidweb.services/NormalLiverFiles.zip and leave input_flat_files unset; + # the flat files are then reconstructed automatically (see Version 4 above). "segmentation_id": ["cell"], "output": "output.zarr", "dataset_id": "bruker_cosmx/bruker_mouse_brain_cosmx/rep1", @@ -126,12 +138,37 @@ def log(msg): # whole thing next to the staged zip overruns the node's ~220 GB scratch and OOMs (137). SKIP_DIRS = {"Morphology3D", "AnalysisResults", "RunSummary"} +# Prefix used for the flat files we reconstruct ourselves (Version 4). Passed to +# sopa as an explicit dataset_id so it doesn't try to infer one from a filename. +RECON_ID = "cosmx" + def _member_wanted(name: str) -> bool: parts = name.split("/") if name.startswith("__MACOSX/") or parts[-1] == ".DS_Store": return False return not any(part in SKIP_DIRS for part in parts) +# When reconstructing (Version 4), the AnalysisResults/RunSummary that the denylist above +# would drop are exactly what we need. Extract an *allowlist* instead: only the per-FOV +# transcript-decoding CSVs, the FOV positions, the cell stats, the cell labels and the 2D +# morphology — everything else (Morphology3D, Morphology2D_Normalized, RnD, the redundant +# target_call_coord/total_unambiguous CSVs, the imaging metrics) is dropped. Footprint for +# the liver hemisphere: ~61 GB vs ~440 GB of Morphology3D alone. +_RECON_KEEP = [ + re.compile(r"(^|/)CellLabels_F\d+\.tif$", re.I), # per-FOV cell labels + re.compile(r"(^|/)CellLabels/[^/]+\.tif$", re.I), # ... or an already-gathered CellLabels dir + re.compile(r"Cell_Stats_F\d+.*\.csv$", re.I), # per-cell centroids/area + re.compile(r"/Morphology2D/[^/]+\.[Tt][Ii][Ff]$"), # 2D morphology image (NOT Morphology2D_Normalized) + re.compile(r"complete_code_cell_target_call_coord\.csv$"), # per-transcript calls (+ cell assignment) + re.compile(r"\.fovs\.csv$"), # FOV stage positions +] + +def _member_wanted_reconstruct(name: str) -> bool: + parts = name.split("/") + if name.startswith("__MACOSX/") or parts[-1] == ".DS_Store": + return False + return any(pat.search(name) for pat in _RECON_KEEP) + def _open_zip_source(src): """Seekable binary handle for a local path or an s3:// URL (streamed, never fully downloaded).""" s = str(src) @@ -142,18 +179,27 @@ def _open_zip_source(src): return fs.open(s, "rb") return open(s, "rb") -def extract_zip(input_zip, output_dir: Path, strip_root: bool = False): +def _zip_has_flat_files(src) -> bool: + """Peek the zip's central directory (cheap, streamed) for a `*_tx_file.csv` flat file.""" + with _open_zip_source(src) as fh, zipfile.ZipFile(fh, "r") as zip_ref: + return any( + n.endswith("_tx_file.csv") or n.endswith("_tx_file.csv.gz") + for n in zip_ref.namelist() + ) + +def extract_zip(input_zip, output_dir: Path, strip_root: bool = False, wanted=_member_wanted): """ - Stream a zip (local path or s3:// URL) into output_dir, skipping the CosMx - directories sopa never reads (see SKIP_DIRS). Only the wanted members are - transferred, so a remote zip streams just the bytes we keep — it is never - downloaded in full first. + Stream a zip (local path or s3:// URL) into output_dir, keeping only the members for + which `wanted(name)` is True (see `_member_wanted` / `_member_wanted_reconstruct`). Only + the wanted members are transferred, so a remote zip streams just the bytes we keep — it is + never downloaded in full first. Arguments: - input_zip: local path or s3:// URL of the input zip. - output_dir: directory where the (filtered) contents are written. - strip_root: if True and all entries share exactly one top-level directory, strip it. + - wanted: predicate on the member name deciding whether to extract it. """ output_dir = Path(output_dir) kept = skipped = 0 @@ -166,7 +212,7 @@ def extract_zip(input_zip, output_dir: Path, strip_root: bool = False): do_strip = strip_root and len(roots) == 1 for member in members: - if not _member_wanted(member.filename): + if not wanted(member.filename): skipped += 1 continue parts = Path(member.filename).parts @@ -183,13 +229,30 @@ def extract_zip(input_zip, output_dir: Path, strip_root: bool = False): shutil.copyfileobj(src, dst) kept += 1 kept_bytes += member.file_size - log(f"Extracted {kept} files ({kept_bytes / 1e9:.1f} GB); skipped {skipped} members " - f"(dirs {sorted(SKIP_DIRS)} + macOS cruft)") + log(f"Extracted {kept} files ({kept_bytes / 1e9:.1f} GB); skipped {skipped} members") + +############################################################# +# Decide whether the flat files need to be reconstructed # +############################################################# +# Reconstruct only when no flat files are available anywhere: a separate `input_flat_files` +# zip always means they exist (mouse brain); otherwise peek inside the raw zip for a +# `*_tx_file.csv`. Absent everywhere => the liver "Raw Data Files" case (Version 4). +if par["input_flat_files"]: + RECONSTRUCT = False +else: + log("No flat files zip provided; checking whether the raw zip contains flat files") + RECONSTRUCT = not _zip_has_flat_files(par["input_raw"]) +log(f"Reconstruct flat files from raw AnalysisResults: {RECONSTRUCT}") # Extract zip files log("Extract zip of raw files") INPUT_RAW_EXTRACTED = TMP_DIR / "input_raw" -extract_zip(par["input_raw"], INPUT_RAW_EXTRACTED, strip_root=True) +extract_zip( + par["input_raw"], + INPUT_RAW_EXTRACTED, + strip_root=True, + wanted=_member_wanted_reconstruct if RECONSTRUCT else _member_wanted, +) log(f"Files and folders in input_raw_extracted ({INPUT_RAW_EXTRACTED})") print(os.listdir(INPUT_RAW_EXTRACTED)) @@ -233,6 +296,124 @@ def extract_zip(input_zip, output_dir: Path, strip_root: bool = False): else: raise AssertionError(f"No CellLabels folder and no per-FOV CellLabels tifs found in {DATA_DIR}") +################################################################### +# Reconstruct the flat files from the raw decoding CSVs (Version 4)# +################################################################### +# See the module docstring. We rebuild the four flat files sopa needs +# (`{id}_fov_positions_file.csv`, `{id}_tx_file.csv`, `{id}_metadata_file.csv`, +# `{id}_exprMat_file.csv`) from AnalysisResults + RunSummary + Cell_Stats and drop them into +# DATA_DIR so the unchanged `sopa.io.cosmx` call below picks them up. We do NOT reconstruct +# `-polygons.csv` (the raw export has no cell boundaries); `cell_boundaries` is optional in +# the API and downstream methods produce their own segmentation. +# +# Coordinate reconstruction (validated end-to-end: 0.999 of assigned transcripts land on +# their own cell in sopa's stitched label image): +# scale = 1e3 / COSMX_PIXEL_SIZE (mm -> px, matching sopa's own fov-positions conversion) +# x_global_px = x_mm * scale + x_local +# y_global_px = y_mm * scale + (H - 1 - y_local) # sopa flips each label tile along y +# The same transform is applied to the Cell_Stats centroids so the table's obsm['spatial'] +# stays consistent with the transcripts and labels. +def reconstruct_flat_files(extracted_root: Path, data_dir: Path, labels_dir: Path, dataset_id: str) -> int: + import numpy as np + import pandas as pd + import tifffile + from sopa.io.reader.cosmx import COSMX_PIXEL_SIZE + + scale = 1e3 / COSMX_PIXEL_SIZE + + # FOV image height (all FOVs share the same shape; sopa asserts this later). + a_label = next(iter(sorted(labels_dir.glob("*.[Tt][Ii][Ff]")))) + with tifffile.TiffFile(a_label) as tif: + H, W = tif.pages[0].shape + log(f"Reconstruct: FOV image shape {(H, W)} (from {a_label.name})") + + # FOV positions from RunSummary/latest.fovs.csv (headerless: ..., x_mm[1], y_mm[2], ..., fov[6]). + fovs_files = sorted(extracted_root.rglob("*.fovs.csv")) + fovs_file = next((f for f in fovs_files if f.name == "latest.fovs.csv"), None) or fovs_files[0] + raw_pos = pd.read_csv(fovs_file, header=None) + fp = raw_pos[[6, 1, 2]].copy() + fp.columns = ["fov", "x_mm", "y_mm"] + fp = fp.drop_duplicates("fov").sort_values("fov").reset_index(drop=True) + fp.to_csv(data_dir / f"{dataset_id}_fov_positions_file.csv", index=False) + pos = fp.set_index("fov") + log(f"Reconstruct: {len(fp)} FOV positions from {fovs_file.name}") + + # Map fov number -> Cell_Stats csv (under CellStatsDir/FOV###/). + cs_paths = {} + for p in data_dir.rglob("*Cell_Stats_F*.csv"): + m = re.search(r"Cell_Stats_F(\d+)", p.name) + if m: + cs_paths[int(m.group(1))] = p + + cc_paths = sorted(extracted_root.rglob("*complete_code_cell_target_call_coord.csv")) + assert cc_paths, f"No complete_code_cell_target_call_coord.csv found under {extracted_root}" + log(f"Reconstruct: {len(cc_paths)} per-FOV transcript files, {len(cs_paths)} cell-stats files") + + tx_path = data_dir / f"{dataset_id}_tx_file.csv" + meta_parts, expr_parts = [], [] + max_cell_id = 0 + n_tx = 0 + with open(tx_path, "w") as tx_out: + for i, cc_path in enumerate(cc_paths): + m = re.search(r"FOV0*(\d+)", cc_path.name) or re.search(r"_F0*(\d+)", cc_path.name) + fov = int(m.group(1)) + if fov not in pos.index: + log(f"Reconstruct: WARNING no FOV position for FOV {fov}, skipping") + continue + xmm, ymm = float(pos.at[fov, "x_mm"]), float(pos.at[fov, "y_mm"]) + + cc = pd.read_csv(cc_path, usecols=["x", "y", "z", "target", "CellId"]) + tx = pd.DataFrame({ + "fov": fov, + "cell_ID": cc["CellId"].astype(int).values, + "target": cc["target"].astype(str).values, + "x_global_px": xmm * scale + cc["x"].values, + "y_global_px": ymm * scale + (H - 1 - cc["y"].values), + "z": cc["z"].values, + }) + tx.to_csv(tx_out, header=(i == 0), index=False) + n_tx += len(tx) + + # Per-cell metadata + counts, keyed on the segmentation cell list (Cell_Stats), + # so metadata and exprMat share the exact same cells in the same order. + cs = pd.read_csv(cs_paths[fov]) + cell_ids = cs["CellId"].astype(int).values + max_cell_id = max(max_cell_id, int(cell_ids.max()), int(tx["cell_ID"].max())) + + assigned = tx[tx["cell_ID"] > 0] + ct = pd.crosstab(assigned["cell_ID"], assigned["target"]).reindex(cell_ids, fill_value=0) + ct.insert(0, "cell_ID", cell_ids) + ct.insert(0, "fov", fov) + expr_parts.append(ct.reset_index(drop=True)) + + meta_parts.append(pd.DataFrame({ + "fov": fov, + "cell_ID": cell_ids, + "CenterX_global_px": xmm * scale + cs["CenterX"].values, + "CenterY_global_px": ymm * scale + (H - 1 - cs["CenterY"].values), + "Area": cs["Area"].values, + })) + + if (i + 1) % 50 == 0: + log(f"Reconstruct: processed {i + 1}/{len(cc_paths)} FOVs ({n_tx:,} transcripts)") + + meta = pd.concat(meta_parts, ignore_index=True) + meta.to_csv(data_dir / f"{dataset_id}_metadata_file.csv", index=False) + + expr = pd.concat(expr_parts, ignore_index=True).fillna(0) + gene_cols = [c for c in expr.columns if c not in ("fov", "cell_ID")] + expr[gene_cols] = expr[gene_cols].astype("int32") + expr = expr[["fov", "cell_ID"] + gene_cols] + expr.to_csv(data_dir / f"{dataset_id}_exprMat_file.csv", index=False) + + log(f"Reconstruct: wrote flat files — {n_tx:,} transcripts, {len(meta):,} cells, {len(gene_cols)} genes") + return max_cell_id + +RECON_MAX_CELL_ID = None +if RECONSTRUCT: + log("Reconstruct flat files from raw decoding CSVs") + RECON_MAX_CELL_ID = reconstruct_flat_files(INPUT_RAW_EXTRACTED, DATA_DIR, labels_dir, RECON_ID) + ############################################# # Memory optimization: lean transcript read # ############################################# @@ -286,6 +467,17 @@ def __getattr__(self, item): _cosmx_mod.pd = _PandasReadCsvProxy(_real_pd) +# When we reconstruct the flat files ourselves, pin sopa's global-cell-id formula to the +# segmentation's true max cell id. sopa otherwise derives that max separately from each of +# the tx / metadata / exprMat frames and asserts they agree — but a cell present in the +# segmentation (Cell_Stats) yet carrying zero transcripts makes the tx-file max smaller, +# tripping the assert. Using one fixed max keeps the ids consistent across all three files. +if RECONSTRUCT: + _RECON_MAX = int(RECON_MAX_CELL_ID) + def _fixed_global_cell_id(self, df): + return df["fov"] * (_RECON_MAX + 1) * (df["cell_ID"] > 0) + df["cell_ID"] + _cosmx_mod._CosMXReader._get_global_cell_id = _fixed_global_cell_id + ######################################### # Convert raw files to spatialdata zarr # ######################################### @@ -293,14 +485,16 @@ def __getattr__(self, item): log("Convert raw files to spatialdata zarr") sdata = sopa.io.cosmx( - DATA_DIR, - dataset_id=None, - fov=None, - read_proteins=False, - cells_labels=True, - cells_table=True, - cells_polygons=True, - flip_image=False + DATA_DIR, + dataset_id=RECON_ID if RECONSTRUCT else None, + fov=None, + read_proteins=False, + cells_labels=True, + cells_table=True, + # The reconstructed (liver) export has no cell-boundary polygons; every other CosMx + # export we handle ships `-polygons.csv`. + cells_polygons=not RECONSTRUCT, + flip_image=False, ) @@ -309,7 +503,7 @@ def __getattr__(self, item): ############### log("Rename keys") elements_renaming_map = { - "stitched_image" : "morphology_mip", + "stitched_image" : "morphology_mip", "stitched_labels" : "cell_labels", "points" : "transcripts", "cells_polygons" : "cell_boundaries", @@ -317,6 +511,10 @@ def __getattr__(self, item): } for old_key, new_key in elements_renaming_map.items(): + if old_key not in sdata: + # e.g. cells_polygons is absent when reconstructing (no boundaries in the raw export) + log(f"Element '{old_key}' not present, skipping rename to '{new_key}'") + continue sdata[new_key] = sdata[old_key] del sdata[old_key] @@ -338,6 +536,9 @@ def __getattr__(self, item): # Add info to metadata table # ############################## log("Add metadata to table") +# The common iST API requires a per-cell `cell_id` in obs; sopa keys the table by the global +# cell id (as the index) but doesn't add it as a column. +sdata["table"].obs["cell_id"] = sdata["table"].obs.index.astype(str) for key in ["dataset_id", "dataset_name", "dataset_url", "dataset_reference", "dataset_summary", "dataset_description", "dataset_organism", "segmentation_id"]: sdata["table"].uns[key] = par[key] @@ -348,4 +549,4 @@ def __getattr__(self, item): sdata.write(par["output"]) -log(f"Done in {datetime.now() - t0}") \ No newline at end of file +log(f"Done in {datetime.now() - t0}") diff --git a/src/methods_transcript_assignment/segger/config.vsh.yaml b/src/methods_transcript_assignment/segger/config.vsh.yaml index f83d61b0e..80130ce0d 100644 --- a/src/methods_transcript_assignment/segger/config.vsh.yaml +++ b/src/methods_transcript_assignment/segger/config.vsh.yaml @@ -141,4 +141,4 @@ runners: - type: executable - type: nextflow directives: - label: [ hightime, midcpu, highmem, gpu ] + label: [ hightime, midcpu, highmem, gpuh100 ] From 3a155be11e3208ae1c21ba1921b5004259f8e777 Mon Sep 17 00:00:00 2001 From: dariarom94 Date: Thu, 30 Jul 2026 00:09:55 +0200 Subject: [PATCH 45/60] update fastreseg to tacco --- .../fastreseg/config.vsh.yaml | 10 ++-- .../fastreseg/input.py | 60 ++++++++++++++----- .../fastreseg/script.R | 4 +- 3 files changed, 53 insertions(+), 21 deletions(-) diff --git a/src/methods_transcript_assignment/fastreseg/config.vsh.yaml b/src/methods_transcript_assignment/fastreseg/config.vsh.yaml index d5c9985ec..33e175e51 100644 --- a/src/methods_transcript_assignment/fastreseg/config.vsh.yaml +++ b/src/methods_transcript_assignment/fastreseg/config.vsh.yaml @@ -78,12 +78,10 @@ engines: run: - Rscript -e 'options(warn = 2); remotes::install_github("Nanostring-Biostats/FastReseg", upgrade = "never", repos = "https://cran.rstudio.com")' - type: python - # `plankton` (pulled in by txsim.run_ssam for ssam cell annotation in - # input.py) still does `from matplotlib.cm import get_cmap`, removed in - # matplotlib 3.9. Pin < 3.9 so run_ssam imports. This is a local setup - # step, so it runs AFTER the __merge__'d spatialdata/txsim partials (which - # otherwise pull matplotlib >= 3.9) and wins. Mirrors methods_cell_type_annotation/ssam. - pypi: [planktonspace, "matplotlib<3.9"] + # tacco provides the cell-type annotation used in input.py (tacco.tl.annotate). + # Mirrors methods_cell_type_annotation/tacco. This replaced the previous ssam + # path, which needed planktonspace + a matplotlib<3.9 pin. + pypi: [anndata, numpy, tacco] __merge__: - /src/base/setup_txsim_partial.yaml - /src/base/setup_spatialdata_partial.yaml diff --git a/src/methods_transcript_assignment/fastreseg/input.py b/src/methods_transcript_assignment/fastreseg/input.py index 9eabfe0ab..207952ee2 100644 --- a/src/methods_transcript_assignment/fastreseg/input.py +++ b/src/methods_transcript_assignment/fastreseg/input.py @@ -4,7 +4,7 @@ import pandas as pd import xarray as xr import dask -import txsim as tx +import tacco import anndata as ad import argparse from scipy.sparse import csr_matrix @@ -90,12 +90,31 @@ def parse_arguments(): output_path_cell_type = args.output_path_cell_type ## potential other parameters (TODO - make configurable) -um_per_pixel = 0.5 sc_celltype_key = 'cell_type' ### reading the data in sdata = sd.read_zarr(input_path) +# The transcripts points can carry a non-unique index: multi-partition parquet where +# each partition's RangeIndex restarts at 0, or concatenated "combined" replicates that +# each kept their own row index. dask-expr aligns on the index when the cell_id column is +# assigned below (`sdata['transcripts']["cell_id"] = ...`), and pandas cannot reindex a +# duplicate-labelled axis -> the following sd.transform then fails with +# "cannot reindex on an axis with duplicate labels". Reset to a unique RangeIndex eagerly +# here: a lazy .reset_index() does not help (the duplicate index already poisons the dask +# graph), and re-parsing via PointsModel.parse breaks the from_dask_array(index=...) assign +# below with a "Missing dependency" graph error, so materialise to pandas and re-wrap as a +# single from_pandas partition, re-attaching the coordinate transformations. No-op when the +# index is already unique. +if not sdata['transcripts'].index.compute().is_unique: + print('Transcripts index has duplicate labels; resetting to a unique index', flush=True) + _transforms = sd.transformations.get_transformation(sdata['transcripts'], get_all=True) + _fixed = dask.dataframe.from_pandas( + sdata['transcripts'].compute().reset_index(drop=True), npartitions=1 + ) + sd.transformations.set_transformation(_fixed, _transforms, set_all=True) + sdata['transcripts'] = _fixed + ### reading in basic segmentation sdata_segm = sd.read_zarr(input_segmentation_path) segmentation_coord_systems = sd.transformations.get_transformation(sdata_segm["segmentation"], get_all=True).keys() @@ -125,10 +144,10 @@ def parse_arguments(): print('Transforming transcripts coordinates', flush=True) transcripts = sd.transform(sdata['transcripts'], to_coordinate_system='global') -# .copy() is required: for a single-partition points object, transcripts.compute() -# returns a reference to the dask graph's backing frame, so an in-place rename here -# would corrupt it and the later transcripts.compute() (passed to run_ssam) would -# yield renamed columns that no longer match the dask _meta -> "Metadata mismatch". +# .copy() defensively: for a single-partition points object, transcripts.compute() +# can return a reference to the dask graph's backing frame, so the in-place rename +# below would otherwise corrupt it and any later compute of `transcripts` would yield +# renamed columns that no longer match the dask _meta -> "Metadata mismatch". transcripts_df = transcripts.compute().copy() transcripts_df.rename(columns = {'feature_name': 'target', 'transcript_id': 'UMI_transID', 'cell_id': 'UMI_cellID'}, inplace = True) @@ -157,18 +176,31 @@ def parse_arguments(): columns=adata_sp.var_names) count_df.to_csv(output_path_counts) -#### run cell annotation with ssam +#### run cell annotation with tacco adata_sc = ad.read_h5ad(input_sc_reference_path) -adata_sc.X = adata_sc.layers["normalized"] print(adata_sc.var_names[1:10]) -shared_genes = [g for g in adata_sc.var_names if g in adata_sp.var_names] -adata_sp = adata_sp[:,shared_genes] +# tacco annotates the cell x gene count matrix directly against the reference, so +# (unlike the previous ssam path) it needs neither the transcript-level spots nor a +# gene subsetting step -- tacco intersects genes internally. It expects raw counts on +# .X for both spatial and reference, mirroring methods_cell_type_annotation/tacco. +adata_sp.X = adata_sp.layers['counts'] +adata_sc.X = adata_sc.layers['counts'] print('Annotating cell types', flush=True) -adata_sp = tx.preprocessing.run_ssam( - adata_sp, transcripts.compute(), adata_sc, um_p_px=um_per_pixel, - cell_id_col='cell_id', gene_col='feature_name', sc_ct_key=sc_celltype_key +cell_type_assignment = tacco.tl.annotate( + adata=adata_sp, + reference=adata_sc, + annotation_key=sc_celltype_key, ) -cell_type_df = adata_sp.obs["ct_ssam"].astype(str) + +# tacco returns an n_obs x n_celltypes proportion matrix (aligned to adata_sp.obs); +# take the argmax per cell for the hard label the FastReseg R step consumes as `clust`. +cell_types = cell_type_assignment.columns +highest_score_idx = np.argmax(cell_type_assignment.to_numpy(), axis=1) +adata_sp.obs[sc_celltype_key] = cell_types[highest_score_idx] + +# Preserve the exact CSV shape the R step reads (read.csv(row.names=1) -> a single +# column of per-cell labels indexed by cell_id). +cell_type_df = adata_sp.obs[sc_celltype_key].astype(str) cell_type_df.to_csv(output_path_cell_type, header=True) \ No newline at end of file diff --git a/src/methods_transcript_assignment/fastreseg/script.R b/src/methods_transcript_assignment/fastreseg/script.R index f1e67810e..bc864d170 100644 --- a/src/methods_transcript_assignment/fastreseg/script.R +++ b/src/methods_transcript_assignment/fastreseg/script.R @@ -81,7 +81,9 @@ if(is.null(refineAll_res_one_FOC$updated_transDF_list[[1]])){ cell_df$UMI_cellID <- as.integer(row.names(cell_df)) transcriptDF <- left_join(cell_df, transcriptDF) names(transcriptDF)[names(transcriptDF) == "UMI_cellID"] <- "updated_cellID" - names(transcriptDF)[names(transcriptDF) == "ct_ssam"] <- "updated_celltype" + # input.py writes the per-cell annotation column under sc_celltype_key ("cell_type"). + # (Was "ct_ssam" when the annotation step used ssam.) + names(transcriptDF)[names(transcriptDF) == "cell_type"] <- "updated_celltype" write.csv(transcriptDF, file = path_to_transcripts_out) } else { write.csv(refineAll_res_one_FOC$updated_transDF_list[[1]], file = path_to_transcripts_out) From acdd6c7216ae5c003f02bb97432d88edf65e32eb Mon Sep 17 00:00:00 2001 From: dariarom94 Date: Thu, 30 Jul 2026 08:42:42 +0200 Subject: [PATCH 46/60] Add annotation + expression-correction parameter sweeps (rctd, ssam, tangram, resolvi) Cloud-only tune-method sweeps under scripts/run_benchmark/param_sweep/. Each sweeps only already-exposed config args so it runs against the build/main container with no rebuild. Params read at launch from the committed yaml via raw-GitHub URL. Co-Authored-By: Claude Opus 4.8 --- .../param_sweep/rctd_params.yaml | 52 ++++++++ .../resolvi_correction_params.yaml | 45 +++++++ .../param_sweep/run_test_rctd_nebius.sh | 113 ++++++++++++++++++ .../run_test_resolvi_correction_nebius.sh | 102 ++++++++++++++++ .../param_sweep/run_test_ssam_nebius.sh | 104 ++++++++++++++++ .../param_sweep/run_test_tangram_nebius.sh | 104 ++++++++++++++++ .../param_sweep/ssam_params.yaml | 34 ++++++ .../param_sweep/tangram_params.yaml | 39 ++++++ 8 files changed, 593 insertions(+) create mode 100644 scripts/run_benchmark/param_sweep/rctd_params.yaml create mode 100644 scripts/run_benchmark/param_sweep/resolvi_correction_params.yaml create mode 100644 scripts/run_benchmark/param_sweep/run_test_rctd_nebius.sh create mode 100644 scripts/run_benchmark/param_sweep/run_test_resolvi_correction_nebius.sh create mode 100644 scripts/run_benchmark/param_sweep/run_test_ssam_nebius.sh create mode 100644 scripts/run_benchmark/param_sweep/run_test_tangram_nebius.sh create mode 100644 scripts/run_benchmark/param_sweep/ssam_params.yaml create mode 100644 scripts/run_benchmark/param_sweep/tangram_params.yaml diff --git a/scripts/run_benchmark/param_sweep/rctd_params.yaml b/scripts/run_benchmark/param_sweep/rctd_params.yaml new file mode 100644 index 000000000..c0d6404ce --- /dev/null +++ b/scripts/run_benchmark/param_sweep/rctd_params.yaml @@ -0,0 +1,52 @@ +# Parameter sweep for the rctd (spacexr RCTD, R) cell type annotation method. +# Committed source of truth, read at runtime from GitHub via a raw URL by +# run_test_rctd_nebius.sh (the Nebius compute env pulls the repo but cannot see the +# launch host's local files). +# +# Consumed by the run_benchmark workflow via the `method_parameters_yaml` setting +# (src/workflows/run_benchmark/main.nf). For every method the workflow builds: +# * one "default" variant using the `default:` args below, and +# * one extra variant per value in each `sweep:` list, with that ONE arg overridden. +# The benchmark varies a SINGLE parameter at a time (a "star" around the default, not +# a full grid), so total rctd variants = 1 default + sum(sweep list lengths) = 13. +# +# See src/methods_cell_type_annotation/rctd/NOTES.md ("Optimization / tuning") for the +# tiers and rationale. +# +# All six args are create.RCTD DE-gene / UMI thresholds. Every config default is a +# DELIBERATELY RELAXED value (walked away from the spacexr default) because iST panels +# are small (~100-500 genes) and low-count; the stock spacexr defaults strand too few DE +# genes and abort with "fewer than 10 regression differentially expressed genes". Each +# sweep list therefore walks that ONE arg back TOWARD its spacexr default (the meaningful +# direction, since the relaxed default already sits at the permissive extreme, and the +# default variant already covers the relaxed point). +# +# CAVEAT: on a very small panel, the high-threshold variants (values at/near the spacexr +# defaults) can drop back under the 10-DE-gene floor and legitimately fail — that failure +# boundary is part of what this sweep measures. +parameters: + rctd: + # Baseline == the component's shipped (relaxed-for-iST) defaults. + default: + gene_cutoff: 0.0 + fc_cutoff: 0.1 + gene_cutoff_reg: 0.0 + fc_cutoff_reg: 0.1 + umi_min: 20 + umi_min_sigma: 20 + sweep: + # ---- Tier 1: DE-gene fold-change thresholds (sharpest levers) ---- + # fc_cutoff: min log-FC for a PLATFORM-EFFECT DE gene. Relaxed 0.1 -> spacexr 0.5. + fc_cutoff: [0.25, 0.5] + # fc_cutoff_reg: min log-FC for a REGRESSION DE gene. Relaxed 0.1 -> spacexr 0.75. + fc_cutoff_reg: [0.4, 0.75] + # ---- Tier 1: DE-gene expression-mean thresholds (companion filters) ---- + # gene_cutoff: min normalized mean expr, platform-effect. 0.0 -> spacexr 0.000125. + gene_cutoff: [0.0000625, 0.000125] + # gene_cutoff_reg: min normalized mean expr, regression. 0.0 -> spacexr 0.0002. + gene_cutoff_reg: [0.0001, 0.0002] + # ---- Tier 1: UMI thresholds (which cells get annotated / fit variance) ---- + # umi_min: min total UMI per spatial cell to annotate it. 20 -> spacexr 100. + umi_min: [50, 100] + # umi_min_sigma: min UMI for cells fitting the platform-effect variance. 20 -> 300. + umi_min_sigma: [100, 300] diff --git a/scripts/run_benchmark/param_sweep/resolvi_correction_params.yaml b/scripts/run_benchmark/param_sweep/resolvi_correction_params.yaml new file mode 100644 index 000000000..303dcdefd --- /dev/null +++ b/scripts/run_benchmark/param_sweep/resolvi_correction_params.yaml @@ -0,0 +1,45 @@ +# Parameter sweep for the resolvi_correction (resolVI, scvi-tools) expression-correction method. +# Committed source of truth read by run_test_resolvi_correction_nebius.sh from GitHub via a +# raw URL (the Nebius compute env pulls the repo but cannot see the launch host's local files). +# +# Consumed by the run_benchmark workflow via the `method_parameters_yaml` setting +# (src/workflows/run_benchmark/main.nf). For every method the workflow builds: +# * one "default" variant using the `default:` args below, and +# * one extra variant per value in each `sweep:` list, with that ONE arg overridden. +# The benchmark varies a SINGLE parameter at a time (a "star" around the default, not a full +# grid), so total resolvi_correction variants = 1 default + sum(sweep list lengths) = 5. +# +# SUBMITTABLE-NOW CONSTRAINT: the Nebius launch runs `--revision build/main`, i.e. the +# build/main container as it exists today. It only honours arguments ALREADY EXPOSED in +# src/methods_expression_correction/resolvi_correction/config.vsh.yaml. That container exposes +# exactly four args -- celltype_key, n_hidden, encode_covariates, downsample_counts -- so the +# sweep touches ONLY those (celltype_key is an input mapping, not a tuning knob). The high-value +# resolVI levers num_samples (posterior samples), max_epochs, and mixture_k are HARD-CODED in +# script.py, NOT exposed; sweeping them would need a config change + container rebuild, so they +# are documented in NOTES.md ("Optimization / tuning") as future work and are NOT swept here. +# +# See src/methods_expression_correction/resolvi_correction/NOTES.md for tiers, rationale, and +# the verified upstream defaults. Trim or widen the lists to taste. +parameters: + resolvi_correction: + # Baseline == the component's shipped defaults, which (verified) all match the scvi-tools + # RESOLVI upstream defaults (n_hidden=32, encode_covariates=False, downsample_counts=True). + default: + celltype_key: cell_type + n_hidden: 32 + encode_covariates: false + downsample_counts: true + sweep: + # ---- Tier 2: quality / capacity trade-offs (the only exposed knobs) ---- + # n_hidden: width of the VAE hidden layers. 32 is the scvi RESOLVI default (small, fast); + # raising it gives the encoder/decoder more capacity to model diffusion + background at a + # time cost. script.py's own grid-search note lists 64 and 128 (default 32 omitted). + n_hidden: [64, 128] + # encode_covariates: BOOLEAN, default false. Flip to true to also feed batch/covariates + # through the encoder (helps when integrating across batches). Forwarded to the RESOLVI + # module via **model_kwargs; only one meaningful non-default value exists. + encode_covariates: [true] + # downsample_counts: BOOLEAN, default true (subsamples per-cell counts to equalise depth + # during training). Flip to false to train on full observed counts; only one meaningful + # non-default value exists. + downsample_counts: [false] diff --git a/scripts/run_benchmark/param_sweep/run_test_rctd_nebius.sh b/scripts/run_benchmark/param_sweep/run_test_rctd_nebius.sh new file mode 100644 index 000000000..438c3a14e --- /dev/null +++ b/scripts/run_benchmark/param_sweep/run_test_rctd_nebius.sh @@ -0,0 +1,113 @@ +#!/bin/bash + +# Nebius test run: all default methods + RCTD (rctd) cell type annotation, with a +# parameter sweep over RCTD's create.RCTD DE-gene / UMI thresholds. +# See src/methods_cell_type_annotation/rctd/NOTES.md ("Optimization / tuning"). +# +# rctd is an R (spacexr) CPU-only method, so it runs on the standard (non-GPU) +# compute env with NO `gpu` label. +# +# The stage default (tacco) is kept enabled alongside rctd so the run also produces +# the baseline annotation — the workflow allows at most ONE non-default variant at a +# time, and rctd is that one non-default method here. +# +# SUBMITTABLE AS-IS: every swept arg (the six create.RCTD thresholds) is ALREADY +# exposed in src/methods_cell_type_annotation/rctd/config.vsh.yaml, so the build/main +# container run by `--revision build/main` accepts them with NO rebuild. +# +# PARAMS-FILE CAVEAT (why this differs from a local script): +# `tw launch --params-file` is read client-side, but `method_parameters_yaml` +# is a path the WORKFLOW opens at runtime on the cloud (readYaml -> Nextflow +# file()). A local /tmp path does not exist there, and /scratch (where results +# publish) is READ-ONLY from the launch host — which is why the binning +# method_params block is commented out in run_test_nebius.sh. file() does stage +# http(s):// though, and this repo is public, so we keep the sweep in a COMMITTED +# file (scripts/run_benchmark/param_sweep/rctd_params.yaml) and read it from GitHub +# via its raw URL. => the params file must be committed AND PUSHED to $params_branch +# before launching (edit the file there, not here, to change the sweep). + +# get the root of the directory +REPO_ROOT=$(git rev-parse --show-toplevel) + +# ensure that the command below is run from the root of the repository +cd "$REPO_ROOT" + +set -e + +resources_test_s3=s3://openproblems-data/resources_test/task_ist_preprocessing +# Results publish to /scratch — created and written by the cloud compute env, so +# the launcher does NOT create it here (it is read-only from the launch host). +publish_dir="/scratch/results/runs/$(date +%Y-%m-%d_%H-%M-%S)_rctd" + +# The sweep lives in a committed file, read from GitHub at runtime. $params_branch +# defaults to the branch you are on; the file must be pushed there on GitHub. (This +# is independent of --revision below, which selects the pipeline CODE to run.) +params_repo="openproblems-bio/task_ist_preprocessing" +params_branch="$(git rev-parse --abbrev-ref HEAD)" +params_url="https://raw.githubusercontent.com/${params_repo}/${params_branch}/scripts/run_benchmark/param_sweep/rctd_params.yaml" + +cat > /tmp/params_settings.yaml << HERE +default_methods: + - custom_segmentation + - basic_transcript_assignment + - basic_count_aggregation + - basic_qc_filter + - alpha_shapes + - normalize_by_volume + - tacco + - no_correction +segmentation_methods: + - custom_segmentation +transcript_assignment_methods: + - basic_transcript_assignment +count_aggregation_methods: + - basic_count_aggregation +qc_filtering_methods: + - basic_qc_filter +volume_calculation_methods: + - alpha_shapes +normalization_methods: + - normalize_by_volume +celltype_annotation_methods: + - tacco + - rctd +# - ssam +# - moscot +# - mapmycells +# - tangram +# - singler +expression_correction_methods: + - no_correction +gene_efficiency_correction_methods: + - no_correction +method_parameters_yaml: $params_url +HERE + +# Write the parameters to file (input_states version, NOTE: enable `-entry_name auto` for this) +cat > /tmp/params.yaml << HERE +input_states: $resources_test_s3/**/state.yaml +rename_keys: 'input_sc:output_sc;input_sp:output_sp' +save_spatial_data: false +settings: '$(yq -o json /tmp/params_settings.yaml | jq -c .)' +output_state: "state.yaml" +publish_dir: "$publish_dir" +HERE + +# Fail early with a clear message if the params file isn't reachable on GitHub yet. +if ! curl -fsSL -o /dev/null "$params_url"; then + echo "ERROR: params file not reachable at:" >&2 + echo " $params_url" >&2 + echo "Commit and push scripts/run_benchmark/param_sweep/rctd_params.yaml to '$params_branch' first." >&2 + exit 1 +fi + +tw launch https://github.com/openproblems-bio/task_ist_preprocessing.git \ + --revision build/main \ + --pull-latest \ + --main-script target/nextflow/workflows/run_benchmark/main.nf \ + --workspace 167877437119966 \ + --compute-env 5hfmdCBxMRd4nHZaJKYEQZ \ + --params-file /tmp/params.yaml \ + --entry-name auto \ + --config src/base/labels_nebius.config \ + --labels task_ist_preprocessing,test,rctd diff --git a/scripts/run_benchmark/param_sweep/run_test_resolvi_correction_nebius.sh b/scripts/run_benchmark/param_sweep/run_test_resolvi_correction_nebius.sh new file mode 100644 index 000000000..a96ab96f0 --- /dev/null +++ b/scripts/run_benchmark/param_sweep/run_test_resolvi_correction_nebius.sh @@ -0,0 +1,102 @@ +#!/bin/bash + +# Nebius test run: all default methods + resolVI (resolvi_correction) expression correction, +# with a parameter sweep over the exposed optimization knobs identified for resolVI. +# See src/methods_expression_correction/resolvi_correction/NOTES.md ("Optimization / tuning"). +# +# resolVI is a scvi-tools VAE (deep model). It carries the `gpu` label (mirroring +# run_gpu_nebius.sh) and runs on the Nebius GPU compute env. The expression-correction stage +# lists BOTH the stage default (no_correction) AND resolvi_correction so the sweep is scored +# against the uncorrected baseline; every OTHER stage stays on its single default (the one +# non-default-variant-at-a-time constraint means resolvi_correction must be the only non-default +# thing in the pipeline). +# +# PARAMS-FILE CAVEAT (why this differs from a local script): +# `tw launch --params-file` is read client-side, but `method_parameters_yaml` is a path the +# WORKFLOW opens at runtime on the cloud (readYaml -> Nextflow file()). A local /tmp path does +# not exist there, and /scratch (where results publish) is READ-ONLY from the launch host. +# file() DOES stage http(s):// though, and this repo is public, so we keep the sweep in a +# COMMITTED file (scripts/run_benchmark/param_sweep/resolvi_correction_params.yaml) and read it +# from GitHub via its raw URL. => the params file must be committed AND PUSHED to +# $params_branch before launching (edit the file there, not here, to change the sweep). This is +# independent of --revision below, which selects the pipeline CODE to run. + +# get the root of the directory +REPO_ROOT=$(git rev-parse --show-toplevel) + +# ensure that the command below is run from the root of the repository +cd "$REPO_ROOT" + +set -e + +resources_test_s3=s3://openproblems-data/resources_test/task_ist_preprocessing +# Results publish to /scratch — created and written by the cloud compute env, so the launcher +# does NOT create it here (it is read-only from the launch host). +publish_dir="/scratch/results/runs/$(date +%Y-%m-%d_%H-%M-%S)_resolvi_correction" + +# The sweep lives in a committed file, read from GitHub at runtime. $params_branch defaults to +# the branch you are on; the file must be pushed there on GitHub. +params_repo="openproblems-bio/task_ist_preprocessing" +params_branch="$(git rev-parse --abbrev-ref HEAD)" +params_url="https://raw.githubusercontent.com/${params_repo}/${params_branch}/scripts/run_benchmark/param_sweep/resolvi_correction_params.yaml" + +cat > /tmp/params_settings.yaml << HERE +default_methods: + - custom_segmentation + - basic_transcript_assignment + - basic_count_aggregation + - basic_qc_filter + - alpha_shapes + - normalize_by_volume + - tacco + - no_correction +segmentation_methods: + - custom_segmentation +transcript_assignment_methods: + - basic_transcript_assignment +count_aggregation_methods: + - basic_count_aggregation +qc_filtering_methods: + - basic_qc_filter +volume_calculation_methods: + - alpha_shapes +normalization_methods: + - normalize_by_volume +celltype_annotation_methods: + - tacco +expression_correction_methods: + - no_correction + - resolvi_correction +gene_efficiency_correction_methods: + - no_correction +method_parameters_yaml: $params_url +HERE + +# Write the parameters to file (input_states version, NOTE: enable `-entry_name auto` for this) +cat > /tmp/params.yaml << HERE +input_states: $resources_test_s3/**/state.yaml +rename_keys: 'input_sc:output_sc;input_sp:output_sp' +save_spatial_data: false +settings: '$(yq -o json /tmp/params_settings.yaml | jq -c .)' +output_state: "state.yaml" +publish_dir: "$publish_dir" +HERE + +# Fail early with a clear message if the params file isn't reachable on GitHub yet. +if ! curl -fsSL -o /dev/null "$params_url"; then + echo "ERROR: params file not reachable at:" >&2 + echo " $params_url" >&2 + echo "Commit and push scripts/run_benchmark/param_sweep/resolvi_correction_params.yaml to '$params_branch' first." >&2 + exit 1 +fi + +tw launch https://github.com/openproblems-bio/task_ist_preprocessing.git \ + --revision build/main \ + --pull-latest \ + --main-script target/nextflow/workflows/run_benchmark/main.nf \ + --workspace 167877437119966 \ + --compute-env 5hfmdCBxMRd4nHZaJKYEQZ \ + --params-file /tmp/params.yaml \ + --entry-name auto \ + --config src/base/labels_nebius.config \ + --labels task_ist_preprocessing,test,resolvi_correction,gpu diff --git a/scripts/run_benchmark/param_sweep/run_test_ssam_nebius.sh b/scripts/run_benchmark/param_sweep/run_test_ssam_nebius.sh new file mode 100644 index 000000000..7ac988081 --- /dev/null +++ b/scripts/run_benchmark/param_sweep/run_test_ssam_nebius.sh @@ -0,0 +1,104 @@ +#!/bin/bash + +# Nebius test run: all default methods + SSAM (ssam) cell-type annotation, with a +# parameter sweep over the one exposed ssam knob (um_per_pixel). +# See src/methods_cell_type_annotation/ssam/NOTES.md ("Optimization / tuning"). +# +# ssam is a CPU-only method (txsim -> plankton density estimation, no GPU), so it +# runs on the standard (non-GPU) compute env with no `gpu` label. +# +# SUBMITTABLE AS-IS: the swept arg (um_per_pixel) is ALREADY EXPOSED in the current +# config.vsh.yaml, so the build/main container run by --revision below already +# understands it -- no viash ns build / container rebuild is required. +# +# PARAMS-FILE CAVEAT (why the sweep is not an inline heredoc): +# `tw launch --params-file` is read client-side, but `method_parameters_yaml` +# is a path the WORKFLOW opens at runtime on the cloud (readYaml -> Nextflow +# file()). A local /tmp path does not exist there, and /scratch (where results +# publish) is READ-ONLY from the launch host -- which is why the binning +# method_params block is commented out in run_test_nebius.sh. file() does stage +# http(s):// though, and this repo is public, so we keep the sweep in a COMMITTED +# file (scripts/run_benchmark/param_sweep/ssam_params.yaml) and read it from GitHub +# via its raw URL. => the params file must be committed AND PUSHED to $params_branch +# before launching (edit the file there, not here, to change the sweep). + +# get the root of the directory +REPO_ROOT=$(git rev-parse --show-toplevel) + +# ensure that the command below is run from the root of the repository +cd "$REPO_ROOT" + +set -e + +resources_test_s3=s3://openproblems-data/resources_test/task_ist_preprocessing +# Results publish to /scratch -- created and written by the cloud compute env, so +# the launcher does NOT create it here (it is read-only from the launch host). +publish_dir="/scratch/results/runs/$(date +%Y-%m-%d_%H-%M-%S)_ssam" + +# The sweep lives in a committed file, read from GitHub at runtime. $params_branch +# defaults to the branch you are on; the file must be pushed there on GitHub. (This +# is independent of --revision below, which selects the pipeline CODE to run.) +params_repo="openproblems-bio/task_ist_preprocessing" +params_branch="$(git rev-parse --abbrev-ref HEAD)" +params_url="https://raw.githubusercontent.com/${params_repo}/${params_branch}/scripts/run_benchmark/param_sweep/ssam_params.yaml" + +cat > /tmp/params_settings.yaml << HERE +default_methods: + - custom_segmentation + - basic_transcript_assignment + - basic_count_aggregation + - basic_qc_filter + - alpha_shapes + - normalize_by_volume + - tacco + - no_correction +segmentation_methods: + - custom_segmentation +transcript_assignment_methods: + - basic_transcript_assignment +count_aggregation_methods: + - basic_count_aggregation +qc_filtering_methods: + - basic_qc_filter +volume_calculation_methods: + - alpha_shapes +normalization_methods: + - normalize_by_volume +celltype_annotation_methods: + - tacco + - ssam +expression_correction_methods: + - no_correction +gene_efficiency_correction_methods: + - no_correction +method_parameters_yaml: $params_url +HERE + +# Write the parameters to file (input_states version, NOTE: enable `-entry_name auto` for this) +cat > /tmp/params.yaml << HERE +input_states: $resources_test_s3/**/state.yaml +rename_keys: 'input_sc:output_sc;input_sp:output_sp' +save_spatial_data: false +settings: '$(yq -o json /tmp/params_settings.yaml | jq -c .)' +output_state: "state.yaml" +publish_dir: "$publish_dir" +HERE + +# Fail early with a clear message if the params file isn't reachable on GitHub yet. +if ! curl -fsSL -o /dev/null "$params_url"; then + echo "ERROR: params file not reachable at:" >&2 + echo " $params_url" >&2 + echo "Commit and push scripts/run_benchmark/param_sweep/ssam_params.yaml to '$params_branch' first." >&2 + exit 1 +fi + +tw launch https://github.com/openproblems-bio/task_ist_preprocessing.git \ + --revision build/main \ + --pull-latest \ + --main-script target/nextflow/workflows/run_benchmark/main.nf \ + --workspace 167877437119966 \ + --compute-env 5hfmdCBxMRd4nHZaJKYEQZ \ + --params-file /tmp/params.yaml \ + --entry-name auto \ + --config src/base/labels_nebius.config \ + --labels task_ist_preprocessing,test,ssam diff --git a/scripts/run_benchmark/param_sweep/run_test_tangram_nebius.sh b/scripts/run_benchmark/param_sweep/run_test_tangram_nebius.sh new file mode 100644 index 000000000..be7c40618 --- /dev/null +++ b/scripts/run_benchmark/param_sweep/run_test_tangram_nebius.sh @@ -0,0 +1,104 @@ +#!/bin/bash + +# Nebius test run: all default methods + Tangram (tangram) cell-type annotation, +# with a parameter sweep over the optimization levers identified for Tangram. +# See src/methods_cell_type_annotation/tangram/NOTES.md ("Optimization / tuning"). +# +# Tangram is a torch deep-learning mapping method (GPU), hence the `gpu` label and +# the Nebius GPU compute env (mirrors run_gpu_nebius.sh). +# +# The annotation stage keeps its default method `tacco` alongside `tangram` so the +# run has a baseline to compare the tangram variants against. Every other stage is +# left on its single default (the swept method must be the only non-default thing). +# +# PARAMS-FILE CAVEAT (why this reads from GitHub): +# `tw launch --params-file` is read client-side, but `method_parameters_yaml` +# is a path the WORKFLOW opens at runtime on the cloud (readYaml -> Nextflow +# file()). A local /tmp path does not exist there, and /scratch (where results +# publish) is READ-ONLY from the launch host. file() does stage http(s):// though, +# and this repo is public, so we keep the sweep in a COMMITTED file +# (scripts/run_benchmark/param_sweep/tangram_params.yaml) and read it from GitHub +# via its raw URL. => the params file must be committed AND PUSHED to +# $params_branch before launching (edit the file there, not here, to change the +# sweep). This is independent of --revision below, which selects the pipeline CODE. + +# get the root of the directory +REPO_ROOT=$(git rev-parse --show-toplevel) + +# ensure that the command below is run from the root of the repository +cd "$REPO_ROOT" + +set -e + +resources_test_s3=s3://openproblems-data/resources_test/task_ist_preprocessing +# Results publish to /scratch — created and written by the cloud compute env, so +# the launcher does NOT create it here (it is read-only from the launch host). +publish_dir="/scratch/results/runs/$(date +%Y-%m-%d_%H-%M-%S)_tangram" + +# The sweep lives in a committed file, read from GitHub at runtime. $params_branch +# defaults to the branch you are on; the file must be pushed there on GitHub. (This +# is independent of --revision below, which selects the pipeline CODE to run.) +params_repo="openproblems-bio/task_ist_preprocessing" +params_branch="$(git rev-parse --abbrev-ref HEAD)" +params_url="https://raw.githubusercontent.com/${params_repo}/${params_branch}/scripts/run_benchmark/param_sweep/tangram_params.yaml" + +cat > /tmp/params_settings.yaml << HERE +default_methods: + - custom_segmentation + - basic_transcript_assignment + - basic_count_aggregation + - basic_qc_filter + - alpha_shapes + - normalize_by_volume + - tacco + - no_correction +segmentation_methods: + - custom_segmentation +transcript_assignment_methods: + - basic_transcript_assignment +count_aggregation_methods: + - basic_count_aggregation +qc_filtering_methods: + - basic_qc_filter +volume_calculation_methods: + - alpha_shapes +normalization_methods: + - normalize_by_volume +celltype_annotation_methods: + - tacco + - tangram +expression_correction_methods: + - no_correction +gene_efficiency_correction_methods: + - no_correction +method_parameters_yaml: $params_url +HERE + +# Write the parameters to file (input_states version, NOTE: enable `-entry_name auto` for this) +cat > /tmp/params.yaml << HERE +input_states: $resources_test_s3/**/state.yaml +rename_keys: 'input_sc:output_sc;input_sp:output_sp' +save_spatial_data: false +settings: '$(yq -o json /tmp/params_settings.yaml | jq -c .)' +output_state: "state.yaml" +publish_dir: "$publish_dir" +HERE + +# Fail early with a clear message if the params file isn't reachable on GitHub yet. +if ! curl -fsSL -o /dev/null "$params_url"; then + echo "ERROR: params file not reachable at:" >&2 + echo " $params_url" >&2 + echo "Commit and push scripts/run_benchmark/param_sweep/tangram_params.yaml to '$params_branch' first." >&2 + exit 1 +fi + +tw launch https://github.com/openproblems-bio/task_ist_preprocessing.git \ + --revision build/main \ + --pull-latest \ + --main-script target/nextflow/workflows/run_benchmark/main.nf \ + --workspace 167877437119966 \ + --compute-env 5hfmdCBxMRd4nHZaJKYEQZ \ + --params-file /tmp/params.yaml \ + --entry-name auto \ + --config src/base/labels_nebius.config \ + --labels task_ist_preprocessing,test,tangram,gpu diff --git a/scripts/run_benchmark/param_sweep/ssam_params.yaml b/scripts/run_benchmark/param_sweep/ssam_params.yaml new file mode 100644 index 000000000..acb08d5ad --- /dev/null +++ b/scripts/run_benchmark/param_sweep/ssam_params.yaml @@ -0,0 +1,34 @@ +# Parameter sweep for the ssam (SSAM signature-based) cell-type-annotation method. +# Committed source of truth for run_test_ssam_nebius.sh (read from GitHub via a raw +# URL, since the Nebius compute env pulls the repo but cannot see the launch host's +# local files). +# +# Consumed by the run_benchmark workflow via the `method_parameters_yaml` setting +# (src/workflows/run_benchmark/main.nf). For every method the workflow builds: +# * one "default" variant using the `default:` args below, and +# * one extra variant per value in each `sweep:` list, with that ONE arg overridden. +# The benchmark varies a SINGLE parameter at a time (a "star" around the default, not +# a full grid), so total ssam variants = 1 default + sum(sweep list lengths) = 5. +# +# See src/methods_cell_type_annotation/ssam/NOTES.md ("Optimization / tuning") for the +# rationale. NOTE: the component exposes exactly ONE tunable knob (`um_per_pixel`); the +# real SSAM levers (kernel_bandwidth, threshold_cor, threshold_exp, patch_length) are +# HARDCODED inside txsim's run_ssam wrapper and cannot be reached without patching +# txsim + rebuilding the image — see NOTES.md "Optimization / tuning" (future work). +parameters: + ssam: + # Baseline == the component's shipped default. NB the txsim tool default for + # um_p_px is 0.325; the component ships 0.5 (a non-default deviation), so 0.325 + # is included on the sweep axis below (non-default audit). + default: + um_per_pixel: 0.5 + sweep: + # um_per_pixel: micrometre-per-pixel factor. txsim's run_ssam multiplies the + # transcript x/y coordinates by this before building the mRNA-density map with + # a FIXED Gaussian kernel_bandwidth of 4 (hardcoded in txsim). It is therefore + # the only reachable dial on the effective KDE smoothing scale: smaller values + # compress coordinates -> coarser smoothing relative to the kernel; larger + # values spread them -> finer effective resolution. Range straddles the txsim + # tool default (0.325) and the shipped default (0.5), which is omitted here + # because the "default" variant above already covers it. + um_per_pixel: [0.1, 0.2, 0.325, 1.0] diff --git a/scripts/run_benchmark/param_sweep/tangram_params.yaml b/scripts/run_benchmark/param_sweep/tangram_params.yaml new file mode 100644 index 000000000..3638b69b1 --- /dev/null +++ b/scripts/run_benchmark/param_sweep/tangram_params.yaml @@ -0,0 +1,39 @@ +# Parameter sweep for the tangram (Tangram, torch deep-learning mapping) +# cell-type-annotation method. +# +# Read from GitHub via a raw URL by run_test_tangram_nebius.sh, since the Nebius +# compute env pulls the repo but cannot see the launch host's local files. +# +# Consumed by the run_benchmark workflow via the `method_parameters_yaml` setting +# (src/workflows/run_benchmark/main.nf). For every method the workflow builds: +# * one "default" variant using the `default:` args below, and +# * one extra variant per value in each `sweep:` list, with that ONE arg overridden. +# The benchmark varies a SINGLE parameter at a time (a "star" around the default, not +# a full grid), so total tangram variants = 1 default + sum(sweep list lengths) = 6. +# +# See src/methods_cell_type_annotation/tangram/NOTES.md ("Optimization / tuning") for +# the tiers, defaults, and rationale. Only ALREADY-EXPOSED args are swept here +# (mode, num_epochs) so this is submittable against the build/main container as-is; +# density_prior / learning_rate / loss lambdas are documented in NOTES as future work +# (they would need config expose + a container rebuild first). +parameters: + tangram: + # Baseline == the component's shipped defaults. + default: + mode: "clusters" + num_epochs: 1000 + sweep: + # ---- Tier 1: highest impact on quality (accuracy vs. runtime) ---- + # num_epochs: gradient-descent steps for the mapping. 1000 == Tangram's own + # default (our baseline). Fewer -> faster but under-converged/noisier labels; + # more -> better-converged mapping at a time cost. Sweep below and above the + # default to find the knee. Default 1000 is omitted (the default variant covers it). + num_epochs: [100, 500, 2000, 3000] + # ---- Tier 2: mapping granularity ---- + # mode: our default 'clusters' maps per-cell-type clusters (fits GPU VRAM on + # large references); 'cells' maps individual reference cells (finer, can be + # more accurate on SMALL references, but its matrix scales with n_sc_cells and + # OOMs on large ones). 'cells' is the one meaningful non-default value; feasible + # on the small test reference. ('constrained' needs target_count -> not a + # drop-in axis for label projection, left out.) + mode: ["cells"] From fa462b89445b600e0d02fef971cd814929e9ffa4 Mon Sep 17 00:00:00 2001 From: dariarom94 Date: Thu, 30 Jul 2026 08:52:55 +0200 Subject: [PATCH 47/60] Add moscot + split parameter sweeps (annotation, expression correction) Same cloud-only param_sweep/ convention; sweeps only already-exposed args so they run against build/main with no rebuild. (singler intentionally omitted: its only exposed knob celltype_key is dead code in script.py, so a sweep would be a no-op until a wiring fix + rebuild.) Co-Authored-By: Claude Opus 4.8 --- .../param_sweep/moscot_params.yaml | 61 ++++++++++ .../param_sweep/run_test_moscot_nebius.sh | 110 +++++++++++++++++ .../param_sweep/run_test_split_nebius.sh | 111 ++++++++++++++++++ .../param_sweep/split_params.yaml | 63 ++++++++++ 4 files changed, 345 insertions(+) create mode 100644 scripts/run_benchmark/param_sweep/moscot_params.yaml create mode 100644 scripts/run_benchmark/param_sweep/run_test_moscot_nebius.sh create mode 100644 scripts/run_benchmark/param_sweep/run_test_split_nebius.sh create mode 100644 scripts/run_benchmark/param_sweep/split_params.yaml diff --git a/scripts/run_benchmark/param_sweep/moscot_params.yaml b/scripts/run_benchmark/param_sweep/moscot_params.yaml new file mode 100644 index 000000000..e615b0f9e --- /dev/null +++ b/scripts/run_benchmark/param_sweep/moscot_params.yaml @@ -0,0 +1,61 @@ +# Parameter sweep for the moscot (MOSCOT, OTT-jax fused Gromov-Wasserstein optimal- +# transport reference mapping) cell-type-annotation method. GPU method. +# +# Committed source of truth for run_test_moscot_nebius.sh, read from GitHub via a raw +# URL (the Nebius compute env pulls the repo but cannot see the launch host's files). +# +# Consumed by the run_benchmark workflow via the `method_parameters_yaml` setting +# (src/workflows/run_benchmark/main.nf). For every method the workflow builds: +# * one "default" variant using the `default:` args below, and +# * one extra variant per value in each `sweep:` list, with that ONE arg overridden. +# The benchmark varies a SINGLE parameter at a time (a "star" around the default, not +# a full grid), so total moscot variants = 1 default + sum(sweep list lengths) = 9. +# +# See src/methods_cell_type_annotation/moscot/NOTES.md ("Optimization / tuning") for +# the tiers, tool defaults, and rationale. +# +# Only ALREADY-EXPOSED args are swept (alpha, epsilon, mapping_mode), so this is +# SUBMITTABLE against the build/main container as-is (no rebuild). Un-exposed knobs +# (n_comps, scale_cost, initializer, convergence thresholds) are documented in NOTES +# as future work. +# +# !!! SMALL-DATA CLAMP — why tau and rank are NOT swept here !!! +# script.py:62-66 forces rank=-1 and tau=1.0 whenever the spatial AnnData has +# < 10000 cells. The test resource (mouse_brain_combined) has n_obs=306, so on this +# run rank and tau are OVERRIDDEN regardless of any value passed -- sweeping them +# would only add identical no-op variants. They deviate from moscot's tool defaults +# (tau 0.3 vs 1.0, rank 500 vs -1) ONLY for >10k-cell datasets, and are documented in +# NOTES.md as the Tier-1 knobs for a real (>10k) dataset sweep. +parameters: + moscot: + # Baseline == the component's shipped defaults (config.vsh.yaml). + default: + alpha: 0.8 + epsilon: 0.01 + tau: 0.3 # clamped to 1.0 on this <10k test data (script.py:65-66) + rank: 500 # clamped to -1 on this <10k test data (script.py:63-64) + batch_size: 1024 # pure memory knob (online GW-cost batching); held fixed + mapping_mode: max + sweep: + # ---- Tier 1: FGW interpolation weight (quality) ---- + # alpha in (0,1] balances the quadratic Gromov-Wasserstein term (intra-domain + # structure: SC 30-dim PCA vs spatial local-pca coords) against the linear term + # (shared-gene expression). alpha->1 = pure GW/structure; alpha->0 = pure + # expression matching. Config default 0.8 DEVIATES from moscot's tool default 0.5 + # (structure-heavy) -> promoted to a sweep axis. 0.8 omitted (the default variant + # covers it); range straddles the tool default (0.5) up through 0.9. + alpha: [0.5, 0.7, 0.9] + # ---- Tier 1: entropic regularization (quality vs. convergence) ---- + # epsilon sets the entropy/stochasticity of the transport plan: higher = blurrier + # map, faster & more stable convergence; lower = sharper, more confident mapping + # but harder to converge. Config default 0.01 == moscot's tool default (not a + # deviation), swept as a legitimate Tier-1 quality axis. 0.01 omitted; two lower + # (sharper) + two higher (diffuse) probe an order of magnitude each way. + epsilon: [0.001, 0.005, 0.05, 0.1] + # ---- Tier 2: label read-out from the transport plan ---- + # mapping_mode: how a per-cell label is pulled from the plan. 'max' (default) takes + # the label of the single highest-mass source cell; 'sum' aggregates the plan mass + # per cell type first, then takes the argmax (smoother, uses the full coupling). + # moscot has NO tool default here (required arg); 'sum' is the one meaningful + # non-default value. + mapping_mode: [sum] diff --git a/scripts/run_benchmark/param_sweep/run_test_moscot_nebius.sh b/scripts/run_benchmark/param_sweep/run_test_moscot_nebius.sh new file mode 100644 index 000000000..7319cfb74 --- /dev/null +++ b/scripts/run_benchmark/param_sweep/run_test_moscot_nebius.sh @@ -0,0 +1,110 @@ +#!/bin/bash + +# Nebius test run: all default methods + MOSCOT (moscot) cell-type annotation, with a +# parameter sweep over the optimization levers identified for moscot. +# See src/methods_cell_type_annotation/moscot/NOTES.md ("Optimization / tuning"). +# +# moscot is an OTT-jax fused Gromov-Wasserstein optimal-transport mapping method that +# runs on GPU (jax[cuda12], gpuh100), hence the `gpu` label and the Nebius GPU compute +# env (mirrors run_gpu_nebius.sh). +# +# The annotation stage keeps its default method `tacco` alongside `moscot` so the run +# has a baseline to compare the moscot variants against. Every other stage is left on +# its single default (the swept method must be the only non-default thing). +# +# SUBMITTABLE-NOW: only already-exposed args are swept (alpha, epsilon, mapping_mode), +# so this launches against the build/main container with no rebuild. tau and rank are +# NOT swept because script.py clamps them to tool defaults on the <10k-cell test data +# (see moscot_params.yaml + NOTES.md). +# +# PARAMS-FILE CAVEAT (why this reads from GitHub): +# `tw launch --params-file` is read client-side, but `method_parameters_yaml` +# is a path the WORKFLOW opens at runtime on the cloud (readYaml -> Nextflow +# file()). A local /tmp path does not exist there, and /scratch (where results +# publish) is READ-ONLY from the launch host. file() does stage http(s):// though, +# and this repo is public, so we keep the sweep in a COMMITTED file +# (scripts/run_benchmark/param_sweep/moscot_params.yaml) and read it from GitHub +# via its raw URL. => the params file must be committed AND PUSHED to +# $params_branch before launching (edit the file there, not here, to change the +# sweep). This is independent of --revision below, which selects the pipeline CODE. + +# get the root of the directory +REPO_ROOT=$(git rev-parse --show-toplevel) + +# ensure that the command below is run from the root of the repository +cd "$REPO_ROOT" + +set -e + +resources_test_s3=s3://openproblems-data/resources_test/task_ist_preprocessing +# Results publish to /scratch — created and written by the cloud compute env, so +# the launcher does NOT create it here (it is read-only from the launch host). +publish_dir="/scratch/results/runs/$(date +%Y-%m-%d_%H-%M-%S)_moscot" + +# The sweep lives in a committed file, read from GitHub at runtime. $params_branch +# defaults to the branch you are on; the file must be pushed there on GitHub. (This +# is independent of --revision below, which selects the pipeline CODE to run.) +params_repo="openproblems-bio/task_ist_preprocessing" +params_branch="$(git rev-parse --abbrev-ref HEAD)" +params_url="https://raw.githubusercontent.com/${params_repo}/${params_branch}/scripts/run_benchmark/param_sweep/moscot_params.yaml" + +cat > /tmp/params_settings.yaml << HERE +default_methods: + - custom_segmentation + - basic_transcript_assignment + - basic_count_aggregation + - basic_qc_filter + - alpha_shapes + - normalize_by_volume + - tacco + - no_correction +segmentation_methods: + - custom_segmentation +transcript_assignment_methods: + - basic_transcript_assignment +count_aggregation_methods: + - basic_count_aggregation +qc_filtering_methods: + - basic_qc_filter +volume_calculation_methods: + - alpha_shapes +normalization_methods: + - normalize_by_volume +celltype_annotation_methods: + - tacco + - moscot +expression_correction_methods: + - no_correction +gene_efficiency_correction_methods: + - no_correction +method_parameters_yaml: $params_url +HERE + +# Write the parameters to file (input_states version, NOTE: enable `-entry_name auto` for this) +cat > /tmp/params.yaml << HERE +input_states: $resources_test_s3/**/state.yaml +rename_keys: 'input_sc:output_sc;input_sp:output_sp' +save_spatial_data: false +settings: '$(yq -o json /tmp/params_settings.yaml | jq -c .)' +output_state: "state.yaml" +publish_dir: "$publish_dir" +HERE + +# Fail early with a clear message if the params file isn't reachable on GitHub yet. +if ! curl -fsSL -o /dev/null "$params_url"; then + echo "ERROR: params file not reachable at:" >&2 + echo " $params_url" >&2 + echo "Commit and push scripts/run_benchmark/param_sweep/moscot_params.yaml to '$params_branch' first." >&2 + exit 1 +fi + +tw launch https://github.com/openproblems-bio/task_ist_preprocessing.git \ + --revision build/main \ + --pull-latest \ + --main-script target/nextflow/workflows/run_benchmark/main.nf \ + --workspace 167877437119966 \ + --compute-env 5hfmdCBxMRd4nHZaJKYEQZ \ + --params-file /tmp/params.yaml \ + --entry-name auto \ + --config src/base/labels_nebius.config \ + --labels task_ist_preprocessing,test,moscot,gpu diff --git a/scripts/run_benchmark/param_sweep/run_test_split_nebius.sh b/scripts/run_benchmark/param_sweep/run_test_split_nebius.sh new file mode 100644 index 000000000..0a20da769 --- /dev/null +++ b/scripts/run_benchmark/param_sweep/run_test_split_nebius.sh @@ -0,0 +1,111 @@ +#!/bin/bash + +# Nebius test run: all default methods + SPLIT (split) expression correction, with a +# parameter sweep over the six create.RCTD DE-gene / UMI thresholds SPLIT exposes. +# See src/methods_expression_correction/split/NOTES.md ("Optimization / tuning"). +# +# split is an R method wrapping spacexr (RCTD) + the SPLIT R package. It is CPU-only, so it +# runs on the standard (non-GPU) compute env with NO `gpu` label (modelled on run_test_nebius.sh). +# +# The stage default (no_correction) is kept enabled alongside split so the run also produces +# the uncorrected baseline the sweep is scored against — the workflow allows at most ONE +# non-default variant at a time, and split is that one non-default method here; every OTHER +# stage stays on its single default. +# +# SUBMITTABLE AS-IS: every swept arg (the six create.RCTD thresholds) is ALREADY exposed in +# src/methods_expression_correction/split/config.vsh.yaml, so the build/main container run by +# `--revision build/main` accepts them with NO rebuild. SPLIT's own core levers +# (DO_purify_singlets, DO_remove_residual_contamination, RCTD doublet_mode) are hard-coded in +# script.R and are documented in NOTES.md as future work, NOT swept here. +# +# PARAMS-FILE CAVEAT (why this differs from a local script): +# `tw launch --params-file` is read client-side, but `method_parameters_yaml` is a path the +# WORKFLOW opens at runtime on the cloud (readYaml -> Nextflow file()). A local /tmp path does +# not exist there, and /scratch (where results publish) is READ-ONLY from the launch host — +# which is why the binning method_params block is commented out in run_test_nebius.sh. file() +# DOES stage http(s):// though, and this repo is public, so we keep the sweep in a COMMITTED +# file (scripts/run_benchmark/param_sweep/split_params.yaml) and read it from GitHub via its +# raw URL. => the params file must be committed AND PUSHED to $params_branch before launching +# (edit the file there, not here, to change the sweep). This is independent of --revision +# below, which selects the pipeline CODE to run. + +# get the root of the directory +REPO_ROOT=$(git rev-parse --show-toplevel) + +# ensure that the command below is run from the root of the repository +cd "$REPO_ROOT" + +set -e + +resources_test_s3=s3://openproblems-data/resources_test/task_ist_preprocessing +# Results publish to /scratch — created and written by the cloud compute env, so +# the launcher does NOT create it here (it is read-only from the launch host). +publish_dir="/scratch/results/runs/$(date +%Y-%m-%d_%H-%M-%S)_split" + +# The sweep lives in a committed file, read from GitHub at runtime. $params_branch defaults to +# the branch you are on; the file must be pushed there on GitHub. (This is independent of +# --revision below, which selects the pipeline CODE to run.) +params_repo="openproblems-bio/task_ist_preprocessing" +params_branch="$(git rev-parse --abbrev-ref HEAD)" +params_url="https://raw.githubusercontent.com/${params_repo}/${params_branch}/scripts/run_benchmark/param_sweep/split_params.yaml" + +cat > /tmp/params_settings.yaml << HERE +default_methods: + - custom_segmentation + - basic_transcript_assignment + - basic_count_aggregation + - basic_qc_filter + - alpha_shapes + - normalize_by_volume + - tacco + - no_correction +segmentation_methods: + - custom_segmentation +transcript_assignment_methods: + - basic_transcript_assignment +count_aggregation_methods: + - basic_count_aggregation +qc_filtering_methods: + - basic_qc_filter +volume_calculation_methods: + - alpha_shapes +normalization_methods: + - normalize_by_volume +celltype_annotation_methods: + - tacco +expression_correction_methods: + - no_correction + - split +gene_efficiency_correction_methods: + - no_correction +method_parameters_yaml: $params_url +HERE + +# Write the parameters to file (input_states version, NOTE: enable `-entry_name auto` for this) +cat > /tmp/params.yaml << HERE +input_states: $resources_test_s3/**/state.yaml +rename_keys: 'input_sc:output_sc;input_sp:output_sp' +save_spatial_data: false +settings: '$(yq -o json /tmp/params_settings.yaml | jq -c .)' +output_state: "state.yaml" +publish_dir: "$publish_dir" +HERE + +# Fail early with a clear message if the params file isn't reachable on GitHub yet. +if ! curl -fsSL -o /dev/null "$params_url"; then + echo "ERROR: params file not reachable at:" >&2 + echo " $params_url" >&2 + echo "Commit and push scripts/run_benchmark/param_sweep/split_params.yaml to '$params_branch' first." >&2 + exit 1 +fi + +tw launch https://github.com/openproblems-bio/task_ist_preprocessing.git \ + --revision build/main \ + --pull-latest \ + --main-script target/nextflow/workflows/run_benchmark/main.nf \ + --workspace 167877437119966 \ + --compute-env 5hfmdCBxMRd4nHZaJKYEQZ \ + --params-file /tmp/params.yaml \ + --entry-name auto \ + --config src/base/labels_nebius.config \ + --labels task_ist_preprocessing,test,split diff --git a/scripts/run_benchmark/param_sweep/split_params.yaml b/scripts/run_benchmark/param_sweep/split_params.yaml new file mode 100644 index 000000000..228a0447d --- /dev/null +++ b/scripts/run_benchmark/param_sweep/split_params.yaml @@ -0,0 +1,63 @@ +# Parameter sweep for the split (SPLIT, R) expression-correction method. +# Committed source of truth, read at runtime from GitHub via a raw URL by +# run_test_split_nebius.sh (the Nebius compute env pulls the repo but cannot see the +# launch host's local files). +# +# Consumed by the run_benchmark workflow via the `method_parameters_yaml` setting +# (src/workflows/run_benchmark/main.nf). For every method the workflow builds: +# * one "default" variant using the `default:` args below, and +# * one extra variant per value in each `sweep:` list, with that ONE arg overridden. +# The benchmark varies a SINGLE parameter at a time (a "star" around the default, not +# a full grid), so total split variants = 1 default + sum(sweep list lengths) = 13. +# +# See src/methods_expression_correction/split/NOTES.md ("Optimization / tuning") for the +# tiers and rationale. +# +# WHAT SPLIT ACTUALLY VARIES HERE: SPLIT runs RCTD (spacexr) INTERNALLY to deconvolve each +# cell, then purifies counts from that deconvolution. The only knobs the component exposes +# are the six create.RCTD DE-gene / UMI thresholds (plus keep_all_cells, a zero-count-cell +# guard that is NOT a tuning lever and can crash if flipped -> left fixed). Every one of the +# six config defaults is a DELIBERATELY RELAXED value walked away from the spacexr default, +# because iST panels are small (~100-500 genes) and low-count; the stock spacexr defaults +# strand too few DE genes and abort with "fewer than 10 regression differentially expressed +# genes". NON-DEFAULT AUDIT: because all six deviate from the spacexr defaults and none is a +# performance/resource knob, each is forced onto a sweep axis. Each list therefore walks that +# ONE arg back TOWARD its spacexr default (the meaningful direction, since the relaxed default +# already sits at the permissive extreme, and the default variant already covers it). +# +# SUBMITTABLE-NOW: the Nebius launch runs `--revision build/main`, so the sweep touches ONLY +# args already exposed in the build/main container -- the six thresholds below all exist in +# src/methods_expression_correction/split/config.vsh.yaml today, so no rebuild is needed. +# SPLIT's OWN core levers (DO_purify_singlets, DO_remove_residual_contamination, RCTD +# doublet_mode) are HARD-CODED in script.R, NOT exposed; sweeping them would need a config +# arg + container rebuild, so they are documented in NOTES.md as future work and NOT here. +# +# CAVEAT: on a very small panel, the high-threshold variants (values at/near the spacexr +# defaults) can drop back under the 10-DE-gene floor and legitimately fail inside create.RCTD +# -- that failure boundary is part of what this sweep measures. +parameters: + split: + # Baseline == the component's shipped (relaxed-for-iST) defaults. + default: + gene_cutoff: 0.0 + fc_cutoff: 0.1 + gene_cutoff_reg: 0.0 + fc_cutoff_reg: 0.1 + umi_min: 20 + umi_min_sigma: 20 + sweep: + # ---- Tier 1: DE-gene fold-change thresholds (sharpest levers) ---- + # fc_cutoff: min log-FC for a PLATFORM-EFFECT DE gene. Relaxed 0.1 -> spacexr 0.5. + fc_cutoff: [0.25, 0.5] + # fc_cutoff_reg: min log-FC for a REGRESSION DE gene. Relaxed 0.1 -> spacexr 0.75. + fc_cutoff_reg: [0.4, 0.75] + # ---- Tier 1: DE-gene expression-mean thresholds (companion filters) ---- + # gene_cutoff: min normalized mean expr, platform-effect. 0.0 -> spacexr 0.000125. + gene_cutoff: [0.0000625, 0.000125] + # gene_cutoff_reg: min normalized mean expr, regression. 0.0 -> spacexr 0.0002. + gene_cutoff_reg: [0.0001, 0.0002] + # ---- Tier 1: UMI thresholds (which cells get deconvolved / fit variance) ---- + # umi_min: min total UMI per spatial cell to include it. 20 -> spacexr 100. + umi_min: [50, 100] + # umi_min_sigma: min UMI for cells fitting the platform-effect variance. 20 -> 300. + umi_min_sigma: [100, 300] From fd37aee81a85d7561c5cce7d83ddf6a6df568556 Mon Sep 17 00:00:00 2001 From: dariarom94 Date: Thu, 30 Jul 2026 09:38:49 +0200 Subject: [PATCH 48/60] singler: read par['celltype_key'] instead of hardcoding "cell_type" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The celltype_key arg (from comp_method_cell_type_annotation.yaml, default cell_type) was declared but never used — script.py hardcoded the reference label column, so any celltype_key override was a no-op. Now it selects the reference obs column, enabling a granularity sweep (cell_type -> cell_type_level2/3/4). Backward-compatible: default is still cell_type. Verified with `viash test` (2/2 scripts pass). Co-Authored-By: Claude Opus 4.8 --- src/methods_cell_type_annotation/singler/script.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/methods_cell_type_annotation/singler/script.py b/src/methods_cell_type_annotation/singler/script.py index c6d539283..921238451 100644 --- a/src/methods_cell_type_annotation/singler/script.py +++ b/src/methods_cell_type_annotation/singler/script.py @@ -38,8 +38,8 @@ mat_ref = mat_ref.sorted_indices() ## magic line to make sure the matrix is in the right format for SingleR ## create the reference from our sc data -built = singler.train_single(ref_data = mat_ref, - ref_labels = sce_ref.get_column_data().column("cell_type"), +built = singler.train_single(ref_data = mat_ref, + ref_labels = sce_ref.get_column_data().column(par['celltype_key']), ref_features = sce_ref.get_row_names(), test_features = features,) From 3449bd0e18691e96f3e0ecf479fdd09e80508d43 Mon Sep 17 00:00:00 2001 From: dariarom94 Date: Thu, 30 Jul 2026 15:06:12 +0200 Subject: [PATCH 49/60] fastreseg --- src/datasets/loaders/bruker_cosmx/script.py | 9 +++++++-- src/methods_transcript_assignment/fastreseg/input.py | 10 ++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/src/datasets/loaders/bruker_cosmx/script.py b/src/datasets/loaders/bruker_cosmx/script.py index 9317fbe8a..8cb45a846 100644 --- a/src/datasets/loaders/bruker_cosmx/script.py +++ b/src/datasets/loaders/bruker_cosmx/script.py @@ -536,8 +536,13 @@ def _fixed_global_cell_id(self, df): # Add info to metadata table # ############################## log("Add metadata to table") -# The common iST API requires a per-cell `cell_id` in obs; sopa keys the table by the global -# cell id (as the index) but doesn't add it as a column. +# The common iST API requires a per-cell unique `cell_id` in obs. sopa keys the table by the +# global cell id (the index), so expose that as `cell_id`. But the CosMx metadata already +# carries a per-FOV-*local* `cell_ID` column, and obs cannot hold two keys that differ only in +# case ('cell_ID' vs 'cell_id') — spatialdata's zarr writer rejects that as an invalid name. +# Rename the vendor local id out of the way first. +if "cell_ID" in sdata["table"].obs.columns: + sdata["table"].obs.rename(columns={"cell_ID": "cell_ID_local"}, inplace=True) sdata["table"].obs["cell_id"] = sdata["table"].obs.index.astype(str) for key in ["dataset_id", "dataset_name", "dataset_url", "dataset_reference", "dataset_summary", "dataset_description", "dataset_organism", "segmentation_id"]: sdata["table"].uns[key] = par[key] diff --git a/src/methods_transcript_assignment/fastreseg/input.py b/src/methods_transcript_assignment/fastreseg/input.py index 207952ee2..09e59ab37 100644 --- a/src/methods_transcript_assignment/fastreseg/input.py +++ b/src/methods_transcript_assignment/fastreseg/input.py @@ -149,6 +149,16 @@ def parse_arguments(): # below would otherwise corrupt it and any later compute of `transcripts` would yield # renamed columns that no longer match the dask _meta -> "Metadata mismatch". transcripts_df = transcripts.compute().copy() + +# Some datasets carry no per-molecule transcript id: e.g. the Vizgen MERSCOPE loader +# renames the vendor's `transcript_id` (a gene/ensembl id) to `ensembl_id`, leaving the +# transcripts without a `transcript_id` column. FastReseg itself does not use one +# (transID_coln = NULL in script.R), but output.py needs a `transcript_id` in the final +# output, so synthesise a unique one from the row position when it is absent. +if 'transcript_id' not in transcripts_df.columns: + print('No transcript_id column found; synthesising a unique transcript_id', flush=True) + transcripts_df['transcript_id'] = np.arange(len(transcripts_df), dtype=np.int64) + transcripts_df.rename(columns = {'feature_name': 'target', 'transcript_id': 'UMI_transID', 'cell_id': 'UMI_cellID'}, inplace = True) From 5961d0a2aa3a72d393f11b5ab52bc730dccf074b Mon Sep 17 00:00:00 2001 From: dariarom94 Date: Thu, 30 Jul 2026 16:13:53 +0200 Subject: [PATCH 50/60] adjust labels --- src/base/labels_nebius.config | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/base/labels_nebius.config b/src/base/labels_nebius.config index 7dd1ffcbe..67ab80153 100644 --- a/src/base/labels_nebius.config +++ b/src/base/labels_nebius.config @@ -153,7 +153,11 @@ process { accelerator = 1 memory = { [ 100.GB * task.attempt, 150.GB ].min() } disk = 200.GB - pod = [[nodeSelector: 'nebius.com/node-group-id=mk8snodegroup-e00dhcgx1xqjskycvc'], [imagePullPolicy: 'Always']] + // runAsUser: 0 — same reason as the gpu label: any non-root GPU image (e.g. + // segger, based on rapidsai/base = uid 1001) otherwise can't write Nextflow's + // root-owned task scratch dir (.command.log / .command.begin / .exitcode -> + // "Permission denied"). Harmless for already-root GPU images. + pod = [[nodeSelector: 'nebius.com/node-group-id=mk8snodegroup-e00dhcgx1xqjskycvc'], [imagePullPolicy: 'Always'], [runAsUser: 0]] containerOptions = { workflow.containerEngine == "singularity" ? '--nv': ( workflow.containerEngine == "docker" ? '--gpus all': null ) } } @@ -164,7 +168,12 @@ process { accelerator = 1 memory = { [ 120.GB * task.attempt, 180.GB ].min() } disk = 200.GB - pod = [[nodeSelector: 'nebius.com/node-group-id=mk8snodegroup-e00jp7hyqr094tmy35'], [imagePullPolicy: 'Always']] + // runAsUser: 0 — segger runs here (label gpuh100) and is based on rapidsai/base, + // which runs as the non-root 'rapids' user (uid 1001); without this the container + // cannot write Nextflow's root-owned task scratch dir (.command.log / + // .command.begin / .exitcode -> "Permission denied"). Harmless for already-root + // GPU images. If segger moves to another gpu label, carry this with it. + pod = [[nodeSelector: 'nebius.com/node-group-id=mk8snodegroup-e00jp7hyqr094tmy35'], [imagePullPolicy: 'Always'], [runAsUser: 0]] containerOptions = { workflow.containerEngine == "singularity" ? '--nv': ( workflow.containerEngine == "docker" ? '--gpus all': null ) } } From 857e16af012bf2152a334dca14b5e135d0fb67e3 Mon Sep 17 00:00:00 2001 From: dariarom94 Date: Fri, 31 Jul 2026 10:09:35 +0200 Subject: [PATCH 51/60] bruker nsclc --- .../bruker_cosmx_nsclc/config.vsh.yaml | 19 +- .../loaders/bruker_cosmx_nsclc/script.py | 356 ++++++++++++------ .../fastreseg/input.py | 10 +- 3 files changed, 261 insertions(+), 124 deletions(-) diff --git a/src/datasets/loaders/bruker_cosmx_nsclc/config.vsh.yaml b/src/datasets/loaders/bruker_cosmx_nsclc/config.vsh.yaml index 71c52fced..9b822cbd2 100644 --- a/src/datasets/loaders/bruker_cosmx_nsclc/config.vsh.yaml +++ b/src/datasets/loaders/bruker_cosmx_nsclc/config.vsh.yaml @@ -5,9 +5,19 @@ argument_groups: - name: Inputs arguments: - type: string - name: --sample - example: "Lung9_Rep1" - description: "Sample id" + name: --input_raw + example: "s3://openproblems-data/resources/raw_data/bruker_cosmx/Lung9_Rep1+SMI+Flat+data.tar.gz" + description: | + URL/path to the flat-files + cell-labels archive (`+SMI+Flat+data.tar.gz`). + Passed as a string (not a staged file) on purpose: the script streams it and extracts + only the members sopa needs. Supports s3:// (via s3fs) or a local path. + - type: string + name: --input_morphology + example: "s3://openproblems-data/resources/raw_data/bruker_cosmx/Lung9_Rep1+RawMorphologyImages.tar.gz" + description: | + URL/path to the raw morphology-images archive (`+RawMorphologyImages.tar.gz`). + Streamed in place; only the 4th z-plane (Z004) of each FOV is extracted. Supports + s3:// (via s3fs) or a local path. - type: string name: --segmentation_id default: ["cell"] @@ -60,9 +70,10 @@ engines: __merge__: - /src/base/setup_spatialdata_partial.yaml setup: - - type: python + - type: python pypi: - sopa + - s3fs # stream the raw tar.gz archives from s3:// and extract only what sopa needs (see --input_raw) - type: native runners: diff --git a/src/datasets/loaders/bruker_cosmx_nsclc/script.py b/src/datasets/loaders/bruker_cosmx_nsclc/script.py index 5085a5b2b..124203b12 100644 --- a/src/datasets/loaders/bruker_cosmx_nsclc/script.py +++ b/src/datasets/loaders/bruker_cosmx_nsclc/script.py @@ -1,57 +1,51 @@ """ -Notes about data structure: - -The directory structure that we need, looks as follows (expected by the sopa.io.cosmx function): -├── CellStatsDir ( = `DATA_DIR`) -│ ├── CellLabels -│ │ ├── CellLabels_F001.tif -│ │ ├── ... -│ │ └── CellLabels_F.tif -│ ├── Morphology2D -│ │ ├── _F001.tif -│ │ ├── ... -│ │ └── _F.tif -│ ├── _exprMat_file.csv -│ ├── _fov_positions_file.csv -│ ├── _metadata_file.csv -│ ├── _tx_file.csv -│ └── -polygons.csv -├── (AnalysisResults) -└── (RunSummary) - -The NSCLC samples download two files: -1. +SMI+Flat+data.tar.gz --> decompressed: - -└── /-Flat_files_and_images ( = `DATA_DIR`) - ├── CellLabels - │ ├── CellLabels_F001.tif - │ ├── ... - │ └── CellLabels_F.tif - ├── _exprMat_file.csv - ├── _fov_positions_file.csv - ├── _metadata_file.csv - ├── _tx_file.csv - └── NOTE: missing: -polygons.csv - -2. +RawMorphologyImages.tar.gz --> decompressed: - -└── 2/-RawMorphologyImages ( = `MORPHOLOGY_DIR`) - ├── _F001_Z001.TIF - ├── _F001_Z002.TIF - ├── ... - ├── _F001_Z008.TIF - ├── ... - └── _F_Z008.TIF - -The morphology images need to be moved to DATA_DIR / "Morphology2D". Also, we only take the 4th plane (Z004) of each image -and rename e.g. _F001_Z004.TIF to _F001.tif. - - +Notes about supported data structure (CosMx NSCLC / lung cancer, "Version 3"): + +The general `bruker_cosmx` loader handles the mouse-brain / liver CosMx releases (one big zip, +optionally a separate flat-files zip). The NSCLC lung-cancer release is shaped differently and +is handled here, but this loader now follows the SAME pattern as the general one: the raw +archives are mirrored to S3 once (see scripts/create_resources/spatial/mirror_bruker_to_s3.sh) +and streamed in place at load time — only the members sopa actually needs are extracted, so the +enormous per-z-plane morphology stacks never touch scratch. The archives are passed as strings +(not staged Viash files) so Nextflow does not copy them first. + +Each NSCLC sample ships TWO gzipped tarballs on the NanoString share: + +1. +SMI+Flat+data.tar.gz (par["input_raw"]) --> decompressed: + + └── /-Flat_files_and_images ( = `DATA_DIR`) + ├── CellLabels + │ ├── CellLabels_F001.tif + │ ├── ... + │ └── CellLabels_F.tif + ├── _exprMat_file.csv + ├── _fov_positions_file.csv + ├── _metadata_file.csv + ├── _tx_file.csv + └── NOTE: missing: -polygons.csv + +2. +RawMorphologyImages.tar.gz (par["input_morphology"]) --> decompressed: + + └── 2/-RawMorphologyImages ( = morphology dir) + ├── _F001_Z001.TIF + ├── ... + ├── _F001_Z008.TIF + ├── ... + └── _F_Z008.TIF + +The morphology images have one TIF per z-plane; sopa expects a single 2D image per FOV under +`DATA_DIR / "Morphology2D"`. We keep only the 4th plane (Z004) — this is the important filter, +it drops ~7/8 of the (very large) morphology archive — and rename e.g. +`_F001_Z004.TIF` to `_F001.tif` under `Morphology2D`. + +The NSCLC flat files carry no `-polygons.csv`, so `cell_boundaries` is derived from the stitched +label image (`sd.to_polygons`), matching the original NSCLC loader behaviour. """ import os +import re import shutil import tarfile from pathlib import Path @@ -61,10 +55,14 @@ ## VIASH START par = { - "sample": "Lung9_Rep1", + # Mirrored from + # https://nanostring-public-share.s3.us-west-2.amazonaws.com/SMI-Compressed/Lung9_Rep1/Lung9_Rep1+SMI+Flat+data.tar.gz + "input_raw": "s3://openproblems-data/resources/raw_data/bruker_cosmx/Lung9_Rep1+SMI+Flat+data.tar.gz", + # https://nanostring-public-share.s3.us-west-2.amazonaws.com/SMI-Compressed/Lung9_Rep1/Lung9_Rep1+RawMorphologyImages.tar.gz + "input_morphology": "s3://openproblems-data/resources/raw_data/bruker_cosmx/Lung9_Rep1+RawMorphologyImages.tar.gz", "segmentation_id": ["cell"], "output": "output.zarr", - "dataset_id": "bruker_cosmx/bruker_human_nsclc_cosmx/lung9_rep1", + "dataset_id": "bruker_cosmx/bruker_human_lung_cancer_cosmx/lung9_rep1", "dataset_name": "value", "dataset_url": "https://nanostring.com/products/cosmx-spatial-molecular-imager/ffpe-dataset/nsclc-ffpe-dataset/", "dataset_reference": "value", @@ -73,71 +71,185 @@ "dataset_organism": "human", } meta = { - #"temp_dir": "./temp/datasets/bruker_cosmx", - "temp_dir": "/Volumes/T7/G3_temp/datasets/cosmx/test_folder" + "temp_dir": "./temp/datasets/bruker_cosmx_nsclc/temp", } - ## VIASH END assert ("cell" in par["segmentation_id"]) and (len(par["segmentation_id"]) == 1), "Currently cell labels are definitely assigned in this script. And cosmx does not provide other segmentations." t0 = datetime.now() -# Download urls for sample -URL = f"https://nanostring-public-share.s3.us-west-2.amazonaws.com/SMI-Compressed/{par['sample']}/{par['sample']}+SMI+Flat+data.tar.gz" -URL_MORPHOLOGY = f"https://nanostring-public-share.s3.us-west-2.amazonaws.com/SMI-Compressed/{par['sample']}/{par['sample']}+RawMorphologyImages.tar.gz" +def log(msg): + print(datetime.now() - t0, msg, flush=True) -# Define temp dir and file names +# Define temp dir TMP_DIR = Path(meta["temp_dir"] or "/tmp") TMP_DIR.mkdir(parents=True, exist_ok=True) -FILE_NAME = TMP_DIR / (par["sample"] + "+SMI+Flat+data.tar.gz") -FILE_NAME_MORPHOLOGY = TMP_DIR / (par["sample"] + "+RawMorphologyImages.tar.gz") -DATA_DIR = TMP_DIR / par["sample"] / (par["sample"] + "-Flat_files_and_images") -MORPHOLOGY_DIR = TMP_DIR / par["sample"] / (par["sample"] + "-RawMorphologyImages") - -# Download flat files and cell labels -print(datetime.now() - t0, "Download flat files and cell labels", flush=True) -os.system(f"wget {URL} -O '{FILE_NAME}'") - -# Extract tar.gz files -print(datetime.now() - t0, "Extract tar.gz of flat files and cell labels", flush=True) -with tarfile.open(FILE_NAME, "r:gz") as tar: - tar.extractall(TMP_DIR) - -# Check that flat files are present -FLAT_FILES_ENDINGS = ["_exprMat_file.csv", "_fov_positions_file.csv", "_metadata_file.csv", "_tx_file.csv"] #, "polygons.csv"] + +# Only the 4th morphology z-plane is kept (see module docstring). The regex is +# case-insensitive because the raw files use `.TIF`. +_Z_PLANE_RE = re.compile(r"_Z004\.tif$", re.I) + + +def _open_source(src): + """Seekable binary handle for a local path or an s3:// URL (streamed, never fully downloaded).""" + s = str(src) + if s.startswith("s3://"): + import s3fs + # openproblems-data is public; anonymous read is deterministic and needs no creds. + fs = s3fs.S3FileSystem(anon=True, default_block_size=32 * 2**20) + return fs.open(s, "rb") + return open(s, "rb") + + +def _member_wanted_default(name: str) -> bool: + parts = name.split("/") + return not (name.startswith("__MACOSX/") or parts[-1] == ".DS_Store") + + +def _member_wanted_morphology(name: str) -> bool: + return _member_wanted_default(name) and bool(_Z_PLANE_RE.search(name)) + + +def extract_targz(input_targz, output_dir: Path, wanted=_member_wanted_default): + """ + Stream a .tar.gz (local path or s3:// URL) into output_dir, writing only the members for + which `wanted(name)` is True. The archive is read once, sequentially (gzip is not seekable), + but only wanted members are decoded to disk — so for the morphology tarball only the ~1/8 of + files that are Z004 planes ever hit scratch. The archive itself is never fully downloaded to + a local file first. + """ + output_dir = Path(output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + kept = skipped = 0 + kept_bytes = 0 + with _open_source(input_targz) as fh, tarfile.open(fileobj=fh, mode="r|gz") as tar: + for member in tar: + if not member.isfile() or not wanted(member.name): + skipped += 1 + continue + # Flatten: strip any leading directory components, keep a stable relative path so + # globbing below is layout-independent. + target = output_dir.joinpath(*Path(member.name).parts) + target.parent.mkdir(parents=True, exist_ok=True) + extracted = tar.extractfile(member) + if extracted is None: + skipped += 1 + continue + with extracted as src, open(target, "wb") as dst: + shutil.copyfileobj(src, dst) + kept += 1 + kept_bytes += member.size + log(f"Extracted {kept} files ({kept_bytes / 1e9:.2f} GB); skipped {skipped} members") + + +######################################### +# Extract raw + morphology archives # +######################################### + +log("Extract tar.gz of flat files and cell labels") +FLAT_EXTRACTED = TMP_DIR / "input_raw" +extract_targz(par["input_raw"], FLAT_EXTRACTED) + +# DATA_DIR is the directory that holds the flat CSVs + CellLabels (the "*-Flat_files_and_images" +# folder). Locate it via the tx flat file rather than hard-coding the sample-name nesting. +tx_files = sorted(FLAT_EXTRACTED.rglob("*_tx_file.csv")) +assert tx_files, ( + f"No '*_tx_file.csv' found under {FLAT_EXTRACTED}. " + f"Contents: {[p.name for p in FLAT_EXTRACTED.rglob('*')][:50]}" +) +DATA_DIR = tx_files[0].parent +log(f"Detected flat-files directory: {DATA_DIR}") + +# Check that all flat files are present +FLAT_FILES_ENDINGS = ["_exprMat_file.csv", "_fov_positions_file.csv", "_metadata_file.csv", "_tx_file.csv"] for ending in FLAT_FILES_ENDINGS: - if any(f.endswith(ending) for f in os.listdir(DATA_DIR)): - print(f"Flat file with ending {ending} is present", flush=True) - else: - print(f"Flat file with ending {ending} is missing", flush=True) - -# Download image files -print(datetime.now() - t0, "Download image files", flush=True) -os.system(f"wget {URL_MORPHOLOGY} -O '{FILE_NAME_MORPHOLOGY}'") - -# Extract tar.gz files -print(datetime.now() - t0, "Extract tar.gz of image files", flush=True) -with tarfile.open(FILE_NAME_MORPHOLOGY, "r:gz") as tar: - tar.extractall(TMP_DIR) - -# Move image files to DATA_DIR / "Morphology2D" -print(datetime.now() - t0, f"Move image files of plane Z004 to {DATA_DIR / 'Morphology2D'} and rename.", flush=True) -(DATA_DIR / "Morphology2D").mkdir(parents=True, exist_ok=True) -for image_file in MORPHOLOGY_DIR.glob("*_Z004.TIF"): - shutil.move(image_file, DATA_DIR / "Morphology2D" / image_file.name.replace("_Z004.TIF", ".tif")) + present = any(f.name.endswith(ending) for f in DATA_DIR.iterdir()) + log(f"Flat file with ending {ending} is {'present' if present else 'MISSING'}") +log("Extract tar.gz of morphology images (Z004 plane only)") +MORPH_EXTRACTED = TMP_DIR / "input_morphology" +extract_targz(par["input_morphology"], MORPH_EXTRACTED, wanted=_member_wanted_morphology) +# Move the Z004 morphology tifs into DATA_DIR / "Morphology2D", renamed so sopa reads one 2D +# image per FOV (e.g. _F001_Z004.TIF -> _F001.tif). +log(f"Move Z004 morphology images to {DATA_DIR / 'Morphology2D'} and rename") +(DATA_DIR / "Morphology2D").mkdir(parents=True, exist_ok=True) +z004_tifs = sorted(MORPH_EXTRACTED.rglob("*_Z004.[Tt][Ii][Ff]")) +assert z004_tifs, f"No '*_Z004.TIF' morphology images found under {MORPH_EXTRACTED}" +for image_file in z004_tifs: + dest_name = _Z_PLANE_RE.sub(".tif", image_file.name) + shutil.move(str(image_file), DATA_DIR / "Morphology2D" / dest_name) +log(f"Moved {len(z004_tifs)} morphology images") + +# sopa expects a CellLabels/ folder. The NSCLC flat archive ships one directly, but some CosMx +# exports only ship per-FOV label tifs inside FOV* folders — handle both, matching the general +# bruker_cosmx loader. See https://github.com/gustaveroussy/sopa/issues/285 +labels_dir = DATA_DIR / "CellLabels" +fov_label_tifs = sorted(DATA_DIR.glob("FOV*/CellLabels_F*.tif")) +if labels_dir.is_dir(): + log("CellLabels folder already present in raw data") +elif fov_label_tifs: + log(f"Symlink {len(fov_label_tifs)} per-FOV CellLabels tifs into {labels_dir}") + labels_dir.mkdir(parents=True, exist_ok=True) + for tif in fov_label_tifs: + os.symlink(tif.resolve(), labels_dir / tif.name) +else: + raise AssertionError(f"No CellLabels folder and no per-FOV CellLabels tifs found in {DATA_DIR}") + +############################################# +# Memory optimization: lean transcript read # +############################################# +# sopa's CosMx reader loads the ENTIRE `*_tx_file.csv` into a pandas DataFrame in one +# `pd.read_csv` (sopa/io/reader/cosmx.py::_CosMXReader.read_transcripts). The string columns +# ('target', 'CellComp', ...) come in as object dtype (~60 B/row) — the peak of the loader. +# We wrap the `read_csv` used inside sopa's cosmx module so that, *only for the transcripts +# file*, we (a) read the 'target' feature column as categorical and (b) keep just the columns +# sopa's stitched-FOV path (read_transcripts + _get_global_cell_id) and the downstream schema +# actually use. Every other sopa read is left untouched. (Same shim as the general loader.) +from sopa.io.reader import cosmx as _cosmx_mod + +_real_pd = _cosmx_mod.pd +_real_read_csv = _real_pd.read_csv + +_TX_KEEP = ["fov", "cell_ID", "target", "x_global_px", "y_global_px", "z"] +_TX_REQUIRED = {"target", "x_global_px", "y_global_px", "fov", "cell_ID"} + + +def _lean_read_csv(filepath_or_buffer, *args, **kwargs): + name = str(filepath_or_buffer) + if "_tx_file.csv" in name and "usecols" not in kwargs: + header = _real_read_csv( + filepath_or_buffer, nrows=0, + compression=kwargs.get("compression", "infer"), + ) + cols = set(header.columns) + if _TX_REQUIRED.issubset(cols): + kwargs["usecols"] = [c for c in _TX_KEEP if c in cols] + dtype = dict(kwargs.get("dtype") or {}) + dtype.setdefault("target", "category") + kwargs["dtype"] = dtype + log(f"Lean transcript read of {Path(name).name}: usecols={kwargs['usecols']}, target=category") + return _real_read_csv(filepath_or_buffer, *args, **kwargs) + + +class _PandasReadCsvProxy: + """Forwards every attribute to pandas, but leans the transcripts read_csv.""" + def __init__(self, real): + self.__dict__["_real"] = real + def read_csv(self, *args, **kwargs): + return _lean_read_csv(*args, **kwargs) + def __getattr__(self, item): + return getattr(self._real, item) + + +_cosmx_mod.pd = _PandasReadCsvProxy(_real_pd) ######################################### # Convert raw files to spatialdata zarr # ######################################### -# from pathlib import Path -# import sopa -# DATA_DIR = Path("/Volumes/T7/G3_temp/datasets/cosmx/test_folder/Lung9_Rep1/Lung9_Rep1-Flat_files_and_images") - -print(datetime.now() - t0, "Convert raw files to spatialdata zarr", flush=True) +log("Convert raw files to spatialdata zarr") # The tif images of the NSCLC samples miss metadata information, so we hardcode the channels here. def fixed_get_morphology_coords(images_dir: Path) -> list[str]: @@ -146,27 +258,26 @@ def fixed_get_morphology_coords(images_dir: Path) -> list[str]: sopa.io.reader.cosmx._get_morphology_coords = fixed_get_morphology_coords sdata = sopa.io.cosmx( - DATA_DIR, - dataset_id=None, - fov=None, - read_proteins=False, - cells_labels=True, - cells_table=True, - cells_polygons=False, - flip_image=False + DATA_DIR, + dataset_id=None, + fov=None, + read_proteins=False, + cells_labels=True, + cells_table=True, + cells_polygons=False, # NSCLC flat files ship no -polygons.csv; derived from labels below + flip_image=False, ) # Retrieve polygons from segmentation (no polygons file in flat files present for NSCLC samples) +log("Derive cell polygons from stitched labels") sdata["cells_polygons"] = sd.to_polygons(sdata["stitched_labels"]) - ############### # Rename keys # ############### -print(datetime.now() - t0, "Rename keys", flush=True) - +log("Rename keys") elements_renaming_map = { - "stitched_image" : "morphology_mip", + "stitched_image" : "morphology_mip", "stitched_labels" : "cell_labels", "points" : "transcripts", "cells_polygons" : "cell_boundaries", @@ -174,38 +285,45 @@ def fixed_get_morphology_coords(images_dir: Path) -> list[str]: } for old_key, new_key in elements_renaming_map.items(): + if old_key not in sdata: + log(f"Element '{old_key}' not present, skipping rename to '{new_key}'") + continue sdata[new_key] = sdata[old_key] del sdata[old_key] -# Rename transcript column (somehow overwriting the 'target' column leads to an error, so instead we add a duplicate with the right name) -#sdata['transcripts'] = sdata['transcripts'].rename(columns={"global_cell_id":"cell_id", "target":"feature_name"}) -sdata['transcripts'] = sdata['transcripts'].rename(columns={"global_cell_id":"cell_id"}) +# Add transcript columns with the names expected downstream. +sdata['transcripts']["cell_id"] = sdata['transcripts']["global_cell_id"] sdata['transcripts']["feature_name"] = sdata['transcripts']["target"] ######################################### # Throw out all channels except of DAPI # ######################################### -print(datetime.now() - t0, "Throw out all channels except of 'DAPI'", flush=True) +log("Throw out all channels except of 'DAPI'") # TODO: in the future we want to keep PolyT and Cellbound1/2/3 stains. Note however, that somehow saving or plotting the sdata fails when # these channels aren't excluded, not sure why... sdata["morphology_mip"] = sdata["morphology_mip"].sel(c=["DAPI"]) - ############################## # Add info to metadata table # ############################## -print(datetime.now() - t0, "Add metadata to table", flush=True) - -#TODO: values as input variables +log("Add metadata to table") +# The common iST API requires a per-cell unique `cell_id` in obs. sopa keys the table by the +# global cell id (the index), so expose that as `cell_id`. But the CosMx metadata already +# carries a per-FOV-*local* `cell_ID` column, and obs cannot hold two keys that differ only in +# case ('cell_ID' vs 'cell_id') — spatialdata's zarr writer rejects that as an invalid name. +# Rename the vendor local id out of the way first. +if "cell_ID" in sdata["table"].obs.columns: + sdata["table"].obs.rename(columns={"cell_ID": "cell_ID_local"}, inplace=True) +sdata["table"].obs["cell_id"] = sdata["table"].obs.index.astype(str) for key in ["dataset_id", "dataset_name", "dataset_url", "dataset_reference", "dataset_summary", "dataset_description", "dataset_organism", "segmentation_id"]: sdata["table"].uns[key] = par[key] ######### # Write # ######### -print(datetime.now() - t0, f"Writing to {par['output']}", flush=True) +log(f"Writing to {par['output']}") sdata.write(par["output"]) -print(datetime.now() - t0, "Done", flush=True) \ No newline at end of file +log(f"Done in {datetime.now() - t0}") diff --git a/src/methods_transcript_assignment/fastreseg/input.py b/src/methods_transcript_assignment/fastreseg/input.py index 09e59ab37..8279a4d65 100644 --- a/src/methods_transcript_assignment/fastreseg/input.py +++ b/src/methods_transcript_assignment/fastreseg/input.py @@ -129,9 +129,17 @@ def parse_arguments(): y_coords = transcripts.y.compute().to_numpy(dtype=np.int64) x_coords = transcripts.x.compute().to_numpy(dtype=np.int64) if isinstance(sdata_segm["segmentation"], xr.DataTree): - label_image = sdata_segm["segmentation"]["scale0"].image.to_numpy() + label_image = sdata_segm["segmentation"]["scale0"].image.to_numpy() else: label_image = sdata_segm["segmentation"].to_numpy() +# Clamp coords to the label-image bounds: transcripts at the crop boundary can +# round a few pixels past the raster edge, or the transcript field of view can +# extend beyond the segmented raster (real data), so label_image[y, x] would +# raise IndexError. Matches basic_transcript_assignment; edge/background at the border. +n_oob = int(np.count_nonzero((y_coords < 0) | (y_coords >= label_image.shape[0]) | (x_coords < 0) | (x_coords >= label_image.shape[1]))) +y_coords = np.clip(y_coords, 0, label_image.shape[0] - 1) +x_coords = np.clip(x_coords, 0, label_image.shape[1] - 1) +print(f"Clamped {n_oob}/{len(x_coords)} transcripts outside the {label_image.shape[0]}x{label_image.shape[1]} label image to its edge", flush=True) cell_id_dask_series = dask.dataframe.from_dask_array( dask.array.from_array( label_image[y_coords, x_coords], chunks=tuple(sdata['transcripts'].map_partitions(len).compute()) From 54f023ef48409760a30e45729170ad24303b8c13 Mon Sep 17 00:00:00 2001 From: dariarom94 Date: Fri, 31 Jul 2026 10:11:26 +0200 Subject: [PATCH 52/60] adjust bruker nsclc loader --- .../process_bruker_cosmx_nsclc/config.vsh.yaml | 10 +++++++--- .../workflows/process_bruker_cosmx_nsclc/main.nf | 3 ++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/datasets/workflows/process_bruker_cosmx_nsclc/config.vsh.yaml b/src/datasets/workflows/process_bruker_cosmx_nsclc/config.vsh.yaml index 572ebd6eb..9d864cb66 100644 --- a/src/datasets/workflows/process_bruker_cosmx_nsclc/config.vsh.yaml +++ b/src/datasets/workflows/process_bruker_cosmx_nsclc/config.vsh.yaml @@ -5,9 +5,13 @@ argument_groups: - name: Inputs arguments: - type: string - name: --sample - example: "Lung9_Rep1" - description: "Sample id" + name: --input_raw + example: "s3://openproblems-data/resources/raw_data/bruker_cosmx/Lung9_Rep1+SMI+Flat+data.tar.gz" + description: "URL/path to the flat-files + cell-labels archive. A string (not a staged file) so the loader can stream it and extract only what sopa needs — see the loader's --input_raw." + - type: string + name: --input_morphology + example: "s3://openproblems-data/resources/raw_data/bruker_cosmx/Lung9_Rep1+RawMorphologyImages.tar.gz" + description: "URL/path to the raw morphology-images archive. Streamed in place; only the Z004 plane per FOV is extracted — see the loader's --input_morphology." - type: string name: --segmentation_id default: ["cell"] diff --git a/src/datasets/workflows/process_bruker_cosmx_nsclc/main.nf b/src/datasets/workflows/process_bruker_cosmx_nsclc/main.nf index 84dd78b09..1a32d3ca4 100644 --- a/src/datasets/workflows/process_bruker_cosmx_nsclc/main.nf +++ b/src/datasets/workflows/process_bruker_cosmx_nsclc/main.nf @@ -20,7 +20,8 @@ workflow run_wf { | bruker_cosmx_nsclc.run( fromState: [ - "sample", + "input_raw", + "input_morphology", "segmentation_id", "dataset_id", "dataset_name", From 6e8b9ea03e9342ef2157d9d77b4bd54068e5a55d Mon Sep 17 00:00:00 2001 From: dariarom94 Date: Fri, 31 Jul 2026 10:13:09 +0200 Subject: [PATCH 53/60] setup --- .../spatial/mirror_bruker_to_s3.sh | 39 ++++++++++++++----- .../spatial/process_bruker_cosmx_nsclc.sh | 32 ++++++++++----- src/base/labels_nebius.config | 15 ++++++- 3 files changed, 65 insertions(+), 21 deletions(-) diff --git a/scripts/create_resources/spatial/mirror_bruker_to_s3.sh b/scripts/create_resources/spatial/mirror_bruker_to_s3.sh index 4a02bf7db..b695b0f08 100644 --- a/scripts/create_resources/spatial/mirror_bruker_to_s3.sh +++ b/scripts/create_resources/spatial/mirror_bruker_to_s3.sh @@ -23,17 +23,30 @@ set -euo pipefail # --- Config ----------------------------------------------------------------- SRC_BASE="https://smi-public.objects.liquidweb.services" -S3_DEST="${S3_DEST:-s3://openproblems-data/resources_raw/bruker_cosmx}" +# NSCLC lung-cancer samples live on a different NanoString host, one folder per sample. +NSCLC_BASE="https://nanostring-public-share.s3.us-west-2.amazonaws.com/SMI-Compressed" +# Where the loaders read from (see the process_bruker_cosmx*.sh run scripts). +S3_DEST="${S3_DEST:-s3://openproblems-data/resources/raw_data/bruker_cosmx}" SCRATCH_DIR="${SCRATCH_DIR:-$PWD/bruker_mirror_scratch}" -# Files to mirror. The URL-encoded names are what the server serves; the second -# column is the (decoded) name to store under on S3. +# Files to mirror. Each entry is "SOURCE|LOCAL_NAME": +# - SOURCE is either a full https URL, or a name relative to SRC_BASE (the mouse/liver host). +# The URL-encoded names are what the liquidweb server serves. +# - LOCAL_NAME is the (decoded) name to store under on S3. FILES=( "HalfBrain.zip|HalfBrain.zip" "Half%20%20Brain%20simple%20%20files%20.zip|Half Brain simple files.zip" "NormalLiverFiles.zip|NormalLiverFiles.zip" ) +# NSCLC lung-cancer samples: each ships a flat-files+cell-labels archive and a +# raw-morphology-images archive. The bruker_cosmx_nsclc loader streams both from S3. +NSCLC_SAMPLES=(Lung5_Rep1 Lung5_Rep2 Lung5_Rep3 Lung6 Lung9_Rep1 Lung9_Rep2 Lung12 Lung13) +for s in "${NSCLC_SAMPLES[@]}"; do + FILES+=("$NSCLC_BASE/$s/${s}+SMI+Flat+data.tar.gz|${s}+SMI+Flat+data.tar.gz") + FILES+=("$NSCLC_BASE/$s/${s}+RawMorphologyImages.tar.gz|${s}+RawMorphologyImages.tar.gz") +done + # --- Preflight -------------------------------------------------------------- command -v curl >/dev/null || { echo "ERROR: curl not found" >&2; exit 1; } @@ -64,7 +77,11 @@ s3_size() { for entry in "${FILES[@]}"; do url_name="${entry%%|*}" local_name="${entry##*|}" - url="$SRC_BASE/$url_name" + if [[ "$url_name" == http*://* ]]; then + url="$url_name" + else + url="$SRC_BASE/$url_name" + fi local_path="$SCRATCH_DIR/$local_name" # S3 key = everything after s3://bucket/ @@ -126,9 +143,11 @@ done echo "================================================================" echo "All files mirrored to $S3_DEST" echo -echo "Next: update the input_raw / input_flat_files URLs in" -echo " scripts/create_resources/spatial/process_bruker_cosmx_nebius.sh" -echo "to point at the S3 paths, e.g.:" -echo " input_raw: $S3_DEST/HalfBrain.zip" -echo " input_flat_files: $S3_DEST/Half Brain simple files.zip" -echo " input_raw: $S3_DEST/NormalLiverFiles.zip" +echo "Next: the process_bruker_cosmx*.sh run scripts already point at these S3 paths, e.g.:" +echo " mouse/liver (process_bruker_cosmx_nebius.sh):" +echo " input_raw: $S3_DEST/HalfBrain.zip" +echo " input_flat_files: $S3_DEST/Half Brain simple files.zip" +echo " input_raw: $S3_DEST/NormalLiverFiles.zip" +echo " nsclc (process_bruker_cosmx_nsclc_nebius.sh), per sample:" +echo " input_raw: $S3_DEST/+SMI+Flat+data.tar.gz" +echo " input_morphology: $S3_DEST/+RawMorphologyImages.tar.gz" diff --git a/scripts/create_resources/spatial/process_bruker_cosmx_nsclc.sh b/scripts/create_resources/spatial/process_bruker_cosmx_nsclc.sh index 210540b88..bce4bc243 100644 --- a/scripts/create_resources/spatial/process_bruker_cosmx_nsclc.sh +++ b/scripts/create_resources/spatial/process_bruker_cosmx_nsclc.sh @@ -10,11 +10,16 @@ set -e publish_dir="s3://openproblems-data/resources/datasets" +# Raw NSCLC archives mirrored to S3 (see scripts/create_resources/spatial/mirror_bruker_to_s3.sh). +# Each sample ships two tar.gz archives: the flat files + cell labels, and the raw morphology images. +raw_dir="s3://openproblems-data/resources/raw_data/bruker_cosmx" + cat > /tmp/params.yaml << HERE param_list: - id: "bruker_cosmx/bruker_human_lung_cancer_cosmx/lung5_rep1" - sample: "Lung5_Rep1" + input_raw: "$raw_dir/Lung5_Rep1+SMI+Flat+data.tar.gz" + input_morphology: "$raw_dir/Lung5_Rep1+RawMorphologyImages.tar.gz" dataset_name: "Bruker CosMx Human Lung Cancer Lung5 Rep1" dataset_url: "https://nanostring.com/products/cosmx-spatial-molecular-imager/ffpe-dataset/nsclc-ffpe-dataset/" dataset_summary: "Bruker CosMx Human Lung Cancer Lung5 Rep1 dataset on FFPE." @@ -23,16 +28,18 @@ param_list: segmentation_id: ["cell"] - id: "bruker_cosmx/bruker_human_lung_cancer_cosmx/lung5_rep2" - sample: "Lung5_Rep2" + input_raw: "$raw_dir/Lung5_Rep2+SMI+Flat+data.tar.gz" + input_morphology: "$raw_dir/Lung5_Rep2+RawMorphologyImages.tar.gz" dataset_name: "Bruker CosMx Human Lung Cancer Lung5 Rep2" dataset_url: "https://nanostring.com/products/cosmx-spatial-molecular-imager/ffpe-dataset/nsclc-ffpe-dataset/" dataset_summary: "Bruker CosMx Human Lung Cancer Lung5 Rep2 dataset on FFPE." dataset_description: "Bruker CosMx Human Lung Cancer Lung5 Rep2 dataset on FFPE. Adenocarcinoma, G1, T2aN2M0, IIIA, 75% tumour content." dataset_organism: "homo_sapiens" segmentation_id: ["cell"] - + - id: "bruker_cosmx/bruker_human_lung_cancer_cosmx/lung5_rep3" - sample: "Lung5_Rep3" + input_raw: "$raw_dir/Lung5_Rep3+SMI+Flat+data.tar.gz" + input_morphology: "$raw_dir/Lung5_Rep3+RawMorphologyImages.tar.gz" dataset_name: "Bruker CosMx Human Lung Cancer Lung5 Rep3" dataset_url: "https://nanostring.com/products/cosmx-spatial-molecular-imager/ffpe-dataset/nsclc-ffpe-dataset/" dataset_summary: "Bruker CosMx Human Lung Cancer Lung5 Rep3 dataset on FFPE." @@ -41,7 +48,8 @@ param_list: segmentation_id: ["cell"] - id: "bruker_cosmx/bruker_human_lung_cancer_cosmx/lung6" - sample: "Lung6" + input_raw: "$raw_dir/Lung6+SMI+Flat+data.tar.gz" + input_morphology: "$raw_dir/Lung6+RawMorphologyImages.tar.gz" dataset_name: "Bruker CosMx Human Lung Cancer Lung6" dataset_url: "https://nanostring.com/products/cosmx-spatial-molecular-imager/ffpe-dataset/nsclc-ffpe-dataset/" dataset_summary: "Bruker CosMx Human Lung Cancer Lung6 dataset on FFPE." @@ -50,7 +58,8 @@ param_list: segmentation_id: ["cell"] - id: "bruker_cosmx/bruker_human_lung_cancer_cosmx/lung9_rep1" - sample: "Lung9_Rep1" + input_raw: "$raw_dir/Lung9_Rep1+SMI+Flat+data.tar.gz" + input_morphology: "$raw_dir/Lung9_Rep1+RawMorphologyImages.tar.gz" dataset_name: "Bruker CosMx Human Lung Cancer Lung9 Rep1" dataset_url: "https://nanostring.com/products/cosmx-spatial-molecular-imager/ffpe-dataset/nsclc-ffpe-dataset/" dataset_summary: "Bruker CosMx Human Lung Cancer Lung9 Rep1 dataset on FFPE." @@ -59,7 +68,8 @@ param_list: segmentation_id: ["cell"] - id: "bruker_cosmx/bruker_human_lung_cancer_cosmx/lung9_rep2" - sample: "Lung9_Rep2" + input_raw: "$raw_dir/Lung9_Rep2+SMI+Flat+data.tar.gz" + input_morphology: "$raw_dir/Lung9_Rep2+RawMorphologyImages.tar.gz" dataset_name: "Bruker CosMx Human Lung Cancer Lung9 Rep2" dataset_url: "https://nanostring.com/products/cosmx-spatial-molecular-imager/ffpe-dataset/nsclc-ffpe-dataset/" dataset_summary: "Bruker CosMx Human Lung Cancer Lung9 Rep2 dataset on FFPE." @@ -68,7 +78,8 @@ param_list: segmentation_id: ["cell"] - id: "bruker_cosmx/bruker_human_lung_cancer_cosmx/lung12" - sample: "Lung12" + input_raw: "$raw_dir/Lung12+SMI+Flat+data.tar.gz" + input_morphology: "$raw_dir/Lung12+RawMorphologyImages.tar.gz" dataset_name: "Bruker CosMx Human Lung Cancer Lung12" dataset_url: "https://nanostring.com/products/cosmx-spatial-molecular-imager/ffpe-dataset/nsclc-ffpe-dataset/" dataset_summary: "Bruker CosMx Human Lung Cancer Lung12 dataset on FFPE." @@ -77,7 +88,8 @@ param_list: segmentation_id: ["cell"] - id: "bruker_cosmx/bruker_human_lung_cancer_cosmx/lung13" - sample: "Lung13" + input_raw: "$raw_dir/Lung13+SMI+Flat+data.tar.gz" + input_morphology: "$raw_dir/Lung13+RawMorphologyImages.tar.gz" dataset_name: "Bruker CosMx Human Lung Cancer Lung13" dataset_url: "https://nanostring.com/products/cosmx-spatial-molecular-imager/ffpe-dataset/nsclc-ffpe-dataset/" dataset_summary: "Bruker CosMx Human Lung Cancer Lung13 dataset on FFPE." @@ -98,4 +110,4 @@ tw launch https://github.com/openproblems-bio/task_ist_preprocessing.git \ --workspace 53907369739130 \ --params-file /tmp/params.yaml \ --config common/nextflow_helpers/labels_tw.config \ - --labels datasets,bruker_cosmx_nsclc \ No newline at end of file + --labels datasets,bruker_cosmx_nsclc diff --git a/src/base/labels_nebius.config b/src/base/labels_nebius.config index 67ab80153..93ab8d178 100644 --- a/src/base/labels_nebius.config +++ b/src/base/labels_nebius.config @@ -173,7 +173,20 @@ process { // cannot write Nextflow's root-owned task scratch dir (.command.log / // .command.begin / .exitcode -> "Permission denied"). Harmless for already-root // GPU images. If segger moves to another gpu label, carry this with it. - pod = [[nodeSelector: 'nebius.com/node-group-id=mk8snodegroup-e00jp7hyqr094tmy35'], [imagePullPolicy: 'Always'], [runAsUser: 0]] + // + // emptyDir Memory -> /dev/shm — Kubernetes defaults a container's /dev/shm to 64 MB, + // but segger's Lightning training uses a torch_geometric multi-worker DataLoader that + // moves each collated graph Batch to the main process through /dev/shm (torch's + // file-descriptor sharing strategy, _new_using_fd_cpu). 64 MB overflows on the very + // first batch -> "No space left on device (28)" writing /torch_* / "Unexpected bus + // error ... insufficient shared memory (shm)", killing the DataLoader workers in the + // pre-train sanity-check. NB: the `*sharedmem` labels' `--shm-size` containerOptions do + // NOT help here — containerOptions is a docker/singularity concept the k8s executor + // ignores; on k8s /dev/shm must be enlarged via a pod volume. The tmpfs only consumes + // RAM for pages actually written (idle co-tenants on gpuh100 pay ~0) and counts against + // the pod memory limit, so sizeLimit=16Gi sits comfortably under the 120-180 GB request. + // Carry this with runAsUser:0 if segger moves labels. + pod = [[nodeSelector: 'nebius.com/node-group-id=mk8snodegroup-e00jp7hyqr094tmy35'], [imagePullPolicy: 'Always'], [runAsUser: 0], [emptyDir: [medium: 'Memory', sizeLimit: '16Gi'], mountPath: '/dev/shm']] containerOptions = { workflow.containerEngine == "singularity" ? '--nv': ( workflow.containerEngine == "docker" ? '--gpus all': null ) } } From 44ad7af04c99a4a3169da0c14971a48c31fb7854 Mon Sep 17 00:00:00 2001 From: dariarom94 Date: Sat, 1 Aug 2026 01:55:09 +0200 Subject: [PATCH 54/60] method correction --- .../fastreseg/input.py | 14 +++ .../segger/script.py | 93 ++++++++++++++----- 2 files changed, 86 insertions(+), 21 deletions(-) diff --git a/src/methods_transcript_assignment/fastreseg/input.py b/src/methods_transcript_assignment/fastreseg/input.py index 8279a4d65..967d9ed41 100644 --- a/src/methods_transcript_assignment/fastreseg/input.py +++ b/src/methods_transcript_assignment/fastreseg/input.py @@ -132,6 +132,20 @@ def parse_arguments(): label_image = sdata_segm["segmentation"]["scale0"].image.to_numpy() else: label_image = sdata_segm["segmentation"].to_numpy() +# Guard against an empty segmentation. If the label image has no cells (all zeros) +# every transcript is assigned to background (cell_id 0), generate_adata drops them +# all, and the downstream tacco annotation then divides by a zero count norm and +# fails with a cryptic "divide by zero" instead of a useful message. Observed for the +# vizgen lung-cancer merscope dataset, whose polygon->label rasterization produced an +# all-zero image (for custom_segmentation this means the dataset's `cell_labels` is +# empty; re-process the dataset). Mirrors the guard in methods_transcript_assignment/pciseq. +if label_image.max() == 0: + raise ValueError( + f"Segmentation '{input_segmentation_path}' contains no cells: the label image " + f"(shape={label_image.shape}, dtype={label_image.dtype}) is entirely zero. FastReseg " + f"cannot assign transcripts to cells " + f"(for custom_segmentation this means the dataset's `cell_labels` is empty)." + ) # Clamp coords to the label-image bounds: transcripts at the crop boundary can # round a few pixels past the raster edge, or the transcript field of view can # extend beyond the segmented raster (real data), so label_image[y, x] would diff --git a/src/methods_transcript_assignment/segger/script.py b/src/methods_transcript_assignment/segger/script.py index 040417644..6bbb28ad6 100644 --- a/src/methods_transcript_assignment/segger/script.py +++ b/src/methods_transcript_assignment/segger/script.py @@ -76,6 +76,25 @@ segmentation_coord_systems = sd.transformations.get_transformation(sdata_segm["segmentation"], get_all=True).keys() assert par['coordinate_system'] in segmentation_coord_systems, f"Coordinate system '{par['coordinate_system']}' not found in input data." +# Guard against an empty (all-background) segmentation. sd.to_polygons() on an all-zero +# label image returns an EMPTY GeoDataFrame, which then crashes deep inside spatialdata's +# ShapesModel.parse (`data["geometry"].iloc[0]` -> IndexError: "single positional indexer +# is out-of-bounds"). This happens for datasets whose cell_labels are all-zero from the +# stale pre-0.8.0 spatialdata rasterize bug -- e.g. 2022_vizgen_human_lung_cancer_merfish_ +# combined via custom_segmentation (which just copies cell_labels). The real fix is to +# re-process that dataset under spatialdata>=0.8.0; fail fast here with an actionable +# message rather than the cryptic pandas IndexError. Mirrors the guard in pciseq/script.py. +_seg_elem = sdata_segm["segmentation"] +_seg_arr = _seg_elem["scale0"].image if isinstance(_seg_elem, xr.DataTree) else _seg_elem +if int(_seg_arr.data.max()) == 0: + raise ValueError( + "Segmentation label image is empty (all background, no cells) -- segger cannot " + "assign transcripts to zero cells. This typically means the dataset's cell_labels " + "are all-zero from the stale pre-0.8.0 spatialdata rasterize bug (e.g. " + "2022_vizgen_human_lung_cancer_merfish_combined via custom_segmentation). " + "Re-process that dataset under spatialdata>=0.8.0 to fix it at the source." + ) + ########################################## # boundaries.parquet (from segmentation) # ########################################## @@ -141,15 +160,35 @@ label_image = sdata_segm["segmentation"]["scale0"].image.to_numpy() else: label_image = sdata_segm["segmentation"].to_numpy() -# Clip to the label image bounds (transcripts can sit just outside after transform). -n_oob = int(np.count_nonzero((y_coords < 0) | (y_coords >= label_image.shape[0]) | (x_coords < 0) | (x_coords >= label_image.shape[1]))) -y_coords = np.clip(y_coords, 0, label_image.shape[0] - 1) -x_coords = np.clip(x_coords, 0, label_image.shape[1] - 1) -print(f"Clamped {n_oob}/{len(x_coords)} transcripts outside the {label_image.shape[0]}x{label_image.shape[1]} label image to its edge", flush=True) -prior_cell_id = label_image[y_coords, x_coords].astype(np.int64) # 0 == background - -# Canonical transcripts frame (native coordinates, clean RangeIndex). Its row -# order matches prior_cell_id and the row_index segger reports back below. +# Transcripts can land outside the label image after the transform (e.g. the combine step +# crops the image/labels to ~20000x20000 but transcripts span a larger field -> here ~38% +# of them fall outside). Those OOB transcripts must NOT be fed to segger: segger tiles the +# transcript field into a heterogeneous graph with 'tx' (transcript) and 'bd' (boundary/ +# cell) node types, and a tile covering a transcripts-only region (no boundaries) produces a +# training mini-batch with ZERO 'bd' nodes. segger's positional encoder then runs +# `torch.zeros((batch.max()+1, 2))` on an empty batch -> `RuntimeError: max(): Expected +# reduction dim to be specified for input.numel() == 0`, which kills training MID-RUN (only +# when the DataLoader's shuffle happens to draw an all-OOB mini-batch, so it can survive +# several epochs first). A transcript outside the segmentation can't belong to any segmented +# cell anyway, so we EXCLUDE it from the segger input and leave it unassigned (cell_id 0) in +# the final output. Datasets whose segmentation covers the whole transcript field are +# unaffected (in_bounds all-True, n_oob == 0). +in_bounds = ( + (y_coords >= 0) & (y_coords < label_image.shape[0]) + & (x_coords >= 0) & (x_coords < label_image.shape[1]) +) +n_oob = int((~in_bounds).sum()) +y_look = np.clip(y_coords, 0, label_image.shape[0] - 1) +x_look = np.clip(x_coords, 0, label_image.shape[1] - 1) +prior_cell_id = label_image[y_look, x_look].astype(np.int64) # 0 == background +prior_cell_id[~in_bounds] = 0 # OOB -> background (excluded from segger regardless) +print(f"{n_oob}/{len(in_bounds)} transcripts fall outside the " + f"{label_image.shape[0]}x{label_image.shape[1]} label image; excluding them from the " + f"segger input (they stay unassigned in the output).", flush=True) + +# Canonical transcripts frame (native coordinates, clean RangeIndex). Its row order matches +# prior_cell_id / in_bounds; the IN-BOUNDS subset's order matches the parquet we write and +# hence segger's reported row_index (mapped back through seg_orig_idx below). tx_pd = transcripts_reset.compute() n_tx = len(tx_pd) @@ -158,19 +197,30 @@ else: transcript_id = np.arange(n_tx, dtype=np.uint64) +# Full-frame positions of the transcripts actually handed to segger. segger's output +# row_index indexes into the (in-bounds) parquet row order; seg_orig_idx maps it back. +seg_orig_idx = np.nonzero(in_bounds)[0] +if seg_orig_idx.size == 0: + raise ValueError( + "No transcripts fall within the segmentation label image, so segger has nothing to " + "assign. This usually means the transcripts and segmentation are in mismatched " + "coordinate frames (check the dataset's crop / transforms)." + ) + +n_in = int(in_bounds.sum()) tx_out = pd.DataFrame({ - "transcript_id": transcript_id, - "x_location": tx_pd["x"].to_numpy().astype(np.float32), - "y_location": tx_pd["y"].to_numpy().astype(np.float32), - "feature_name": tx_pd["feature_name"].astype(str).to_numpy(), + "transcript_id": transcript_id[in_bounds], + "x_location": tx_pd["x"].to_numpy().astype(np.float32)[in_bounds], + "y_location": tx_pd["y"].to_numpy().astype(np.float32)[in_bounds], + "feature_name": tx_pd["feature_name"].astype(str).to_numpy()[in_bounds], # segger's Xenium loader keys transcripts to boundaries by cell_id string; # background (0) becomes the UNASSIGNED sentinel. - "cell_id": np.where(prior_cell_id > 0, prior_cell_id.astype(str), "UNASSIGNED"), - "qv": np.full(n_tx, 40.0, dtype=np.float32), - "overlaps_nucleus": (prior_cell_id > 0).astype(np.int8), + "cell_id": np.where(prior_cell_id[in_bounds] > 0, prior_cell_id[in_bounds].astype(str), "UNASSIGNED"), + "qv": np.full(n_in, 40.0, dtype=np.float32), + "overlaps_nucleus": (prior_cell_id[in_bounds] > 0).astype(np.int8), }) if "z" in tx_pd.columns: - tx_out.insert(3, "z_location", tx_pd["z"].to_numpy().astype(np.float32)) + tx_out.insert(3, "z_location", tx_pd["z"].to_numpy().astype(np.float32)[in_bounds]) tx_out.to_parquet(XENIUM_DIR / "transcripts.parquet", index=False) del transcripts, y_coords, x_coords, label_image @@ -291,14 +341,15 @@ def run_segger(node_dim): seg = seg[keep_mask] print(f"segger kept {len(seg)} assignments of {n_tx} transcripts", flush=True) -# segger reports row_index into the transcripts.parquet we wrote (i.e. tx_pd -# row order). Factorize the (string) segger_cell_id into contiguous positive -# integers; unassigned transcripts stay 0. +# segger reports row_index into the transcripts.parquet we wrote, which is the IN-BOUNDS +# subset (OOB transcripts were excluded above). Map that back to the full-frame position via +# seg_orig_idx, then factorize the (string) segger_cell_id into contiguous positive integers; +# unassigned / out-of-bounds transcripts stay 0. cell_id_per_tx = np.zeros(n_tx, dtype=np.int64) if len(seg): row_idx = seg["row_index"].to_numpy().astype(np.int64) codes, _ = pd.factorize(seg["segger_cell_id"].astype(str), sort=True) - cell_id_per_tx[row_idx] = codes.astype(np.int64) + 1 + cell_id_per_tx[seg_orig_idx[row_idx]] = codes.astype(np.int64) + 1 ############################################# # Build transcript-assignment output object # From 2b8177cb31923c0b859adcc83e1fbb765615da76 Mon Sep 17 00:00:00 2001 From: dariarom94 Date: Sat, 1 Aug 2026 21:40:53 +0200 Subject: [PATCH 55/60] pin anndata --- src/base/setup_spatialdata_partial.yaml | 2 +- .../resolvi_correction/config.vsh.yaml | 2 +- .../gene_efficiency_correction/config.vsh.yaml | 2 +- src/methods_qc_filter/basic_qc_filter/config.vsh.yaml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/base/setup_spatialdata_partial.yaml b/src/base/setup_spatialdata_partial.yaml index 7e3f4be48..c887a67f0 100644 --- a/src/base/setup_spatialdata_partial.yaml +++ b/src/base/setup_spatialdata_partial.yaml @@ -1,3 +1,3 @@ setup: - type: python - pypi: ["spatialdata>=0.7.3", "anndata>=0.12.0", "zarr>=3.0.0"] + pypi: ["spatialdata>=0.7.3", "anndata>=0.12.0,<0.13", "zarr>=3.0.0"] diff --git a/src/methods_expression_correction/resolvi_correction/config.vsh.yaml b/src/methods_expression_correction/resolvi_correction/config.vsh.yaml index d494ebcc0..543c379aa 100644 --- a/src/methods_expression_correction/resolvi_correction/config.vsh.yaml +++ b/src/methods_expression_correction/resolvi_correction/config.vsh.yaml @@ -48,7 +48,7 @@ engines: - /src/base/setup_txsim_partial.yaml setup: - type: python - pypi: ["anndata>=0.12.0", scvi-tools] + pypi: ["anndata>=0.12.0,<0.13", scvi-tools] - type: native runners: diff --git a/src/methods_gene_efficiency_correction/gene_efficiency_correction/config.vsh.yaml b/src/methods_gene_efficiency_correction/gene_efficiency_correction/config.vsh.yaml index 6f9ea9013..5be364d9c 100644 --- a/src/methods_gene_efficiency_correction/gene_efficiency_correction/config.vsh.yaml +++ b/src/methods_gene_efficiency_correction/gene_efficiency_correction/config.vsh.yaml @@ -31,7 +31,7 @@ engines: - /src/base/setup_txsim_partial.yaml setup: - type: python - pypi: ["anndata>=0.12.0"] + pypi: ["anndata>=0.12.0,<0.13"] - type: native runners: diff --git a/src/methods_qc_filter/basic_qc_filter/config.vsh.yaml b/src/methods_qc_filter/basic_qc_filter/config.vsh.yaml index fca3c2bd6..624672965 100644 --- a/src/methods_qc_filter/basic_qc_filter/config.vsh.yaml +++ b/src/methods_qc_filter/basic_qc_filter/config.vsh.yaml @@ -33,7 +33,7 @@ engines: - /src/base/setup_txsim_partial.yaml setup: - type: python - pypi: ["anndata>=0.12.0"] + pypi: ["anndata>=0.12.0,<0.13"] - type: native runners: From 6c167078a1d56e3ec83df615e7f91b4342cc7368 Mon Sep 17 00:00:00 2001 From: dariarom94 Date: Sat, 1 Aug 2026 21:41:45 +0200 Subject: [PATCH 56/60] mirror nsclc --- .../spatial/mirror_bruker_to_s3.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/scripts/create_resources/spatial/mirror_bruker_to_s3.sh b/scripts/create_resources/spatial/mirror_bruker_to_s3.sh index b695b0f08..4d4a3aebe 100644 --- a/scripts/create_resources/spatial/mirror_bruker_to_s3.sh +++ b/scripts/create_resources/spatial/mirror_bruker_to_s3.sh @@ -19,7 +19,7 @@ # SCRATCH_DIR=/path/to/big/scratch ./mirror_bruker_to_s3.sh set -euo pipefail - +SCRATCH_DIR="/Volumes/SeagateHHD" # --- Config ----------------------------------------------------------------- SRC_BASE="https://smi-public.objects.liquidweb.services" @@ -33,11 +33,11 @@ SCRATCH_DIR="${SCRATCH_DIR:-$PWD/bruker_mirror_scratch}" # - SOURCE is either a full https URL, or a name relative to SRC_BASE (the mouse/liver host). # The URL-encoded names are what the liquidweb server serves. # - LOCAL_NAME is the (decoded) name to store under on S3. -FILES=( - "HalfBrain.zip|HalfBrain.zip" - "Half%20%20Brain%20simple%20%20files%20.zip|Half Brain simple files.zip" - "NormalLiverFiles.zip|NormalLiverFiles.zip" -) +#FILES=( +# "HalfBrain.zip|HalfBrain.zip" +# "Half%20%20Brain%20simple%20%20files%20.zip|Half Brain simple files.zip" +# "NormalLiverFiles.zip|NormalLiverFiles.zip" +#) # NSCLC lung-cancer samples: each ships a flat-files+cell-labels archive and a # raw-morphology-images archive. The bruker_cosmx_nsclc loader streams both from S3. @@ -132,7 +132,7 @@ for entry in "${FILES[@]}"; do # Upload to S3 echo " Uploading to s3://$bucket/$key ..." - aws s3 cp "$local_path" "s3://$bucket/$key" + aws s3 cp "$local_path" "s3://$bucket/$key" --profile op # Free scratch space before the next (much larger) file echo " Removing local copy to free space" From 5618fb8d5745a2b97444f0d0f7027daccf4ca82d Mon Sep 17 00:00:00 2001 From: dariarom94 Date: Sun, 2 Aug 2026 11:07:28 +0200 Subject: [PATCH 57/60] sync the vizgen files --- .../spatial/upload_vizgen_merscope_2d.sh | 185 ++++++++++++++++++ 1 file changed, 185 insertions(+) create mode 100644 scripts/create_resources/spatial/upload_vizgen_merscope_2d.sh diff --git a/scripts/create_resources/spatial/upload_vizgen_merscope_2d.sh b/scripts/create_resources/spatial/upload_vizgen_merscope_2d.sh new file mode 100644 index 000000000..3fff704fe --- /dev/null +++ b/scripts/create_resources/spatial/upload_vizgen_merscope_2d.sh @@ -0,0 +1,185 @@ +#!/bin/bash + +# Mirror a z3-only (2D) copy of the Vizgen MERSCOPE FFPE showcase raw data from Google +# Cloud Storage to S3, so the Nebius/Seqera benchmark can stage its inputs without any +# Google Cloud credentials. +# +# WHY THIS EXISTS +# process_vizgen_merscope_nebius.sh used to point `input:` directly at +# gs://vz-ffpe-showcase/. That bucket is Vizgen's access-controlled +# data-release bucket (NOT public: an unauthenticated GET returns +# "Anonymous caller does not have storage.objects.get access ..."). The Nebius compute +# env has no GCP credentials, so Nextflow staged the input as an anonymous caller and +# the run died before the loader ran. Mirroring the data to s3://openproblems-data +# (which the Nebius env already reads for every other spatial dataset) removes the +# GCP-auth dependency from every future run. +# +# WHAT IS KEPT (lossless for this 2D pipeline) +# The vizgen_merscope loader calls spatialdata_io.merscope(..., z_layers=3), i.e. it +# only ever reads z-plane 3 of the mosaic images. So we mirror ONLY *_z3.tif and drop +# every *_z{0,1,2,4,5,6}.tif (7 focal planes -> 1). Everything else is copied verbatim: +# - cell_boundaries/*.hdf5 (OLD showcase format; the loader rebuilds +# cell_boundaries.parquet from these at run time) +# - images/mosaic_DAPI_z3.tif (DAPI only, by default — the loader keeps ONLY the DAPI +# channel: `sdata["morphology_mip"].sel(c=["DAPI"])`, so +# dropping PolyT/Cellbound stains is lossless for this +# pipeline and cuts the image bytes to ~1/5. If the +# keep-more-stains TODO in the loader is ever done, re-run +# with KEEP_ONLY_DAPI=0 to mirror all stains at z3.) +# - images/micron_to_mosaic_pixel_transform.csv, images/manifest.json +# - cell_by_gene.csv, cell_metadata.csv, detected_transcripts.csv +# KEEP_ONLY_DAPI defaults to 1 (DAPI-only). Set KEEP_ONLY_DAPI=0 to keep all stains at z3. +# +# PARALLEL +# Each patient ships ~1100-2500 tiny cell_boundaries/*.hdf5 files (~15k total across the 7 +# patients). Streaming them one-at-a-time is dominated by per-file spawn/handshake overhead +# (~15 s/file => ~2-3 DAYS). This script runs the transfers through a pool of $JOBS parallel +# workers (default 16), which hides that overhead and cuts the whole mirror to a few hours. +# Big z3 tifs / detected_transcripts.csv are bandwidth-bound, so a handful running +# concurrently just keeps the uplink saturated. Tune with JOBS= (e.g. 32 for the +# mostly-small-file tail). +# +# REQUIREMENTS (run this on a machine that is authenticated to BOTH clouds) +# - gcloud CLI, authenticated with an account granted access to gs://vz-ffpe-showcase +# (Vizgen data-release-program access — request it via https://info.vizgen.com/ffpe-showcase). +# - aws CLI, with a profile that can write s3://openproblems-data (default: profile "op"). +# The transfer STREAMS each object gcloud->aws (no local disk staging). +# +# IDEMPOTENT + RESUMABLE +# Objects already in S3 with a matching byte size are skipped (one bulk `aws s3 ls` up +# front, so resuming does not cost a HEAD per object). A partially-copied file is aborted +# by aws and reads back as absent, so it is simply re-sent on the next run. Just re-run. +# +# Usage: +# AWS_PROFILE=op ./upload_vizgen_merscope_2d.sh +# # more workers for the small-file tail: +# JOBS=32 AWS_PROFILE=op ./upload_vizgen_merscope_2d.sh +# # or override anything: +# GCS_BASE=gs://vz-ffpe-showcase \ +# S3_DEST=s3://openproblems-data/resources/raw_data/txSim_custom/vizgen_merscope \ +# SAMPLES="HumanBreastCancerPatient1 HumanLungCancerPatient1" \ +# JOBS=24 AWS_PROFILE=op ./upload_vizgen_merscope_2d.sh + +set -euo pipefail + +GCS_BASE="${GCS_BASE:-gs://vz-ffpe-showcase}" +S3_DEST="${S3_DEST:-s3://openproblems-data/resources/raw_data/txSim_custom/vizgen_merscope}" +AWS_PROFILE="${AWS_PROFILE:-op}" +Z="${Z:-3}" # which single z-plane to keep +KEEP_ONLY_DAPI="${KEEP_ONLY_DAPI:-1}" # 1 = DAPI-only (default, lossless for this loader); 0 = all stains at z3 +JOBS="${JOBS:-12}" # number of concurrent gcloud->aws transfers (each = 2 TLS + # conns; >~16 tends to trigger transient TLS/read-timeout + # errors, which the worker retries with backoff anyway) + +# The 7 patients currently active (uncommented) in process_vizgen_merscope_nebius.sh. +# Additional patients (melanoma/ovarian/prostate/uterine) are commented-out there and in +# the s3 sibling process_vizgen_merscope.sh — add their folder names here to mirror them. +SAMPLES=(${SAMPLES:-\ + HumanBreastCancerPatient1 \ + HumanLiverCancerPatient1 \ + HumanLiverCancerPatient2 \ + HumanLungCancerPatient1 \ + HumanLungCancerPatient2 \ + HumanColonCancerPatient1 \ + HumanColonCancerPatient2}) + +export AWS_PROFILE +command -v gcloud >/dev/null || { echo "ERROR: gcloud CLI not found (needed to read gs://vz-ffpe-showcase)" >&2; exit 1; } +command -v aws >/dev/null || { echo "ERROR: aws CLI not found (needed to write $S3_DEST)" >&2; exit 1; } + +gcs_bucket="$(echo "$GCS_BASE" | sed -E 's#^gs://([^/]+).*#\1#')" +s3_bucket="$(echo "$S3_DEST" | sed -E 's#^s3://([^/]+)/.*#\1#')" +s3_prefix="$(echo "$S3_DEST" | sed -E 's#^s3://[^/]+/(.*)#\1#')" +export gcs_bucket s3_bucket s3_prefix + +# Should this object be dropped from the mirror? +drop_object() { + local rel="$1" base="${1##*/}" + # keep only z-plane $Z: drop any *_z.tif (matches mosaic_*_z*.tif and boundaries_z*.tif) + if [[ "$base" =~ _z([0-9]+)\.tif$ && "${BASH_REMATCH[1]}" != "$Z" ]]; then return 0; fi + # optional: drop non-DAPI stains entirely + if [[ "$KEEP_ONLY_DAPI" == "1" && "$base" =~ ^mosaic_ && ! "$base" =~ ^mosaic_DAPI_ ]]; then return 0; fi + return 1 +} + +# Worker: stream one "|" record gcloud->aws. Exported so `xargs -P` can run it in +# parallel `bash -c` subshells. Kept free of bash-4 features so it works on the stock macOS +# /bin/bash 3.2. The already-uploaded files are filtered out up front (see below), so the +# worker just transfers. A failed transfer is logged, not fatal: the batch keeps going and the +# file (absent, since aws aborts a partial multipart) is retried on the next resumable run. +transfer_one() { + local rec="$1" + local size="${rec%%|*}" + local rel="${rec#*|}" + local attempt=0 max=4 + # Retry transient GCS/S3 errors (read timeouts, SSL/TLS handshake failures) that show up under + # high concurrency. pipefail (set by the caller) makes a gcloud-side failure fail the pipe too. + while :; do + attempt=$((attempt+1)) + if gcloud storage cat "gs://$gcs_bucket/$rel" 2>/dev/null \ + | aws s3 cp - "s3://$s3_bucket/$s3_prefix/$rel" --expected-size "$size" --only-show-errors 2>/dev/null; then + printf ' OK %s (%s B)\n' "$rel" "$size" + return 0 + fi + if [ "$attempt" -ge "$max" ]; then + printf ' FAIL %s (after %s attempts; retry on re-run)\n' "$rel" "$attempt" >&2 + return 0 + fi + sleep $((attempt * 3)) # linear backoff: 3s, 6s, 9s + done +} +export -f transfer_one + +present="$(mktemp -t viz_present)" +wanted_raw="$(mktemp -t viz_wanted_raw)" +wanted="$(mktemp -t viz_wanted)" +worklist="$(mktemp -t viz_worklist)" +trap 'rm -f "$present" "$wanted_raw" "$wanted" "$worklist"' EXIT + +# 1) One bulk listing of what is already in S3 -> "\t" (sorted). One API call for the +# whole resume state, instead of a HEAD per object. +echo "Listing existing objects under $S3_DEST ..." +aws s3 ls "s3://$s3_bucket/$s3_prefix/" --recursive 2>/dev/null \ + | awk -v p="$s3_prefix/" 'BEGIN{OFS="\t"} {k=$4; sub("^"p,"",k); if(k!="") print k,$3}' \ + | sort > "$present" +echo " $(wc -l < "$present" | tr -d ' ') objects already in S3." + +# 2) Enumerate GCS, apply the keep-rules -> "\t". Written to a file (not piped) so +# the counters stay in this shell and the progress echoes go to stderr, not into the data. +total=0; dropped=0 +for sample in "${SAMPLES[@]}"; do + echo "Enumerating $GCS_BASE/$sample ..." >&2 + while IFS= read -r line; do + [[ "$line" == *"gs://"* ]] || continue # skip the TOTAL: summary line + url="$(awk '{print $NF}' <<<"$line")" + size="$(awk '{print $1}' <<<"$line")" + [[ "$url" == */ ]] && continue # skip directory placeholders + [[ "$size" =~ ^[0-9]+$ ]] || continue # skip anything without a numeric size + rel="${url#gs://$gcs_bucket/}" + total=$((total+1)) + if drop_object "$rel"; then dropped=$((dropped+1)); continue; fi + printf '%s\t%s\n' "$rel" "$size" >> "$wanted_raw" + done < <(gcloud storage ls -l "$GCS_BASE/$sample/**") +done +sort "$wanted_raw" > "$wanted" + +# 3) Work list = wanted objects not already in S3 at the same size (comm on the sorted files). +comm -23 "$wanted" "$present" | awk -F'\t' '{print $2"|"$1}' > "$worklist" +queued="$(wc -l < "$worklist" | tr -d ' ')" +echo "================================================================" +echo "Enumerated $total objects: $dropped dropped, $(( $(wc -l < "$wanted" | tr -d ' ') - queued )) already in S3, $queued to transfer." +echo "Transferring with $JOBS parallel workers ..." + +# 4) Run the transfers in parallel. -I REC feeds one record per worker; -P runs $JOBS at once. +# Use the SAME bash that is running this script ($BASH) so the exported transfer_one +# imports correctly regardless of which bash is first on PATH. +if [[ "$queued" -gt 0 ]]; then + xargs -P "$JOBS" -I REC "${BASH:-bash}" -c 'set -o pipefail; transfer_one "$@"' _ REC < "$worklist" +fi + +echo "================================================================" +echo "Done. objects=$total dropped=$dropped queued=$queued (JOBS=$JOBS)" +echo "Loader --input paths (use these in process_vizgen_merscope_nebius.sh):" +for sample in "${SAMPLES[@]}"; do + echo " $S3_DEST/$sample" +done From b68c100ff48978a176251669b7798a3d45cc74ad Mon Sep 17 00:00:00 2001 From: dariarom94 Date: Sun, 2 Aug 2026 11:08:43 +0200 Subject: [PATCH 58/60] adjust mem for allen brain --- src/base/labels_nebius.config | 1 + .../loaders/allen_brain_cell_atlas_merfish/config.vsh.yaml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/base/labels_nebius.config b/src/base/labels_nebius.config index 93ab8d178..5dc68c20e 100644 --- a/src/base/labels_nebius.config +++ b/src/base/labels_nebius.config @@ -193,6 +193,7 @@ process { withLabel: hightime { time = 12.h } withLabel: veryhightime { time = 24.h } +withLabel: veryveryhightime { time = 48.h } // 2 days (e.g. allen_brain_cell_atlas_merfish loader) // make sure publishstates gets enough disk space and memory withName:'.*publishStatesProc' { diff --git a/src/datasets/loaders/allen_brain_cell_atlas_merfish/config.vsh.yaml b/src/datasets/loaders/allen_brain_cell_atlas_merfish/config.vsh.yaml index 700616235..0fe590ecf 100644 --- a/src/datasets/loaders/allen_brain_cell_atlas_merfish/config.vsh.yaml +++ b/src/datasets/loaders/allen_brain_cell_atlas_merfish/config.vsh.yaml @@ -91,4 +91,4 @@ runners: - type: executable - type: nextflow directives: - label: [highmem, midcpu, hightime] + label: [veryhighmem, midcpu, veryveryhightime] From 98dbc69033d9a8e7258fa91406e77c797198e2aa Mon Sep 17 00:00:00 2001 From: dariarom94 Date: Sun, 2 Aug 2026 11:09:08 +0200 Subject: [PATCH 59/60] claude notes --- src/datasets/loaders/bruker_cosmx/NOTES.md | 255 +++++++++++++++++++++ 1 file changed, 255 insertions(+) create mode 100644 src/datasets/loaders/bruker_cosmx/NOTES.md diff --git a/src/datasets/loaders/bruker_cosmx/NOTES.md b/src/datasets/loaders/bruker_cosmx/NOTES.md new file mode 100644 index 000000000..e970957dc --- /dev/null +++ b/src/datasets/loaders/bruker_cosmx/NOTES.md @@ -0,0 +1,255 @@ +# bruker_cosmx loader — notes + +## What this component is + +Dataset loader (`datasets/loaders` namespace) that converts a **Bruker / NanoString +CosMx** export into the standardized `raw_ist` SpatialData zarr consumed by the rest of the +pipeline (API: `/src/api/file_common_ist.yaml`, merged as `--output` in `config.vsh.yaml`). + +It is a thin wrapper around **`sopa.io.cosmx`** (the sopa dependency is unpinned in +`config.vsh.yaml` → currently sopa ~2.2.x). sopa does the heavy lifting: it stitches the +per-FOV morphology images and cell-label tifs into one global image/label, reads the +transcripts flat file, and builds the cells table. Our script's job is (1) massage the +on-disk directory layout into the shape sopa expects, (2) rename sopa's element keys to our +API names, (3) drop non-DAPI image channels, (4) attach dataset metadata, (5) write. + +Some CosMx exports ship **no AtoMx flat files at all** (the liver "Raw Data Files" release — +see Version 4 below). For those the script first **reconstructs** the flat files sopa needs +from the raw per-FOV decoding CSVs, then feeds the same `sopa.io.cosmx` call. This is +detected automatically (no config knob): if `input_flat_files` is unset AND the raw zip has +no `*_tx_file.csv`, reconstruction mode is on. + +Sibling loader `bruker_cosmx_nsclc` handles the CosMx lung-cancer layout (per-sample +subdirs, per-z-plane images) — that's a *different* export structure (see the docstring +"Version 3"), not covered here. + +Datasets driven through this loader (see `scripts/create_resources/spatial/process_bruker_cosmx_nebius.sh`): +- `bruker_cosmx/bruker_mouse_brain_cosmx/rep1` — full mouse-brain hemisphere (`HalfBrain.zip` + + separate `Half Brain simple files.zip`). **The big one** — the source of the OOM this + NOTES documents. +- `bruker_cosmx/bruker_human_liver_cosmx` — human liver (`NormalLiverFiles.zip`). **No flat + files exist for this dataset** (NanoString only released raw images + TileDB + Seurat), so + it runs through the **reconstruction** path (Version 4 / see section below). `input_flat_files` + is left unset in the run script. + +## Supported input layouts + +The script's module docstring (`script.py:3-80`) is the authoritative catalogue of the three +CosMx export shapes seen in the wild. The two this loader handles: + +- **Flat files bundled in the raw zip** (liver): `input_flat_files` is left unset. +- **Flat files in a separate zip** (mouse brain): `input_flat_files` is set, and the CSVs are + symlinked into `CellStatsDir` (`script.py:170-184`). + +In both, sopa wants a `CellStatsDir/CellLabels/` folder, but many exports only ship the +per-FOV `CellLabels_F###.tif` inside `FOV*/` folders. `script.py:186-200` gathers those into +a `CellLabels/` folder via symlinks (see sopa issue #285). + +## script.py — step by step + +1. **Extract** `input_raw` (strip the single wrapper dir) → find `CellStatsDir` + (`script.py:155-168`). +2. **Extract flat files** if provided, symlink the `*.csv` into `CellStatsDir` + (`script.py:170-184`). +3. **Assemble `CellLabels/`** from per-FOV tifs if needed (`script.py:186-200`). +4. **Lean transcript-read shim** installed here (`script.py:202-253`) — see next section. +5. **`sopa.io.cosmx(...)`** with `cells_labels/table/polygons=True`, `read_proteins=False` + (`script.py:261`). Stitched image/labels come back as **lazy dask** — they are *not* the + memory peak; they're written chunked. +6. **Rename element keys** to API names (`script.py:277-288`): + `stitched_image→morphology_mip`, `stitched_labels→cell_labels`, `points→transcripts`, + `cells_polygons→cell_boundaries`. +7. **Add API transcript columns** (`script.py:290-292`): `cell_id = global_cell_id`, + `feature_name = target` (dask-lazy copies; cheap now that `target` is categorical). +8. **Keep only the DNA channel** of the morphology image (`script.py:300`, + `.sel(c=["DNA"])`) — we treat the "DNA" stain as DAPI. TODO in code: keep PolyT / + Cellbound channels one day (currently saving/plotting fails when they're kept). +9. **Attach dataset metadata** to `table.uns` (`script.py:303-307`). +10. **`sdata.write(par["output"])`** (`script.py:315`). + +## Flat-file reconstruction for raw-only exports (Version 4, liver — 2026-07-29) + +`NormalLiverFiles.zip` contains only imaging + segmentation + raw decoding output; the AtoMx +"flat files" were never released for the liver dataset (verified: no `*_tx_file.csv` / +`*_fov_positions_file.csv` anywhere in the zip, and no companion "simple files" zip on S3 — +only TileDB/Seurat). `sopa.io.cosmx` therefore died at `_infer_dataset_id` +(`ValueError: Could not infer dataset_id ...`). The raw material to rebuild the flat files is +present, so the loader now reconstructs them. + +**Detection** (`script.py`, `RECONSTRUCT`): on when `input_flat_files` is unset *and* the raw +zip has no `*_tx_file.csv` (peeked cheaply from the zip central directory via +`_zip_has_flat_files`). Off for mouse brain (separate flat-files zip) and any export that +ships flat files inside the raw zip. + +**Extraction** switches from the denylist to an *allowlist* (`_member_wanted_reconstruct` / +`_RECON_KEEP`) that keeps only what reconstruction + sopa need: per-FOV `CellLabels_F*.tif`, +`*Cell_Stats_F*.csv`, `Morphology2D/*.TIF`, `AnalysisResults/**/*complete_code_cell_target_call_coord.csv`, +and `RunSummary/*.fovs.csv`. Footprint for the liver hemisphere ≈ **61 GB** (vs 440 GB of +Morphology3D alone). Crucially this keeps `AnalysisResults`/`RunSummary`, which the flat-file +denylist drops. + +**`reconstruct_flat_files()`** writes four flat files into `CellStatsDir` (`DATA_DIR`): +- `{id}_fov_positions_file.csv` ← `RunSummary/latest.fovs.csv` (headerless; `x_mm`=col 1, + `y_mm`=col 2, `fov`=col 6 — matches the Metrics4D `Xpos_um`/`Ypos_um`). +- `{id}_tx_file.csv` ← concat of the 304 `complete_code_cell_target_call_coord.csv` + (`fov, cell_ID=CellId, target, x_global_px, y_global_px, z`), written FOV-by-FOV so only + one FOV is in RAM (≈229 M transcripts total for liver). +- `{id}_metadata_file.csv` ← `Cell_Stats_F*.csv` (`fov, cell_ID, CenterX/Y_global_px, Area`). +- `{id}_exprMat_file.csv` ← per-cell gene counts crosstab'd from the assigned transcripts, + reindexed to the Cell_Stats cell list so metadata and counts share the exact same cells/order. +- No `-polygons.csv`: the raw export has no cell boundaries → `cells_polygons=False`, + `cell_boundaries` absent (optional in the API; downstream methods segment themselves). + +**Coordinate formula** (the load-bearing part — validated end-to-end at **0.998** of assigned +transcripts landing on their own cell in sopa's stitched labels, on a 3-FOV subset run through +the real `sopa.io.cosmx`): +``` +scale = 1e3 / COSMX_PIXEL_SIZE # 0.120280945 µm/px; == sopa's own mm→px factor +x_global_px = x_mm * scale + x_local +y_global_px = y_mm * scale + (H - 1 - y_local) # H=4256; sopa flips each label tile along y +``` +The transcript local `(x, y)` map DIRECTLY to `CellLabels[row=y, col=x]` (top-origin, no flip; +measured 0.993 raw), but sopa's stitcher does `da.flip(tile, axis=y)`, so the `(H-1-y)` term +undoes that flip. The same transform is applied to the Cell_Stats centroids so the table's +`obsm['spatial']` stays consistent with transcripts + labels. `fov_shift` is left at sopa's +inferred default (False — no polygons to infer from). + +**Global-cell-id assert workaround:** sopa derives `max(cell_ID)` separately from each of tx / +metadata / exprMat and asserts they agree; a segmented cell with zero transcripts makes the +tx-file max smaller and trips the assert. In reconstruction mode we monkeypatch +`_CosMXReader._get_global_cell_id` to use one fixed max (the segmentation's true max from +Cell_Stats), keeping ids consistent across all three files. + +**Not yet run end-to-end on the full 304-FOV liver dataset on Nebius** — validated on a 3-FOV +subset. Watch memory (≈229 M-row tx read, mitigated by the lean-read shim) and wall-time +(reconstruction + stitch + zarr write) against the loader's `hightime` (8 h) cap; bump to +`veryhightime` if it times out. + +## The real OOM: scratch exhaustion during extraction (the actual fix, 2026-07-27) + +**Symptom:** `bruker_mouse_brain_cosmx` (full hemisphere) is OOM-killed (**exit 137**) on +every retry at the *very first* step — the `log("Extract zip of raw files")` line, 58 µs in — +**before any sopa / transcript / image code runs**. + +**Root cause (measured — NOT the transcript read):** the raw `HalfBrain.zip` is **208 GB +uncompressed** (3990 files), but sopa only ever reads **Morphology2D (13.6 GB) + +`FOV*/CellLabels` (0.12 GB)**. ~190 GB of the zip is `Morphology3D` (per-z-plane 3D stacks — +sopa uses the 2D composite), `AnalysisResults` (decoding CSVs), and `RunSummary` — none of +which sopa touches. The loader extracted all of it. And because `--input_raw` was a Viash +`file`, Nextflow first **staged the 187.7 GB zip** into the task's scratch, then the script +extracted ~208 GB *alongside* it. The `veryhighmem` node has only ~220 GB ephemeral disk and a +**RAM-backed scratch** (which is why a *disk* operation OOMs as *memory*): staged zip (187) + +extraction blow the 400→480 GB memory limit → 137. Retries can't help — `disk` is pinned at +200 GB in `labels_nebius.config` and memory only reaches the 480 GB cap. + +**Fix (2026-07-27):** +1. `--input_raw` is now `type: string` in **both** the loader and the `process_bruker_cosmx` + workflow config, so Nextflow no longer stages the 187 GB zip (verified: `main.nf` passes it + through `fromState`). +2. `extract_zip` (`script.py`) streams the zip **in place** — local path via `open()`, + `s3://` via **s3fs** (`_open_zip_source`, 32 MB blocks) — and extracts only members **not** + under `SKIP_DIRS = {Morphology3D, AnalysisResults, RunSummary}` (`_member_wanted`). + Footprint drops **208 GB → 14.6 GB** (916 of 3990 files); S3 transfer drops ~14×. Verified + end-to-end against the real S3 zip (dry-run filter count + streamed a Morphology2D and a + CellLabels tif → valid TIFFs, `strip_root` correct). New dep **s3fs** in `config.vsh.yaml` + pypi ⇒ **the container must be rebuilt** before a run picks this up. + +Denylist (not allowlist) on purpose: it drops only the three known-huge unused dirs and keeps +every small file, so it can't accidentally drop the flat CSVs the **liver** dataset ships +*inside* its raw zip. + +## Secondary optimization: lean transcript read (NOT the OOM — kept anyway) + +The *earlier* hypothesis (recorded here before 2026-07-27) was that the OOM was sopa's +`_CosMXReader.read_transcripts` doing one `pd.read_csv` of the whole `*_tx_file.csv`. That read +is genuinely wasteful (the mouse-brain tx file is 7 GB / ~100M rows, object-dtype strings) and +the lean shim below is worth keeping — but it was **never the OOM**: the job died at +extraction, long before transcripts are read, and the panel is only ~960 genes (so sopa's +table densification is a non-issue too). The shim is only now *reached* for the first time, +once extraction succeeds. + +**Lean transcript read (`script.py`):** we replace `sopa.io.reader.cosmx.pd` with a thin proxy +(`_PandasReadCsvProxy`) that forwards every attribute to real pandas but intercepts +`read_csv`. For the transcripts file only (name contains `_tx_file.csv`), it: +- reads `target` as **categorical** (object → int codes; the single biggest saver), and +- restricts `usecols` to the columns sopa's stitched-FOV path + `_get_global_cell_id` + the + downstream schema actually use: `fov, cell_ID, target, x_global_px, y_global_px, z` + (dropping `cell`, `CellComp`, `x_local_px`, `y_local_px`). + +It pre-reads the header (`nrows=0`) to intersect the keep-list with the real columns and only +leans the read if sopa's required columns are present — otherwise it falls through to a normal +read, so it degrades gracefully on an unexpected export. Every other sopa read (fov positions, +metadata, counts, polygons) is left untouched because their filenames don't match +`_tx_file.csv`. **All of sopa's coordinate/stitching math runs unchanged** on the leaner frame. + +Why the proxy (not patching `pandas.read_csv` globally, and not a private-API reimplement): +sopa is unpinned, so we avoid touching its internals; the proxy only rebinds the module-level +`pd` name *inside sopa's cosmx module*, so no global pandas mutation and no dependence on +sopa's private `read_transcripts` signature. + +Measured ~5.6× smaller (82%) transcript frame on a synthetic CosMx-shaped file; larger on the +real data. Safe to drop the vendor-specific columns because the pipeline is vendor-agnostic — +downstream methods consume the standardized API columns (`x, y, feature_name, cell_id, z`); +nothing depends on CosMx-only `cell`/`CellComp` (verified: the `df["cell"]` refs in +`methods_transcript_assignment/baysor/script_no_sopa.py` are Baysor's *own* output, not our +input). + +## Gotcha: obs `cell_id` vs vendor `cell_ID` (spatialdata write, 2026-07-30) + +`sdata.write()` failed for the mouse brain with `ValidationError: SpatialData contains elements +with invalid names`. spatialdata's writer forbids two keys in a table attribute that differ only +in **case**. sopa's table already carries the CosMx per-FOV-local **`cell_ID`** obs column; the +loader then adds **`cell_id`** (the global unique index — required by `file_common_ist.yaml`'s +`obs.cell_id`). `cell_ID` vs `cell_id` collide → invalid. + +Fix (`script.py`, "Add info to metadata table"): rename the vendor local id to `cell_ID_local` +*before* adding `cell_id`. Note this is a **case-insensitive-key** rule, not a bad-character one +— the message's "invalid names" is a catch-all. (Aside: the mouse panel also has `/` gene names +like `Tuba1a/b/c`; those live in the var **index**, which spatialdata does **not** validate — so +they are harmless for `write` and are left untouched. Only obs/var *column* names, obsm/uns +*keys*, and element names are checked.) + +## Arguments + +| Argument | Required | Notes | +|----------|----------|-------| +| `--input_raw` | yes | **`type: string`** (not a staged file) — URL/path to the raw zip. Streamed in place (s3:// via s3fs) and selectively extracted; see the OOM section. | +| `--input_flat_files` | no | Second zip with the `*_.csv` flat files, when not in the raw zip (mouse brain needs it) | +| `--segmentation_id` | default `["cell"]` | Must be exactly `["cell"]` — asserted at `script.py`; CosMx ships only the cell segmentation | +| `--dataset_*` metadata | mixed | Written into `table.uns` | + +## Setup / Docker + +`config.vsh.yaml`: `openproblems/base_python:1` + `__merge__` of +`/src/base/setup_spatialdata_partial.yaml` + `pypi: [sopa, s3fs]`. **sopa is unpinned** — a +load-bearing risk given the transcript shim reaches into `sopa.io.reader.cosmx` (see risk +points). **s3fs** was added 2026-07-27 for streaming the raw zip from `s3://` — adding it means +the container must be rebuilt/pushed before a run sees the extraction fix. + +## Wiring + +- Registered as a Nextflow module dep of the per-dataset workflow + `src/datasets/workflows/process_bruker_cosmx/` (`main.nf` runs loader → optional + `crop_region` → `setState`). +- Resource label: **`[veryhighmem, midcpu, hightime]`** in `config.vsh.yaml`. +- Run script: `scripts/create_resources/spatial/process_bruker_cosmx_nebius.sh` (launches via + Seqera `tw launch` against `build/main`). +- The workflow's `crop_region` step (gated on `crop_region_min_x`) is the escape hatch to + tile/crop the section if it still won't fit — the mouse-brain params don't set it today. + +## Risk points / gotchas + +- **sopa is unpinned** and the transcript fix depends on `sopa.io.reader.cosmx` existing and + reading the tx file via the module-level `pd.read_csv`. If a future sopa refactors the + reader (renames the module, switches to `dask`/`pyarrow`, changes the tx filename match), + the shim silently stops leaning (falls through to a normal read → OOM returns). Consider + pinning sopa if this bites. +- **Secondary spike, not yet fixed:** sopa's `read_tables` densifies the cell×gene matrix via + `csr_matrix(counts.values)` (`io/reader/cosmx.py`). Negligible for a ~1k panel; large if the + mouse-brain export is **WTx (~19k genes)**. Can't fix without reimplementing sopa's table + read — next lever if OOM persists after the transcript fix. +- **Script-only change:** no image rebuild needed, but it must reach `origin/main` and be + regenerated into `build/main` before a Nebius run picks it up (use the `check-component` + skill to confirm deploy-freshness). +- **Not yet confirmed end-to-end:** the fix passes local syntax + logic tests; it has **not** + yet been validated on a real mouse-brain run on Nebius. From fa23294462f43d01114cca1af001e2c2bf2422fa Mon Sep 17 00:00:00 2001 From: dariarom94 Date: Sun, 2 Aug 2026 11:09:47 +0200 Subject: [PATCH 60/60] claude notes --- .../moscot/NOTES.md | 155 +++++++++ .../rctd/NOTES.md | 142 ++++++++ .../singler/NOTES.md | 147 ++++++++ .../ssam/NOTES.md | 108 ++++++ .../tangram/NOTES.md | 190 +++++++++++ .../resolvi_correction/NOTES.md | 126 +++++++ .../split/NOTES.md | 146 ++++++++ src/methods_segmentation/binning/NOTES.md | 201 +++++++++++ src/methods_segmentation/cellpose/NOTES.md | 316 +++++++++++++++++ src/methods_segmentation/cellposev4/NOTES.md | 291 ++++++++++++++++ src/methods_segmentation/stardist/NOTES.md | 228 +++++++++++++ src/methods_segmentation/watershed/NOTES.md | 238 +++++++++++++ .../fastreseg/NOTES.md | 139 ++++++++ .../segger/NOTES.md | 323 ++++++++++++++++++ 14 files changed, 2750 insertions(+) create mode 100644 src/methods_cell_type_annotation/moscot/NOTES.md create mode 100644 src/methods_cell_type_annotation/rctd/NOTES.md create mode 100644 src/methods_cell_type_annotation/singler/NOTES.md create mode 100644 src/methods_cell_type_annotation/ssam/NOTES.md create mode 100644 src/methods_cell_type_annotation/tangram/NOTES.md create mode 100644 src/methods_expression_correction/resolvi_correction/NOTES.md create mode 100644 src/methods_expression_correction/split/NOTES.md create mode 100644 src/methods_segmentation/binning/NOTES.md create mode 100644 src/methods_segmentation/cellpose/NOTES.md create mode 100644 src/methods_segmentation/cellposev4/NOTES.md create mode 100644 src/methods_segmentation/stardist/NOTES.md create mode 100644 src/methods_segmentation/watershed/NOTES.md create mode 100644 src/methods_transcript_assignment/fastreseg/NOTES.md create mode 100644 src/methods_transcript_assignment/segger/NOTES.md diff --git a/src/methods_cell_type_annotation/moscot/NOTES.md b/src/methods_cell_type_annotation/moscot/NOTES.md new file mode 100644 index 000000000..5b3c92caa --- /dev/null +++ b/src/methods_cell_type_annotation/moscot/NOTES.md @@ -0,0 +1,155 @@ +# moscot — cell-type annotation (MOSCOT / optimal transport) + +## What this component is + +Cell-type-annotation stage method (API `src/api/comp_method_cell_type_annotation.yaml`, +subtype `method_cell_type_annotation`). It labels each spatial cell by **optimal-transport +mapping** of an scRNA-seq reference onto the spatial cells and then transferring the +reference's cell-type labels through the learned transport plan. + +- Docs: https://moscot.readthedocs.io — Repo: https://github.com/theislab/moscot +- Paper (config `references.doi`, **verified correct**): Klein, Palla, Lange, Klein et al., + "Mapping cells through time and space with moscot", *Nature* (2025), + DOI `10.1038/s41586-024-08453-2`. This is the moscot method paper — **no citation caveat**. +- **GPU-only** in practice: the engine installs `jax[cuda12]` + `moscot` + `flax` + `diffrax` + on `openproblems/base_pytorch_nvidia:1.1.0`, and the Nextflow directives carry `gpuh100`. + +Under the hood it uses `moscot.problems.space.MappingProblem`, a **fused Gromov-Wasserstein +(FGW)** problem solved with OTT-jax: a quadratic (Gromov-Wasserstein) term matches +intra-domain structure (the SC feature space vs the spatial coordinates) and a linear term +matches shared-gene expression across the two domains; `alpha` interpolates between them. + +## script.py — step by step + +1. **`:13-15` — pop `LD_LIBRARY_PATH` before importing jax.** JAX's `jax[cuda12]` wheel ships + its own CUDA/cuDNN; a system `LD_LIBRARY_PATH` pointing at an older cuDNN otherwise wins and + crashes with "Loaded runtime CuDNN library: 9.1.0 but source was compiled with: 9.8.0". + **Load-bearing** — do not remove. +2. `:18-25` — jax GPU sanity prints (version, backend, devices). +3. `:58-59` — read `input_scrnaseq_reference` and `input_spatial_normalized_counts` (h5ad). +4. **`:62-66` — SMALL-DATA GUARD (the key gotcha).** If `adata_sp.n_obs < 10000` it + **overrides** `par['rank'] = -1` (full rank) and `par['tau'] = 1.0` (balanced OT), + regardless of what was passed in. See "Risk points". +5. `:69-71` — assert a `"normalized"` layer exists in both AnnDatas and `centroid_x`/`centroid_y` + exist in the spatial `obs`. +6. `:74-76` — set `X = layers["normalized"]` for both; build `adata_sp.obsm["spatial"]` from the + centroid columns. +7. `:78` — `sc.pp.pca(adata_sc, n_comps=30)` — the SC quadratic cost is built from a **hardcoded + 30-dim PCA**, not raw genes. +8. `:81-86` — `MappingProblem(adata_sc, adata_sp).prepare(sc_attr={obsm, X_pca}, + xy_callback="local-pca")`: SC intra-domain cost = the 30-dim PCA; spatial intra-domain cost = + a local-PCA callback over the spatial coordinates; the linear (fused) term links shared genes. +9. `:92-99` — `mp.solve(alpha=, epsilon=, tau_a=tau, tau_b=tau, rank=, batch_size=)`. +10. `:102-108` — `mp.annotation_mapping(mapping_mode=, annotation_label=celltype_key, + source="src", forward=False)`; the returned per-cell label is written to + `adata_sp.obs[celltype_key]`. (`forward=False`/`source="src"` are hardcoded.) +11. `:111` — write the annotated spatial AnnData. + +## Arguments + +| Arg | Type | Config default | moscot tool default | Effective on the <10k test data | Maps to | +|-----|------|----------------|---------------------|---------------------------------|---------| +| `--alpha` | double | **0.8** | **0.5** | used as-is | `solve(alpha=)` — FGW quadratic↔linear weight | +| `--epsilon` | double | 0.01 | 0.01 | used as-is | `solve(epsilon=)` — entropic regularization | +| `--tau` | double | **0.3** | **1.0** | **forced to 1.0** (`:65-66`) | `solve(tau_a=tau_b=)` — marginal relaxation / unbalancedness | +| `--rank` | integer | **500** | **-1** | **forced to -1** (`:63-64`) | `solve(rank=)` — low-rank OT (−1 = full rank) | +| `--batch_size` | integer | 1024 | `None` | used (but n_obs10k-cell) dataset**: for a real-data sweep, `tau ∈ [0.1, 0.2, 0.3, 1.0]` (the config + `# TODO` notes it "seems only to work with tau=1 on our data") and `rank ∈ [500, 1000, 2000, + 5000, -1]` (the `# TODO` scales rank with cell count, ~5000 for 300k cells). + +**Tier 2 — label read-out (swept):** +- **`mapping_mode`** *(exposed; config `max`, moscot has no tool default — required arg).* + `max` = label of the single highest-mass source cell; `sum` = aggregate plan mass per cell type + then argmax (uses the full coupling, smoother). One meaningful non-default value: `[sum]`. + +**Tier 3 — high-value knobs NOT exposed today (future work; would need config expose + script +wiring + a container rebuild, so NOT in the submittable sweep):** +- **`n_comps`** — the SC PCA dimensionality is hardcoded at 30 (`:78`); it sets the resolution of + the quadratic cost. Highest-value un-exposed knob. +- **`scale_cost`** *(tool default "mean")* — hardcoded (unset ⇒ tool default); how the cost + matrices are normalized before the solve, affects the effective `epsilon`. +- **`initializer`** — for low-rank solves (`rank>0`) the initializer (e.g. `"rank2"`/`"k-means"`) + materially changes convergence; only relevant once `rank` is swept on a large dataset. +- **`scale_by_marginals`** *(annotation_mapping, tool default True)* and **`forward`/`source`** + (hardcoded `forward=False`, `source="src"`) — the label-projection direction/weighting. +- **`threshold` / `max_iterations`** *(convergence: tool `threshold=1e-3`)* — Sinkhorn/LR-GW + convergence controls (quality↔runtime). + +To sweep any Tier-3 knob: add it to `config.vsh.yaml` `arguments:` (default = the moscot default), +thread it into the `solve`/`prepare`/`annotation_mapping` call in `script.py`, then +`viash ns build` + rebuild the container (see `check-component`). A sweep launched against a stale +`build/main` container silently ignores a freshly-added arg — which is why these stay out of the +submittable-now sweep. + +## Risk points / gotchas + +- **The `<10k` small-data guard silently overrides `tau` and `rank`** (`:62-66`). On the 306-cell + test resource this means those two args do nothing — any sweep over them is a no-op. This is + *intended* behavior (unbalanced/low-rank OT is only for large references), not a bug. +- **GPU-only + not installable/verifiable locally.** moscot/jax[cuda12] is not in the local env; + all tool defaults and signatures in this NOTES were read from moscot's upstream source and docs, + not executed here. The sweep has **not** been run end-to-end yet. +- **A hard argmax label is emitted** — the transport plan / per-cell confidence is not exported; + downstream metrics see labels only. +- **`batch_size` is a pure memory knob** (online GW-cost batching to bound GPU memory on large + references); it does not change the output and is held fixed (`n_obs=306 < 1024` ⇒ no batching + on the test data anyway). diff --git a/src/methods_cell_type_annotation/rctd/NOTES.md b/src/methods_cell_type_annotation/rctd/NOTES.md new file mode 100644 index 000000000..568cbb843 --- /dev/null +++ b/src/methods_cell_type_annotation/rctd/NOTES.md @@ -0,0 +1,142 @@ +# RCTD — cell type annotation (NOTES) + +Authoritative how-it-works / why-the-setup / where-it-breaks reference for the RCTD +component. The running iteration log lives in the memory file +`rctd-split-zero-umi-reference-nan.md` (repo auto-memory) — that pointer + log, this +file the depth. + +## What this component is + +- **Stage / API:** cell type annotation (`src/api/comp_method_cell_type_annotation.yaml`). + Inputs `input_spatial_normalized_counts` (the aggregated cell x gene `.h5ad` with + centroids) and `input_scrnaseq_reference` (labelled scRNA-seq atlas); outputs the same + spatial object with a `cell_type` column in `.obs`/`colData`. +- **Unusual:** this is an **R** component (`script.R`, `openproblems/base_r:1`) that wraps + **spacexr::RCTD** (Robust Cell Type Decomposition). It installs `spacexr` from GitHub + (`dmcable/spacexr`) at build time — an un-pinned `install_github`, so the exact spacexr + version floats with the build date (last observed live: 2.2.1). +- **Links:** docs/repo `https://github.com/dmcable/spacexr`. +- **Publication:** Cable et al., *Robust decomposition of cell type mixtures in spatial + transcriptomics*, Nat Biotechnol 2022 (DOI `10.1038/s41587-021-00830-w`, PMID 33603203). + **The config DOI is correct** (it is the RCTD method paper, not a generic framework + paper) — no citation caveat. The paper's headline contribution is the `doublet_mode` + decomposition (each pixel assigned at most two cell types), and platform-effect / DE-gene + selection is the machinery the exposed thresholds control. + +## script.R — step by step + +1. **L27** read the spatial `.h5ad` as a `SingleCellExperiment`. +2. **L30-35** build a `SpatialRNA` "puck" from `centroid_x/centroid_y` and the raw + `counts` assay (RCTD works on **raw integer counts**, not the normalized layer — despite + the input being named `*_normalized_counts`, the raw `counts` assay is what is read). +3. **L38** read the scRNA-seq reference as a `SingleCellExperiment`. +4. **L45-49 — zero-UMI reference fix (load-bearing).** After `process_dataset` subsets the + reference to the shared spatial panel (~hundreds of genes), some reference cells express + none of those genes (`nUMI == 0`). spacexr's `get_cell_type_info` divides each cell by + its `nUMI` → `NaN`; a single `NaN` column poisons `other_mean` (rowMeans) for **every** + cell type in `get_de_genes` → all logFC `NaN` → 0 DE genes → `create.RCTD` aborts with + "fewer than 10 regression differentially expressed genes". Dropping zero-count cells here + is the fix (see memory `rctd-split-zero-umi-reference-nan.md`). **This — not the + thresholds — was the true root cause of the historical abort.** +5. **L52-53 — CELL_MIN_INSTANCE, applied by hand.** Keep only reference cell types with + >=25 cells. This replicates RCTD's `CELL_MIN_INSTANCE=25` default but is a **hardcoded + pre-filter, not an exposed arg** (so it never reaches `create.RCTD`). +6. **L82-87** sanitize cell-type factor levels containing `/` (spacexr's + `check_cell_types` rejects them); keep a safe→original name map to restore labels later. +7. **L89** `Reference(ref_counts, cell_types, min_UMI = 0)` — `min_UMI` hardcoded to 0 so + the (now zero-UMI-free) cells all pass. +8. **L100-105** `create.RCTD(...)` — the **6 exposed threshold args** are forwarded here: + `gene_cutoff`, `fc_cutoff`, `gene_cutoff_reg`, `fc_cutoff_reg`, `UMI_min`, + `UMI_min_sigma`. `max_cores` = `meta$cpus` (performance only). +9. **L106** `run.RCTD(myRCTD, doublet_mode = "doublet")` — `doublet_mode` is **hardcoded**, + not exposed. +10. **L109-114** take `results_df$first_type`; pixels with `spot_class == "reject"` are + relabelled `None_sp`. +11. **L117-128** write predictions back into `colData(sce)$cell_type` (restoring the + original `/`-containing names) and `write_h5ad` the result. + +## Arguments + +Six exposed args, all `create.RCTD` DE-gene / UMI thresholds. **Every config default is +deliberately relaxed away from the spacexr default** — RCTD's defaults are tuned for +whole-transcriptome references (~20k genes) with high per-cell UMIs, whereas iST uses small +curated panels (~100-500 genes) with compressed fold-changes and low per-cell counts, so the +stock defaults strand too few DE genes / drop too many cells. + +| Arg (config) | Config default | spacexr default | Maps to | Meaning | +|---|---|---|---|---| +| `--gene_cutoff` | 0.0 | 0.000125 | `create.RCTD(gene_cutoff)` | min normalized mean expr for a platform-effect DE gene | +| `--fc_cutoff` | 0.1 | 0.5 | `create.RCTD(fc_cutoff)` | min log-FC for a platform-effect DE gene | +| `--gene_cutoff_reg` | 0.0 | 0.0002 | `create.RCTD(gene_cutoff_reg)` | min normalized mean expr for a regression DE gene | +| `--fc_cutoff_reg` | 0.1 | 0.75 | `create.RCTD(fc_cutoff_reg)` | min log-FC for a regression DE gene | +| `--umi_min` | 20 | 100 | `create.RCTD(UMI_min)` | min total UMI per spatial cell to annotate it | +| `--umi_min_sigma` | 20 | 300 | `create.RCTD(UMI_min_sigma)` | min UMI for cells used to fit platform-effect variance | + +Hardcoded (NOT exposed): `doublet_mode="doublet"` (L106), the `>=25` cells/type filter +(≈`CELL_MIN_INSTANCE`, L52), `Reference(min_UMI=0)` (L89). + +## Setup / Docker + +Base `openproblems/base_r:1`; installs `SingleCellExperiment`, `anndataR`, `rhdf5`, +`devtools` via Bioc, then `devtools::install_github('dmcable/spacexr', build_vignettes=FALSE)`. +The `SingleCellExperiment` reinstall comment (config L63-66) is a workaround for a +`SpatialExperiment`/Seurat install-order bug. **The `install_github` is not commit-pinned**, +so the spacexr version floats — a risk point if upstream changes DE-gene defaults or the +`create.RCTD`/`run.RCTD` signatures. + +## Wiring + +- Registered as `rctd` in the `celltype_annotation_methods` stage of the run_benchmark + workflow. Stage default is `tacco`. +- Nextflow labels (config L75): `hightime, midcpu, highmem`. +- Sweep scripts: `scripts/run_benchmark/param_sweep/rctd_params.yaml` + + `run_test_rctd_nebius.sh` (see below). + +## Risk points / gotchas + +- **Small-panel DE-gene floor.** Even with the zero-UMI fix, raising the fold-change + thresholds toward the spacexr defaults shrinks the DE-gene set; on a very small panel this + can drop back under the 10-gene floor and re-trigger the `create.RCTD` abort. The relaxed + defaults exist precisely to stay clear of that floor. **Consequence for the sweep: the + high-threshold variants (values approaching the spacexr defaults) may legitimately fail on + small-panel datasets** — that failure boundary is part of what the sweep measures. +- Un-pinned spacexr (version floats with build date). +- `doublet_mode` and the `CELL_MIN_INSTANCE`-equivalent filter are hardcoded — see + Optimization / tuning below. + +## Optimization / tuning + +Impact tiers (grounding: Cable et al. 2022 + spacexr `create.RCTD`/`run.RCTD` docs). The +sweep in `param_sweep/rctd_params.yaml` varies only **already-exposed** args (so it is +submittable against the current build/main container without a rebuild). + +- **Tier 0 — input, not a parameter.** The scRNA-seq **reference**: its cell-type + granularity and, above all, its **overlap with the spatial gene panel**. RCTD learns + profiles only on the shared-panel genes, so panel size/quality dominates everything below. + Not a sweep axis. +- **Tier 1 — highest impact on quality (EXPOSED, all six on the sweep).** The DE-gene / + UMI thresholds. `fc_cutoff` / `fc_cutoff_reg` are the sharpest levers (they set which genes + are "informative"); `umi_min` sets which cells get annotated at all vs dropped; + `gene_cutoff` / `gene_cutoff_reg` / `umi_min_sigma` are companion filters. **All six are + set to a non-default (relaxed) value, so per the non-default rule all six are on a sweep + axis, each range straddling the relaxed config default and the spacexr default.** Because + the relaxed default already sits at/near the permissive extreme, the meaningful sweep + direction is *toward* the stricter spacexr default. +- **Tier 1 but NOT exposed → Tier 3 (deferred).** `doublet_mode` (`run.RCTD`, hardcoded + `"doublet"`). This is RCTD's single biggest behavioural lever: `"doublet"` assigns ≤2 + types/pixel (the paper's headline mode, aimed at mixed Slide-seq pixels), `"full"` does + unrestricted deconvolution, `"multi"` is a greedy multi-type extension. For one-cell-per- + object segmented iST, `doublet` + `first_type` is a reasonable default, but `full`/`multi` + could change results materially. **Worth exposing** as `--doublet_mode` (thread into the + `run.RCTD` call). Not in this sweep — exposing it needs `viash ns build` + a container + rebuild, so a stale build/main container would silently ignore it. +- **Tier 3 (deferred) — other un-exposed knobs.** `CELL_MIN_INSTANCE` (currently the + hardcoded `>=25` cells/type pre-filter, L52) — exposing it would let the sweep trade rare- + cell-type coverage against profile stability. `Reference(min_UMI=0)` (L89) is intentional + given the zero-UMI fix and is best left fixed. +- **Performance only (fixed, never swept):** `max_cores` (= `meta$cpus`). + +**To promote any Tier-3 knob into a real sweep:** add the arg to `config.vsh.yaml` +(`type`, `default` = the spacexr default), thread it into the `run.RCTD`/`create.RCTD` call +in `script.R`, `viash config view` to validate, then `viash ns build` + rebuild the +container (see `check-component`) before it can be swept on the cloud. diff --git a/src/methods_cell_type_annotation/singler/NOTES.md b/src/methods_cell_type_annotation/singler/NOTES.md new file mode 100644 index 000000000..23de849d5 --- /dev/null +++ b/src/methods_cell_type_annotation/singler/NOTES.md @@ -0,0 +1,147 @@ +# singler — developer notes + +## What this component is + +Cell-type annotation method (stage API `src/api/comp_method_cell_type_annotation.yaml`, +subtype `method_cell_type_annotation`). It is a **thin Python wrapper around +[singler-py](https://github.com/SingleR-inc/singler-py)** (the Python port of the +Bioconductor `SingleR`), a **reference-correlation labeller**: it builds per-label marker +sets from an annotated scRNA-seq reference, then assigns each spatial cell the label whose +reference profile its expression correlates with best (Spearman, per-label score at a +correlation quantile), optionally refined by fine-tuning. + +- CPU-only (no GPU); resource label `[ midtime, midcpu, midmem ]`. +- Docker: `openproblems/base_python:1` + `pypi: [singler]`, merging + `/src/base/setup_spatialdata_partial.yaml`. Merges the base setup and needs no version + gymnastics — nothing load-bearing to record here. +- Links: docs/repo `https://github.com/SingleR-inc/singler-py`. + +### Citation note +`references.doi: 10.1038/s41590-018-0276-y` is **correct** — Aran et al., *Nat Immunol* +2019, the paper that introduced SingleR. (Unlike some components in this repo, the DOI is +the method-specific paper, not a generic framework paper, so no fix is needed.) + +## script.py — step by step + +1. `sce.read_h5ad(input_spatial_normalized_counts)` and a parallel `ad.read_h5ad` of the + same file (L27-28). The `SingleCellExperiment` is used for the matrix; the `AnnData` is + the object we write labels back onto. +2. `sce_ref = sce.read_h5ad(input_scrnaseq_reference)` (L30). +3. Test matrix = the spatial **`counts`** assay, `mat.sorted_indices()` (L34-35). Reference + matrix = the reference **`normalized`** assay, `sorted_indices()` (L37-38). + `sorted_indices()` is the "magic line" that puts the CSR/CSC into the layout singler + expects. +4. `singler.train_single(ref_data=mat_ref, ref_labels=, + ref_features=..., test_features=...)` (L41-44) — builds the prebuilt reference. +5. `singler.classify_single(mat, ref_prebuilt=built)` (L47) — classifies. +6. `adata_sp.obs["cell_type"] = output["best"]` (L49), then `adata_sp.write(output)` (L53). + +### Two things that bite (read before touching this) + +- **The two config-exposed non-IO args are DEAD.** `--celltype_key` (default `cell_type`) + and `--labels_key` (default `cell_labels`) are declared in the merged config but the + script **never reads `par['celltype_key']` or `par['labels_key']`**. `ref_labels` is + hardcoded to `sce_ref.get_column_data().column("cell_type")` (L42), and the output column + is hardcoded to `"cell_type"` (L49). So changing either arg on the command line (or via a + sweep) currently has **no effect whatsoever**. See "Optimization / tuning" — wiring + `celltype_key` is the single highest-leverage fix and the basis of the prepared sweep. +- **Test matrix is raw `counts`, reference is `normalized`.** SingleR scores with a + rank-based (Spearman) correlation, which is invariant to per-cell monotonic scaling, so a + raw-counts test side is mostly harmless; but marker detection on the reference does expect + log-normalized input, which is why the reference uses `normalized`. Not a tuning axis, + just context. + +## Arguments + +| arg | default | what it maps to upstream | status | +|-----|---------|--------------------------|--------| +| `--input_spatial_normalized_counts` | — | test matrix (`counts` assay) + AnnData to annotate | used | +| `--input_scrnaseq_reference` | — | `train_single(ref_data=…)` (`normalized` assay) | used | +| `--input_transcript_assignments` | — | (optional, unused by script) | ignored | +| `--celltype_key` | `cell_type` | *should* select the reference label column for `ref_labels` | **declared but NOT wired** (hardcoded `"cell_type"`) | +| `--labels_key` | `cell_labels` | (spatial label key) | **declared but NOT wired** (never read) | +| `--output` | — | `adata_sp.write(...)` | used | + +None of singler-py's algorithm knobs (`marker_method`, `num_de`, `quantile`, +`use_fine_tune`, `fine_tune_threshold`, `aggregate`) are exposed — `train_single` / +`classify_single` are called entirely at their upstream defaults. + +## Wiring + +- Registered in `src/workflows/run_benchmark/config.vsh.yaml`: dependency list + (`methods_cell_type_annotation/singler`) and the `--celltype_annotation_methods` default + string `ssam:tacco:moscot:mapmycells:tangram:singler:rctd`; fan-out in + `src/workflows/run_benchmark/main.nf` (~L383). Present on `build/main` (config + `target/`). +- Sweep scripts: `scripts/run_benchmark/param_sweep/singler_params.yaml` + + `run_test_singler_nebius.sh`. + +## Optimization / tuning + +**Grounded in the singler-py source** (`_train_single.py`, `_classify_single.py`, +fetched from `SingleR-inc/singler-py`), whose real defaults are: + +`train_single(..., marker_method="classic", num_de=None, aggregate=False, ...)` and +`classify_single(..., quantile=0.8, use_fine_tune=True, fine_tune_threshold=0.05, ...)`. + +### Tier 0 — the input, not a parameter +The **reference** dominates SingleR accuracy: which scRNA-seq atlas, how well its panel +overlaps the iST gene set, and **which annotation granularity** it is labelled at. The +mouse-brain test reference carries several nested label columns — `cell_type`, +`cell_type_level2`, `cell_type_level3`, `cell_type_level4` (coarse→fine). Selecting among +them is exactly what `--celltype_key` is *meant* to do (see Tier 1). + +### Tier 1 — highest impact on output quality +- **`celltype_key` (annotation granularity)** — ALREADY EXPOSED but **UNWIRED** (see the + "dead args" note). Coarser labels → higher per-class accuracy but less resolution; finer + labels (`level3/4`) → more classes, harder correlation separation on a small iST panel. + This is the one meaningful axis for a reference labeller and the basis of the prepared + sweep — **but it only varies output after this one-line fix + a container rebuild**: + + ```python + # L42, replace: + ref_labels = sce_ref.get_column_data().column("cell_type") + # with: + ref_labels = sce_ref.get_column_data().column(par["celltype_key"]) + ``` + +- **`marker_method`** `{"classic","auc","cohens_d"}`, default `"classic"` — chooses how + per-label markers are ranked. On small, low-count iST panels `cohens_d`/`auc` can pick + more robust discriminative genes than classic pairwise log-FC. **Not exposed.** +- **`use_fine_tune`** (default `True`) / **`fine_tune_threshold`** (default `0.05`) — + fine-tuning re-scores the top candidate labels using only their mutual markers; the + single biggest accuracy lever between similar cell types. Currently left at the + (good) default `True`. **Not exposed** (would need exposing to sweep the threshold or to + measure the cost of turning it off). + +### Tier 2 — quality/speed trade-offs +- **`num_de`** (default: auto for classic, else 10) — markers per pairwise comparison. **Not exposed.** +- **`quantile`** (default `0.8`) — quantile of the correlation distribution used for each + label's score; lower is more robust to a few high-correlation outlier genes. **Not exposed.** +- **`aggregate`** (default `False`) — pseudo-bulk the reference for speed on large atlases, + small accuracy change. **Not exposed.** + +### Tier 3 — not exposed by the component (future work) +`marker_method`, `num_de`, `quantile`, `use_fine_tune`, `fine_tune_threshold`, `aggregate` +are all reachable in singler-py but **not surfaced** in `config.vsh.yaml`, and the script +passes none of them. Exposing any of these means: add the arg to `config.vsh.yaml` +`arguments:`, thread it into the `train_single`/`classify_single` call in `script.py`, then +`viash ns build` + rebuild the container (see the `check-component` skill) — a +**stale `build/main` container silently ignores a newly-added arg**, so a sweep over any of +these is NOT submittable until the rebuild lands. + +### The prepared sweep (and its caveat) +`singler_params.yaml` sweeps **`celltype_key`** over `[cell_type_level2, cell_type_level3, +cell_type_level4]` (default `cell_type` covered by the default variant) → 4 variants total. +`celltype_key` already exists in `build/main`'s config, so the sweep **launches without a +rebuild**. HOWEVER, because of the dead-arg bug above, the deployed `build/main` script +hardcodes the `"cell_type"` column and ignores the value — so **until the one-line wiring +fix is applied and the container is rebuilt, the four variants produce identical output** +(the sweep is a no-op that measures nothing). Apply the fix + rebuild before using this +sweep to draw conclusions. + +### Non-default audit +No exposed argument sits at a non-default (non-performance) value that deviates from an +upstream tool default: `celltype_key`/`labels_key` are data-key strings (no upstream tool +default to deviate from), and every singler-py algorithm knob is left at its library +default. Nothing was promoted onto the sweep axis by the non-default audit; `celltype_key` +is on the axis by design choice (Tier-1 granularity), not because it was mis-defaulted. diff --git a/src/methods_cell_type_annotation/ssam/NOTES.md b/src/methods_cell_type_annotation/ssam/NOTES.md new file mode 100644 index 000000000..776cc98f1 --- /dev/null +++ b/src/methods_cell_type_annotation/ssam/NOTES.md @@ -0,0 +1,108 @@ +# ssam — cell-type annotation (SSAM) + +## What this component is + +Cell-type-annotation stage method (API `src/api/comp_method_cell_type_annotation.yaml`, +subtype `method_cell_type_annotation`). It labels each spatial cell with a cell type by the +**SSAM** approach: signature-based, cell-segmentation-free inference of cell types from an +mRNA-density map, then a per-cell majority vote over the transcripts assigned to that cell. + +- Docs: https://ssam.readthedocs.io — Repo: https://github.com/HiDiHlabs/ssam +- Paper (config `references.doi`, verified correct): Park, Choi, Tiesmeyer et al., + "Cell segmentation-free inference of cell types from in situ transcriptomics data", + *Nat Commun* 12, 3545 (2021), DOI `10.1038/s41467-021-23807-4` (PMID 34112806). + No citation caveat — the config DOI is the SSAM method paper. + +**Important implementation caveat.** Despite the name/DOI, this component does **not** run +the original `ssam` package. It calls `txsim.preprocessing.run_ssam` (txsim from +`theislab/txsim@dev`), which internally uses the **`plankton`/`planktonspace`** +re-implementation: `from plankton.utils import ssam`. The Docker setup installs +`planktonspace` + `matplotlib<3.9` on top of the spatialdata/txsim base. So the algorithm +is plankton's `ssam()`, with defaults that differ from the paper (see below). + +## script.py — step by step + +1. Assert `input_transcript_assignments` and `input_scrnaseq_reference` are provided + (both required for this method). +2. Read `input_spatial_normalized_counts` (h5ad), the `transcripts` element of the + `transcript_assignments.zarr`, and the SC reference; set `adata_sc.X` to its + `layers["normalized"]`. +3. Subset the spatial AnnData to genes shared with the SC reference. +4. Call `tx.preprocessing.run_ssam(adata_sp, transcripts.compute(), adata_sc, + um_p_px=par['um_per_pixel'], cell_id_col='cell_id', gene_col='feature_name', + sc_ct_key=par['celltype_key'])`. +5. Copy `obs['ct_ssam']` -> `obs['cell_type']` (string) and write output. + +**What `run_ssam` actually does** (txsim `preprocessing/_ctannotation.py::run_ssam`): +builds a `plankton.SpatialData` from the transcript gene column and `x*um_p_px`, +`y*um_p_px`; derives mean expression signatures per SC cell type; calls +`ssam(sdata, signatures=..., kernel_bandwidth=4, patch_length=1500, threshold_cor=0.2, +threshold_exp=0.1)` to produce a cell-type map; samples the map at every molecule; then +majority-votes a cell type per `cell_id`. `ct_ssam_cert` records the fraction of a cell's +spots that agree with the winning label. + +Two live `# TODO`s in the script flag a known correctness risk: transcripts are passed in +**physical (µm) space**, not pixel space, so `um_p_px` scaling interacts with the fixed +kernel in a way the author suspects yields poor results ("ssam most likely outputs bad +results since the transcripts are provided in physical space instead of pixel space"). This +is exactly why `um_per_pixel` is the meaningful thing to sweep. + +## Arguments + +| Arg | Type | Config default | txsim/tool default | Maps to | +|-----|------|----------------|--------------------|---------| +| `--um_per_pixel` | double | **0.5** | `um_p_px=0.325` | scales transcript x/y before the density map (only exposed tuning knob) | +| `--celltype_key` | string | `cell_type` (from API) | `sc_ct_key='celltype'` | which SC `obs` column holds cell types (schema arg, not a tuning knob) | +| `--input_*` / `--output` | file | — | — | standard stage I/O | + +Note: the config `um_per_pixel` default (0.5) **deviates** from txsim's `um_p_px` default +(0.325). The `# TODO` on the arg ("Should be able to infer this from transcripts") indicates +it is a placeholder rather than a tuned value. + +## Setup / Docker + +Merges `setup_spatialdata_partial.yaml` + `setup_txsim_partial.yaml` on +`openproblems/base_python:1`, then `pypi: [planktonspace, "matplotlib<3.9"]`. The +`matplotlib<3.9` pin is load-bearing (see commit `80c18d704 "ssam matplotlib"`): plankton's +plotting import breaks against matplotlib >= 3.9. No further Docker drama recorded. + +## Wiring + +- Registered as `celltype_annotation_methods` in the run_benchmark workflow config; the + stage default is `tacco`. `main.nf` fans out one variant per enabled annotation method. +- Param sweep: `scripts/run_benchmark/param_sweep/ssam_params.yaml` + + `run_test_ssam_nebius.sh` (CPU compute env, no `gpu` label; enables `tacco` + `ssam`). + +## Optimization / tuning + +**Tier 0 — input / coordinate space (biggest lever, not a sweepable arg).** Transcripts +arrive in µm (physical) space, contradicting SSAM's pixel-grid assumption. Fixing the +coordinate convention (or deriving the true µm/px from the transcript table, per the arg's +TODO) would likely matter more than any single-knob sweep. + +**Tier 1 — the only exposed knob: `um_per_pixel`.** With the SSAM Gaussian +`kernel_bandwidth` fixed at 4 (µm-equivalent) inside txsim, `um_per_pixel` is the *only* +reachable control over the effective KDE smoothing scale: it rescales the point cloud +relative to the fixed kernel. Smaller -> coarser smoothing; larger -> finer. Swept over +`[0.1, 0.2, 0.325, 1.0]` (straddling the txsim default 0.325 and the shipped 0.5, which the +default variant already covers). + +**Tier 3 — high-value knobs NOT exposed today (future work; NOT in this sweep).** These are +the true SSAM quality dials, but they are **hardcoded inside txsim's `run_ssam`** call to +`ssam(...)`, so the component cannot forward them. Exposing them means editing +`txsim/preprocessing/_ctannotation.py` (add params to the `run_ssam` signature + the `ssam()` +call), then adding matching `config.vsh.yaml` args and rebuilding the image +(`viash ns build` + container rebuild on `build/main` — see the `check-component` skill). +Until then they must NOT go in `ssam_params.yaml`: + +- `kernel_bandwidth` (txsim hardcodes **4**; SSAM's own default is 2.5 µm) — the KDE + bandwidth; the single biggest quality dial. Lower = sharper/noisier, higher = smoother. +- `threshold_cor` (hardcoded **0.2**) — minimum correlation between a pixel's local + expression vector and a signature for a cell type to be called; SSAM typically uses ~0.6. + Directly trades recall vs precision of assignments. +- `threshold_exp` (hardcoded **0.1**) — minimum total expression for a pixel to be + classified at all (a foreground/vector-field threshold). +- `patch_length` (hardcoded **1500**) — tiling size; primarily a memory/speed knob (would + stay fixed even if exposed). +- `no_ct_assigned_value` (`run_ssam` default `'None_sp'`) — label for unassigned cells; + behavioural, not a quality dial. diff --git a/src/methods_cell_type_annotation/tangram/NOTES.md b/src/methods_cell_type_annotation/tangram/NOTES.md new file mode 100644 index 000000000..40b5c6b53 --- /dev/null +++ b/src/methods_cell_type_annotation/tangram/NOTES.md @@ -0,0 +1,190 @@ +# Tangram cell-type annotation — implementation notes + +Reference for anyone (human or Claude) working on this component. Captures how it +works, which of Tangram's many knobs are actually wired up, why the two shipped +defaults were chosen, and where to tune for quality vs. time. + +Links: docs https://tangram-sc.readthedocs.io · repo +https://github.com/broadinstitute/Tangram. + +**Citation:** the config's `references.doi` is `10.1038/s41592-021-01264-7` — +Biancalani et al., "Deep learning and alignment of spatially resolved single-cell +transcriptomes with Tangram", *Nature Methods* 18, 1352–1362 (2021), PMID +34711971. Verified via PubMed: this **is** the method-specific paper (not a generic +framework reference), so no citation caveat is needed. + +## What this component is + +One of the methods at the **cell-type-annotation** stage of the iST preprocessing +benchmark (`src/methods_cell_type_annotation/`). It implements the standard API +`src/api/comp_method_cell_type_annotation.yaml`: + +- in: `input_spatial_normalized_counts` (`.h5ad`, cell × gene with a `normalized` + layer) + `input_scrnaseq_reference` (`.h5ad`, reference atlas with a `normalized` + layer and a `celltype_key` obs column, default `cell_type`). +- out: the spatial AnnData with a per-cell label written to `obs[celltype_key]`. + +It is a **GPU deep-learning method**: Tangram (torch) learns a soft mapping matrix +between the reference and the spatial cells by gradient descent, then projects the +reference labels through that mapping. Nextflow directives are +`[ midtime, midcpu, midmem, gpuh100 ]` — the `gpuh100` label is what schedules it +onto a GPU node. + +## script.py — step by step + +1. **Device pick** (`:19-23`) — `"cuda:0"` if `torch.cuda.is_available()` else + `"cpu"`. Tangram runs on CPU but is much slower there. +2. **Read inputs** (`:30-31`) — spatial + reference AnnData. +3. **Use the log1p-normalized layer as `X`** (`:33-35`) — both adatas' `X` is + replaced by their `normalized` layer before mapping. (That layer is produced + upstream by the normalization stage / the SC process workflow, not here.) + `adata_sp_orig` is stashed so the final output carries the original layers, not + Tangram's intermediate additions. +4. **Markers = the full spatial panel** (`:39-40`) — every `adata_sp.var_name` is + used as a training gene. iST panels are small (hundreds of genes) so "all genes" + is reasonable; there is no marker-selection step. +5. **`tg.pp_adatas`** (`:44`) — intersects genes across the two adatas, drops + all-zero genes, and stores density priors. Mutates both adatas in place. +6. **Build `map_kwargs` and map** (`:56-66`) — calls `tg.map_cells_to_space` with + `mode=par['mode']`, `num_epochs=par['num_epochs']`, `density_prior='uniform'` + (hardcoded), `device`. In `clusters` mode it also passes + `cluster_label=par['celltype_key']` (`:64-65`) — required so Tangram averages + reference cells within each cluster. +7. **Project labels** (`:69-73`) — `tg.project_cell_annotations` writes a + spot × cell-type score frame to `adata_sp.obsm['tangram_ct_pred']`. +8. **Argmax → label** (`:76-80`) — restore `adata_sp_orig`, then set + `obs[celltype_key] = tangram_ct_pred.idxmax(axis=1)` (hard label = highest-scoring + type). The commented block (`:83-87`) would additionally store a per-cell + confidence score and the full score matrix — currently disabled. +9. **Write** (`:90`). + +## Arguments + +Only **two** knobs are exposed; everything else in `map_cells_to_space` is left at +the Tangram default or hardcoded in the script. + +| config arg | default | Tangram default | forwarded to | note | +|---|---|---|---|---| +| `--mode` | `clusters` | `cells` | `map_cells_to_space(mode=…)` | **deviates** from tool default; see below | +| `--num_epochs` | `1000` | `1000` | `map_cells_to_space(num_epochs=…)` | matches tool default | + +**Why `mode: clusters` (a deliberate non-default).** In `cells` mode Tangram learns +an `(n_sc_cells × n_spatial_cells)` mapping matrix on the GPU, which exhausts VRAM +for large references (the config comment cites the 101k-cell MPII lung reference). +`clusters` mode maps per-cell-type *clusters* instead, shrinking the matrix by +~cells-per-type so it fits comfortably. It is also the natural mode for **label +projection** (we only want the cell-type label, not an individual-cell +correspondence). So the deviation is a VRAM/scale choice — but note it *does* change +the output (cluster-level vs. cell-level mapping), so it is treated as a tunable +axis, not a silent baseline (see "Optimization / tuning"). + +**Hardcoded (not a config arg):** `density_prior='uniform'` (`:62`). Tangram's tool +default is `'rna_count_based'`. The docstring guidance (`:48-50`) is: use `'uniform'` +when the spatial voxels are at **single-cell resolution** (MERFISH/Xenium-style), +and `'rna_count_based'` when density is expected to track RNA counts. Since iST here +is single-cell-resolution, `'uniform'` is the appropriate choice — but it is a +deviation from the tool default and is **not exposed**, so it can only be swept +after being added to the config (see Tier 3). + +## config.vsh.yaml — the Docker setup + +- Base image `openproblems/base_python:1` + `pypi: [tangram-sc]`. +- The config carries a **history comment**: `openproblems/base_pytorch_nvidia:1` + (the CUDA/torch base used by other GPU methods) was tried first and "leads to + dependency issues", so it was reverted to the plain python base and lets + `tangram-sc` pull its own torch. A `TODO` notes a different pytorch+CUDA base + might be worth trying. If you touch the image, that dependency clash is the thing + to re-verify. +- `native` engine is also declared (for local/CI without Docker). + +## Optimization / tuning + +Two framing facts before tuning: +1. Unlike the segmentation methods, the tangram defaults are **not** aggressively + speed-floored — `num_epochs=1000` is Tangram's own recommended default. The one + real deviation is `mode` (chosen for VRAM/scale, see above). +2. All ranges below are grounded in the Tangram docs signature (verified against + `tangram/mapping_utils.py`: `map_cells_to_space(mode="cells", + learning_rate=0.1, num_epochs=1000, lambda_g1=1, lambda_d=0, + density_prior='rna_count_based', …)`). + +**Tier 0 — the input, not a parameter.** The biggest lever is the *reference*: which +SC atlas, how well its cell types match the tissue, and the shared gene panel. +`pp_adatas` restricts training to genes present in both — a poorly matched reference +caps achievable accuracy regardless of any knob. Also note markers = the entire +spatial panel (no marker curation). + +**Tier 1 — highest impact on quality (quality vs. time): `num_epochs`** +*(exposed; default 1000 = tool default).* Tangram optimizes the mapping by gradient +descent for `num_epochs` steps; more epochs = better-converged mapping (to a point), +fewer = faster but under-converged and noisier labels. This is the primary +accuracy↔runtime dial. Sweep below and around 1000 (e.g. 100 / 500 / 2000 / 3000) to +find the knee — where extra epochs stop improving the argmax labels. + +**Tier 2 — `mode`** *(exposed; default `clusters`, tool default `cells`).* Not just a +resource knob — it changes the mapping granularity and therefore the labels. +`cells` maps individual reference cells (finer, can be more accurate on **small** +references) but its matrix scales with `n_sc_cells` and OOMs on large ones; the +config picked `clusters` for that reason. The one meaningful non-default value to +sweep is `cells` (feasible on the small test reference; expect OOM on large +references such as MPII lung on limited VRAM). Tangram also offers `constrained`, +but that mode is designed for **cell filtering** and needs `target_count` + +`lambda_count`/`lambda_f_reg` to be meaningful, so it is not a drop-in axis for pure +label projection — left out of the sweep. + +**Tier 3 — high-value knobs NOT exposed today (future work; would need config +expose + container rebuild, so NOT in the submittable sweep):** +- **`density_prior`** — hardcoded `'uniform'`; tool default `'rna_count_based'`. + Directly shapes how mass is distributed across spatial cells. Expose it (choices + `uniform` / `rna_count_based`) to test whether the single-cell-resolution + assumption actually helps on each dataset. Highest-value un-exposed knob. +- **`learning_rate`** *(tool default 0.1)* — step size of the Adam optimizer; + interacts with `num_epochs` (lower LR needs more epochs). Worth a small sweep + (0.05 / 0.1 / 0.5) once epochs are set. +- **`lambda_g1` / `lambda_d` / `lambda_r`** *(1 / 0 / 0)* — the loss weights + (gene-expression term, density term, entropy regularizer). Advanced; only worth + touching after epochs + density_prior are settled. + +To sweep any Tier-3 knob you must add it to `config.vsh.yaml`'s `arguments:` (type + +default = the Tangram default), thread it into the `map_kwargs` dict in `script.py`, +then `viash ns build` + rebuild the container (see the `check-component` skill). A +sweep launched against a stale `build/main` container would silently ignore a +freshly-added arg — which is exactly why these stay out of the current +submittable-now sweep. + +## Risk points / gotchas + +- **`cells` mode OOMs on large references.** The whole reason `clusters` is the + default. Sweeping `mode: cells` is safe on the small test reference but expect + VRAM blow-ups on 100k-cell references at limited VRAM. +- **`density_prior` is hardcoded, not a config arg.** Changing it requires editing + `script.py`, not the sweep file. +- **Only a hard argmax label is emitted** (`:80`); the per-cell confidence score + block is commented out (`:83-87`). Downstream metrics see labels only, no scores. +- **GPU base-image fragility.** The plain `base_python:1` + `tangram-sc` combo is + deliberate; `base_pytorch_nvidia:1` caused dependency issues (see config comment). +- **No end-to-end tuning run has been done yet** — this NOTES.md documents the setup + and the sweep design; results are not yet in. + +## Wiring + +- `src/workflows/run_benchmark/config.vsh.yaml`: dependency + `methods_cell_type_annotation/tangram` (`:181`); part of the annotation default + string `ssam:tacco:moscot:mapmycells:tangram:singler:rctd` (`:101`). +- `src/workflows/run_benchmark/main.nf:382`: imported in the annotation fan-out. +- Run scripts: enabled (uncommented) in `run_gpu_nebius.sh`; commented in the plain + `run_test_nebius.sh` / `run_test_local.sh` (GPU + time cost). +- **Parameter sweep:** `scripts/run_benchmark/param_sweep/tangram_params.yaml` + (committed default + sweep) and `run_test_tangram_nebius.sh` (Nebius GPU launch, + reads the params file from GitHub raw). The stage default `tacco` is kept alongside + `tangram` so the run has a baseline to compare against. + +## References + +- Docs: https://tangram-sc.readthedocs.io (the + `tangram.mapping_utils.map_cells_to_space` page has the full parameter reference). +- Repo: https://github.com/broadinstitute/Tangram +- **Paper (the DOI in `config.vsh.yaml`, verified correct):** Biancalani T. et al., + "Deep learning and alignment of spatially resolved single-cell transcriptomes with + Tangram", *Nature Methods* 18, 1352–1362 (2021), DOI 10.1038/s41592-021-01264-7, + PMID 34711971. diff --git a/src/methods_expression_correction/resolvi_correction/NOTES.md b/src/methods_expression_correction/resolvi_correction/NOTES.md new file mode 100644 index 000000000..67d24e88c --- /dev/null +++ b/src/methods_expression_correction/resolvi_correction/NOTES.md @@ -0,0 +1,126 @@ +# resolvi_correction — NOTES + +Authoritative how-it-works / why-the-setup / where-it-breaks reference for the +`resolvi_correction` expression-correction method. Committed with the code. + +## What this component is + +- **Stage / API:** `methods_expression_correction` (merges + `/src/api/comp_method_expression_correction.yaml`). Runs late in the iST pipeline: it takes + `spatial_with_cell_types.h5ad` (a cell x gene matrix that already has cell-type labels, + centroids, and a `normalized` layer) and rewrites its `counts` / `normalized` layers with a + denoised, spillover-/background-corrected version. +- **Underlying tool:** **resolVI** (`scvi.external.RESOLVI`) from scvi-tools — a variational + autoencoder that models each observed cell's expression as a mix of (α0) true self-expression, + (α1) diffusion/spillover from spatial neighbours, and (α2) unspecific background, then samples + the posterior to reassign misplaced molecules. +- **GPU:** deep VAE model → tagged `gpu` in the nextflow directives. NB the docker `image:` is + currently `openproblems/base_python:1` (CPU base) with an explicit TODO to move to a GPU + pytorch image (`base_pytorch_nvidia:1`) — see the commented line in `config.vsh.yaml`. So the + component requests a GPU node but the container is a CPU-first python image with `scvi-tools` + pip-installed; whether it actually lights up CUDA depends on the torch build in that image. +- **Links:** docs https://docs.scvi-tools.org/en/latest/user_guide/models/resolvi.html , + repo https://github.com/scverse/scvi-tools . + +## Citation + +`references.doi: 10.1101/2025.01.20.634005` is **correct** (verified via bioRxiv): Ergen & Yosef, +2025, *"ResolVI - addressing noise and bias in spatial transcriptomics"* — the method-specific +paper, not a generic framework citation. No caveat needed. + +## script.py — step by step + +1. Read `input_spatial_with_cell_types`; stash the incoming `normalized` layer as + `normalized_uncorrected` (script.py L32-33). +2. `sc.pp.filter_cells(min_genes=5)` — drop near-empty cells (L36). +3. Build the spatial neighbour input: stack `obs.centroid_x/centroid_y` into + `obsm["X_spatial"]` (L38-39) — this is what resolVI uses to find neighbours. +4. `RESOLVI.setup_anndata(labels_key=celltype_key, layer="counts")` (L45). +5. Construct `RESOLVI(..., semisupervised=True, n_hidden=, encode_covariates=, downsample_counts=)` + (L48-51). `semisupervised=True` is **hard-coded** (tutorials recommend the supervised variant + when cell-type labels are available, which they are at this stage). +6. `train(max_epochs=100)` — **hard-coded** (L52). +7. Two `sample_posterior` calls (**`num_samples=20`, `batch_size=4000` both hard-coded**): + - `model_corrected` → `px_rate` summarised at the q50/median (L55-59). + - `model_residuals` → `mixture_proportions, mean_poisson, per_gene_background, + per_neighbor_diffusion, px_r_inv` (L63-70). The exact `return_sites` set matters — see the + scvi-tools issue #3252 link in the code. +8. Corrected counts = `counts * px_rate_q50 / (1 + px_rate_q50 + mean_poisson)` (L80-81), then a + size-factor renormalisation into the `normalized` layer (L84-86). The renormalisation is + acknowledged as non-ideal in a long code comment (L88-97): the pipeline runs normalization + *before* correction, so the corrected counts are re-normalised here by a crude size factor + rather than re-running the real normalization method. + +**Input-contract note:** the code comment at L28 claims `par['input_sc']` is required, but the +script never reads `input_scrnaseq_reference` — only `input_spatial_with_cell_types`. The SC +reference is optional in the API and unused here. + +## Arguments (exposed in config.vsh.yaml) + +| arg | type | config default | upstream (scvi RESOLVI) default | notes | +|-----|------|----------------|--------------------------------|-------| +| `--celltype_key` | string | `cell_type` | n/a | `labels_key` for `setup_anndata`; input mapping, not a tuning knob (Tier 0) | +| `--n_hidden` | integer | `32` | **32** (matches) | VAE hidden-layer width; model capacity | +| `--encode_covariates` | boolean | `false` | `False` (matches; via `**model_kwargs`) | feed batch/covariates through the encoder | +| `--downsample_counts` | boolean | `true` | `True` (matches) | subsample per-cell counts to equalise depth in training | + +**Hard-coded (NOT exposed) knobs that matter:** `max_epochs=100`, `num_samples=20` (upstream +`sample_posterior` default is **1000**), `mixture_k` (upstream default **50**, unset here), +`batch_size=4000`, `semisupervised=True`, `n_latent=10`, `n_layers=2`, `dropout_rate=0.05`. + +## Optimization / tuning + +Verified upstream defaults (scvi-tools RESOLVI): `n_hidden=32`, `n_hidden_encoder=128`, +`n_latent=10`, `n_layers=2`, `dropout_rate=0.05`, `mixture_k=50`, `downsample_counts=True`; +`sample_posterior(num_samples=1000)`. + +**Non-default audit result:** all three exposed *tunable* args (`n_hidden`, `encode_covariates`, +`downsample_counts`) sit at their scvi upstream defaults — nothing was silently deviated, so +nothing is *forced* onto a sweep axis. They are swept because they are the only exposed levers. + +Tiers: + +- **Tier 0 — input, not a parameter.** `celltype_key` (which obs column carries the labels), + the spatial neighbour graph built from `centroid_x/y`, and the `semisupervised=True` choice. + Biggest lever overall is segmentation/assignment quality upstream, not resolVI's own knobs. +- **Tier 1 — highest posterior-quality impact, but NOT EXPOSED (see below).** `num_samples` + (posterior samples; script uses **20** vs upstream **1000** — the single biggest + quality-vs-time dial for denoising stability) and `max_epochs` (**100**, hard-coded). +- **Tier 2 — exposed quality/capacity trade-offs (what the sweep varies).** + - `n_hidden`: 32 (default) → **64, 128**. More capacity to model diffusion + background, + slower. (script.py's own grid-search note lists exactly 32/64/128.) + - `encode_covariates`: false → **true** (boolean flip). + - `downsample_counts`: true → **false** (boolean flip). +- **Tier 3 — high-value knobs NOT surfaced by the component (future work; would need a config + arg + container rebuild — NOT in the submittable sweep):** + - `num_samples` (posterior samples): expose as e.g. `--num_samples`, sweep `[50, 100, 200]` + straddling the script's 20 toward upstream's 1000. Highest expected quality gain. + - `max_epochs`: expose as `--n_epochs`, sweep around 100 (e.g. `[50, 100, 200, 400]`). + - `mixture_k`: number of background/diffusion mixture components (upstream 50); expose and + sweep e.g. `[10, 50, 100]`. + + Exposing any of these needs: add to `config.vsh.yaml arguments:`, thread `par[...]` into the + matching `RESOLVI(...)` / `train(...)` / `sample_posterior(...)` call in `script.py`, then + `viash ns build` + a container rebuild (see `check-component`) before a build/main Nebius run + would honour them. + +## Wiring + +- Enabled at the `expression_correction` stage. Present (commented) in the stock run scripts + (`run_test_nebius.sh`), active in `run_gpu_nebius.sh` alongside `no_correction`. +- Sweep: `scripts/run_benchmark/param_sweep/resolvi_correction_params.yaml` (committed params) + + `scripts/run_benchmark/param_sweep/run_test_resolvi_correction_nebius.sh` (Nebius launcher, + GPU compute env `5hfmdCBxMRd4nHZaJKYEQZ`, labels include `gpu`). Total variants = + 1 default + 2 (`n_hidden`) + 1 (`encode_covariates`) + 1 (`downsample_counts`) = **5**. + +## Risk points / gotchas + +- **Submittable-now constraint:** the Nebius run uses `--revision build/main`, so the sweep can + only vary args already in the build/main container (the four above). Any newly-exposed Tier-3 + arg is silently ignored until rebuilt. +- **`return_sites` set is load-bearing** (script.py L63-70, scvi issue #3252) — changing it can + break the residual sampling. +- **CPU image / gpu label mismatch** — see "What this component is". Confirm the image actually + provides a CUDA torch before relying on GPU acceleration. +- **Renormalisation is crude** (script.py L84-97) — the corrected `normalized` layer is a + size-factor rescale, not a re-run of the pipeline's real normalization step. diff --git a/src/methods_expression_correction/split/NOTES.md b/src/methods_expression_correction/split/NOTES.md new file mode 100644 index 000000000..de8090f55 --- /dev/null +++ b/src/methods_expression_correction/split/NOTES.md @@ -0,0 +1,146 @@ +# split — NOTES + +Authoritative how-it-works / why-the-setup / where-it-breaks reference for the `split` +expression-correction method. Committed with the code. + +## What this component is + +- **Stage / API:** `methods_expression_correction` (merges + `/src/api/comp_method_expression_correction.yaml`). Runs late in the iST pipeline: it takes + `spatial_with_cell_types.h5ad` (cell x gene matrix with cell-type labels, centroids, a + `counts` and a `normalized` layer) plus the `scrnaseq_reference.h5ad`, and rewrites the + spatial object's `counts` / `normalized` layers with purified (contamination-corrected) ones. + Unlike resolvi_correction, split **requires the SC reference** (it deconvolves against it). +- **Underlying tools:** **SPLIT** (`bdsc-tds/SPLIT`, R) layered on top of **RCTD** (spacexr). + RCTD deconvolves each spatial cell against the SC reference in `doublet_mode`; SPLIT then + "purifies" the counts — for a doublet it splits the mixed transcripts to the primary cell + type, removing spillover/contamination. R method (`script.R`), CPU-only. +- **GPU:** none. Nextflow directives `[ hightime, highcpu, highmem ]`; no `gpu` label. +- **Links:** docs/repo https://github.com/bdsc-tds/SPLIT . + +## Citation + +`references.doi: 10.1101/2025.04.23.649965` is **correct** (verified via bioRxiv): Bilous, +Buszta, et al., 2025, *"From Transcripts to Cells: Dissecting Sensitivity, Signal +Contamination, and Specificity in Xenium Spatial Transcriptomics"* — the paper that introduces +SPLIT (Spatial Purification of Layered Intracellular Transcripts). Method-specific, not a +generic framework citation. No caveat needed. + +## script.R — step by step + +1. Read `input_spatial_with_cell_types` twice — as `SingleCellExperiment` (`sce`) and as a + `Seurat` object (`xe`) (L30-31). `sce` carries the assays/coords; `xe` supplies the raw + `counts` layer handed to `SPLIT::purify`. +2. If `!keep_all_cells`: drop zero-count cells from both objects (L34-38). +3. Build the `SpatialRNA` puck from `centroid_x/centroid_y` + the `counts` assay (L41-46). +4. Read the SC reference; **drop reference cells with zero panel-gene counts** (L56-60) — after + subsetting to the shared ~few-hundred-gene panel, some cells have `nUMI==0`, which makes + spacexr's per-cell normalisation `NaN` and poisons every cell type's DE-gene means (see the + memory note `rctd-split-zero-umi-reference-nan`). This guard is load-bearing. +5. Keep only cell types with >=25 cells (RCTD minimum) (L62-64); sanitise `/` out of cell-type + names (spacexr's `check_cell_types` rejects them) (L76); build the `Reference(min_UMI=0)`. +6. `create.RCTD(puck, reference, max_cores, gene_cutoff, fc_cutoff, gene_cutoff_reg, + fc_cutoff_reg, UMI_min, UMI_min_sigma)` (L89-94) — **this is where all six exposed threshold + args are forwarded.** Then `run.RCTD(doublet_mode = "doublet")` (L95) — `doublet_mode` is + **hard-coded**. +7. `SPLIT::run_post_process_RCTD(myRCTD)` (L106) then `SPLIT::purify(counts=, + rctd=RCTD, DO_purify_singlets=TRUE)` (L110-114) — `DO_purify_singlets` is **hard-coded TRUE** + and `DO_remove_residual_contamination` is **left at its package default (not set)**. +8. Stash the incoming `normalized` as `normalized_uncorrected` (L121); write purified counts + into `corrected_counts` (copy counts, then overwrite the updated cells) (L124-127); + `logNormCounts` with library-size factors -> new `normalized` (L130-131); `write_h5ad` (L136). + +## Arguments (exposed in config.vsh.yaml) + +All six thresholds map straight into the `create.RCTD(...)` call; `keep_all_cells` is a +component-local zero-cell guard (no upstream equivalent). + +| arg | type | config default | spacexr (RCTD) default | notes | +|-----|------|----------------|------------------------|-------| +| `--keep_all_cells` | boolean | `false` | n/a | drop zero-count cells; description warns TRUE "may cause errors". Robustness flag, not a tuning lever. | +| `--gene_cutoff` | double | `0.0` | **0.000125** | min normalized mean expr, platform-effect DE gene | +| `--fc_cutoff` | double | `0.1` | **0.5** | min log-FC, platform-effect DE gene | +| `--gene_cutoff_reg` | double | `0.0` | **0.0002** | min normalized mean expr, regression DE gene | +| `--fc_cutoff_reg` | double | `0.1` | **0.75** | min log-FC, regression DE gene | +| `--umi_min` | integer | `20` | **100** | min total UMI per spatial cell to include | +| `--umi_min_sigma` | integer | `20` | **300** | min UMI for cells fitting the platform-effect variance | + +**Why the six defaults are relaxed (deliberate, not accidental):** RCTD's DE-gene / UMI +thresholds are tuned for whole-transcriptome references (~20k genes). iST panels are small +(~100-500 genes) and low-count, so the stock spacexr thresholds strand too few DE genes and +`create.RCTD` aborts with *"fewer than 10 regression differentially expressed genes"*. These +defaults mirror the standalone `rctd` cell-type-annotation component exactly. + +**Hard-coded (NOT exposed) SPLIT/RCTD knobs that matter:** `doublet_mode="doublet"` (L95), +`DO_purify_singlets=TRUE` (L113), `DO_remove_residual_contamination` (unset -> package default, +L110-114), `Reference(min_UMI=0)` (L78), the `>=25`-cells-per-type filter (L63). + +## Optimization / tuning + +**Verified upstream defaults** — spacexr `create.RCTD`: `gene_cutoff=0.000125`, `fc_cutoff=0.5`, +`gene_cutoff_reg=0.0002`, `fc_cutoff_reg=0.75`, `UMI_min=100`, `UMI_min_sigma=300`. SPLIT +`purify` (from the repo README): `DO_purify_singlets` and `DO_remove_residual_contamination` +(the latter "removes an additional 2-5% of counts" for improved specificity, v0.3.0+). + +**Non-default audit result:** all six exposed thresholds sit at RELAXED, non-default values +(walked away from the spacexr defaults). None is a performance/resource knob, so per the +phase-3 audit **each is forced onto a sweep axis** — the deviation is itself evidence the knob +matters. `keep_all_cells` is the one exposed arg NOT swept: it has no upstream tool default to +straddle and its own description warns that flipping it to TRUE "may cause errors", so it stays +fixed at `false`. (This is the "do not sweep every knob" — 6 of the 7 exposed args are swept.) + +Tiers: + +- **Tier 0 — input, not a parameter.** The SC reference quality/annotation and the shared panel + gene set (subsetting to a small panel is what forces the relaxed thresholds in the first + place); upstream segmentation/assignment quality. Biggest levers overall. +- **Tier 1 — the exposed thresholds (what the sweep varies).** Each relaxed default is walked + back TOWARD its spacexr default (the meaningful direction; the relaxed default already sits at + the permissive extreme): + - `fc_cutoff`: 0.1 -> **0.25, 0.5** (sharpest — platform-effect DE-gene selection) + - `fc_cutoff_reg`: 0.1 -> **0.4, 0.75** (sharpest — regression DE-gene selection) + - `gene_cutoff`: 0.0 -> **0.0000625, 0.000125** (companion expression-mean filter) + - `gene_cutoff_reg`: 0.0 -> **0.0001, 0.0002** (companion expression-mean filter) + - `umi_min`: 20 -> **50, 100** (which spatial cells get deconvolved) + - `umi_min_sigma`: 20 -> **100, 300** (which cells fit the platform-effect variance) + Total variants = 1 default + 6 axes x 2 values = **13**. +- **Tier 3 — high-value SPLIT-specific knobs NOT surfaced by the component (future work; would + need a config arg + `viash ns build` + container rebuild — NOT in the submittable sweep):** + - `DO_purify_singlets` (SPLIT `purify`, hard-coded TRUE, script.R L113): whether to also + purify singlet cells, not just doublets. Arguably the #1 SPLIT-specific lever. Expose as a + boolean (`--do_purify_singlets`, default true) and sweep the flip to `false`. + - `DO_remove_residual_contamination` (SPLIT `purify`, unset, v0.3.0+): removes an extra 2-5% + of non-reference-supported counts for higher specificity. Expose as a boolean + (`--remove_residual_contamination`, default false) and sweep the flip to `true`. + - `doublet_mode` (RCTD `run.RCTD`, hard-coded "doublet", script.R L95): "doublet" is what + SPLIT's purification consumes; "full"/"multi" change behaviour materially — expose only if + the purification path is adapted to accept them. + + Exposing any of these needs: add to `config.vsh.yaml arguments:`, thread `par[...]` into the + matching `SPLIT::purify(...)` / `run.RCTD(...)` call in `script.R`, then `viash ns build` + a + container rebuild (see `check-component`) before a build/main Nebius run would honour them. + +## Wiring + +- Registered in the run_benchmark workflow: dep list + `src/workflows/run_benchmark/config.vsh.yaml` L186; present in the `--expression_correction_methods` + default string L107 (`no_correction:resolvi_correction:split`). +- Sweep: `scripts/run_benchmark/param_sweep/split_params.yaml` (committed params) + + `scripts/run_benchmark/param_sweep/run_test_split_nebius.sh` (Nebius launcher, standard CPU + compute env `5hfmdCBxMRd4nHZaJKYEQZ`, labels `task_ist_preprocessing,test,split` — no `gpu`). + The launch keeps `no_correction` enabled alongside `split` so the sweep is scored against the + uncorrected baseline; every other stage stays on its single default. + +## Risk points / gotchas + +- **Submittable-now constraint:** the Nebius run uses `--revision build/main`, so the sweep can + only vary args already in the build/main container (the six thresholds). Any newly-exposed + Tier-3 arg is silently ignored until rebuilt. +- **Zero-UMI reference-cell guard (script.R L56-60) is load-bearing** — without it small-panel + references abort RCTD with "fewer than 10 DE genes" (memory: `rctd-split-zero-umi-reference-nan`). +- **High-threshold sweep variants can legitimately fail** — pushing a threshold back to the + spacexr default on a very small panel can drop under the 10-DE-gene floor and abort inside + `create.RCTD`. That failure boundary is part of what the sweep measures, not a bug. +- **SPLIT installed from HEAD** (`remotes::install_github("bdsc-tds/SPLIT")`, unpinned) — the + `purify(counts=, rctd=, ...)` convenience signature the script relies on tracks whatever is on + the default branch; a breaking upstream API change would surface only at container rebuild. diff --git a/src/methods_segmentation/binning/NOTES.md b/src/methods_segmentation/binning/NOTES.md new file mode 100644 index 000000000..646fd3538 --- /dev/null +++ b/src/methods_segmentation/binning/NOTES.md @@ -0,0 +1,201 @@ +# Binning segmentation — implementation notes + +Reference for anyone (human or Claude) working on this component. Binning is the +deliberate **"poor segmentation" baseline** of the iST preprocessing benchmark: it +does not look at the image at all — it lays a fixed square grid over the image +extent and calls each tile a "cell." The only knob is the tile (bin) edge length. + +Links: docs/repo https://github.com/openproblems-bio/task_ist_preprocessing. + +**Citation caveat:** the config's `references.doi` is currently +`10.1101/2023.02.13.528102` = Marco Salas et al., *"Optimizing Xenium In Situ data +utility by quality assessment and best practice analysis workflows"* (bioRxiv 2023, +Nilsson lab; theislab co-authors incl. Kuemmerle, Tiesmeyer, Luecken, Theis). That +is the **txsim-associated Xenium best-practices / benchmarking paper** that +benchmarks segmentation tools — an appropriate framework citation for a txsim +segmentation baseline (binning itself has no dedicated publication; it's a trivial +grid). This DOI is **fine as-is**. Note it was *corrected*: the introducing commit +`dd3dff390` had `10.1101/2024.11.07.622434`, which is an **unrelated** single-author +paper ("Integrative cell bin segmentation ... by Voronoi", Ming Lin, Nottingham) — +a wrong DOI, already fixed to the current one. + +## What this component is + +A method at the **segmentation** stage (`src/methods_segmentation/`), implementing +the standard API `src/api/comp_method_segmentation.yaml`: `raw_ist.zarr` in → +`segmentation.zarr` out (a 2D cell **label map** + a stub `table` carrying +`cell_id`/`region`). + +Unlike every other segmentation method here (cellpose, cellposev4, stardist, +watershed), binning **ignores pixel content**. It only uses the image's **shape** +(and, when tuning in microns, its coordinate transform) to tile the field of view +into `bin_size × bin_size` squares. Transcripts are then assigned to whichever tile +they fall in by the downstream transcript-assignment stage, so each square becomes a +"pseudo-cell." This is intentionally a lower bound on segmentation quality — it +captures no cell morphology, splits and merges real cells arbitrarily. + +The heavy lifting is a two-line txsim call, `txsim.preprocessing.segment_binning`. + +## script.py — step by step + +1. **Build hyperparameters** (`:61-65`) — `par.copy()`, map the literal string + `"None"` → `None`, drop `input`/`output`. Leaves `bin_size` (and `bin_size_um`). +2. **Read input** (`:67`) — `sd.read_zarr(par["input"])`. +3. **Pull image + transform** (`:70-71`) — `sdata['image']['scale0'].image` → + `.compute().to_numpy()` for the full-res `(c, y, x)` array, and `.transform.copy()` + for the `scale0` coordinate transform (reattached to the output label map so it + lands in the same coordinate space as the transcripts). +4. **Resolve bin size** (`:73-82`) — if `bin_size_um` is set, convert microns → + pixels via `pixel_size_um(...)` (see below) and `round`; else use `bin_size` + (raw pixels) directly. +5. **Tile** (`:84`) — `tx.preprocessing.segment_binning(image[0], bin_size)`. + `image[0]` is the first channel (a 2D `(y, x)` plane); only its **shape** is + used. The txsim function is literally: + ```python + x = np.floor(np.mgrid[0:n, 0:m][0] / bin_size) # row // bin_size + y = np.floor(np.mgrid[0:n, 0:m][1] / bin_size) # col // bin_size + bins = x*(np.ceil(m/bin_size)) + y + 1 # unique 1-based label per tile + ``` + → a full-coverage label map: every pixel belongs to exactly one square, labels + are `1..N`, no background (label 0). So there are **no gaps** — every transcript + lands in some pseudo-cell. +6. **Downcast + parse** (`:85-88`) — `convert_to_lower_dtype` shrinks the label + array to the smallest uint dtype that holds `max label`; wrap as an + `xarray.DataArray(dims=('y','x'))`, `Labels2DModel.parse` with the copied + transform, store as `labels['segmentation']`. +7. **Carry the metadata table** (`:89-108`) — copies a stub `table` from the input's + `tables["metadata"]`, resolving the downstream-required `cell_id` in priority + order: `obs["cell_id"]` → the table's `instance_key` column + (`uns["spatialdata_attrs"]["instance_key"]`) → the obs index; optional `region` + carried if present. **This block is duplicated in `cellpose/`, + `cellposev4/` and other segmentation scripts** (the instance_key fallback was + added for the Xenium WTA preview behind the Atera dataset). Keep the copies in + sync. Note the table describes the *reference* cells from metadata, not the + binning tiles — downstream counting/aggregation turns the label map into the + actual cell × gene matrix. +8. **Write** (`:110-113`) — `rmtree` any existing output, then `sd_output.write`. + +### `pixel_size_um` helper (`:28-49`) — added this tuning pass + +Reads microns-per-pixel from the image element's coordinate transform: +`element.transform` (dict of coord_system → transformation) → pick `"global"` (fall +back to the first) → `to_affine_matrix(('y','x'),('y','x'))` → average of the +`|diagonal|` scale magnitudes. On failure it prints a warning and returns **1.0** +(the standardized grid — see below), so a micron sweep still produces correct +results on current data even if the transform can't be read. + +## The bin_size unit gotcha (load-bearing) + +`bin_size` is in **PIXELS of the `image` element**, not microns — the txsim +function divides pixel indices by it. The physical size of a bin therefore depends +on the image's pixel size. + +The saving grace: this benchmark's `raw_ist.zarr` **images are rasterized onto a +standardized ~1 µm/pixel grid** (`target_unit_to_pixels: 1` in the loaders, e.g. +`src/datasets/loaders/vizgen_merscope/script.py:245`). Verified directly on the +mouse-brain Xenium test data: the `image` `scale0 → global` transform is +identity-scale + a translation (scale `[1,1,1]`), i.e. **1 px = 1 µm**, and s0 is +`[1, 1000, 1000]` (a 1 mm² crop). So on the standardized grid **`bin_size` in pixels +≈ bin size in microns**. (Contrast the *raw* Xenium morphology at ~0.2125 µm/px — +that resolution is gone by the time binning sees the rasterized `image`.) + +`--bin_size_um` (added this pass, opt-in) makes this explicit and robust: give a +physical edge length and the script converts to pixels using the actual transform, +so bins are comparable even if a future dataset is gridded at a different resolution. + +## config.vsh.yaml — setup + +- Base image `openproblems/base_python:1`; merges `setup_txsim_partial.yaml` + (`theislab/txsim@dev`, for `segment_binning`) + `setup_spatialdata_partial.yaml` + (`spatialdata>=0.7.3`, `anndata>=0.12.0`, `zarr>=3.0.0`). No GPU, no model + download — nothing exotic; it merges the base setups and works. +- Nextflow directives: `[ midtime, midcpu, midmem ]` (4 h / 15 CPU / 50 GB). This is + generous — the actual work is one `np.mgrid` over the image and a write; the cost + is dominated by reading the image and writing the label zarr. CPU-only. + +## Arguments + +| arg | type | default | maps to | +|---|---|---|---| +| `--bin_size` | integer | `30` | tile edge length in **pixels** → `segment_binning(img, bin_size)` | +| `--bin_size_um` | double | *(unset)* | tile edge length in **microns**; when set, overrides `--bin_size` after µm→px conversion | + +`--bin_size_um` is **newly exposed this pass** — it needs `viash ns build` + a +binning **container rebuild** before a run can use it (see check-component). Default +behaviour (bin_size_um unset → use bin_size pixels) is byte-identical to before. + +## Optimization / tuning + +The only lever is the **bin edge length**. There is genuinely nothing else — the +underlying `segment_binning` takes exactly `(img, bin_size)`. + +**Tier 0 — the input.** Binning uses only the image *extent* and pixel size, not +its content, so image channel/stain choice is irrelevant here (the opposite of the +real segmenters). What matters is the **pixel size** of the grid, which sets what a +given `bin_size` means physically — handled by `--bin_size_um` (see gotcha above). + +**Tier 1 — `bin_size` / `bin_size_um` (the whole knob).** Sets the pseudo-cell +size. Grounded in the ~1 µm/px grid and brain/Xenium cell scale (nuclei ~5–8 µm, +whole cells ~10–15 µm): +- too **small** (≈ 10 µm) → many bins per cell → over-segmentation, transcripts of + one cell split across tiles; +- ≈ **15 µm** → roughly one whole cell per bin → the best a blind grid can do; +- default **30 µm** → 2–3 cells per bin → coarse; +- too **large** (≈ 40–50 µm) → strong under-segmentation, several cells merged. +Because binning is a *baseline*, the point of the sweep is to bracket this from +sub-cellular to super-cellular, not to find a "great" value — it shows how much the +downstream benchmark score degrades as pseudo-cells drift from real cell size. + +**Tier 3 — worth adding for a serious study:** none beyond the micron knob already +added. (If ever needed: non-square / hexagonal binning, or an offset/anchor for the +grid origin, would require changes to `segment_binning` upstream in txsim, not just +this component.) + +**Recommended sweep** (see `scripts/run_benchmark/binning_params.yaml`): sweep +`bin_size_um` over `[10, 15, 20, 40, 50]` around the `30` default → 6 variants (a +"star" around the default, one value at a time). Light, as befits a one-knob +baseline. + +## Risk points / gotchas + +- **`bin_size` is pixels, not microns** — the whole "unit gotcha" section. Safe only + because the benchmark grid is standardized to ~1 µm/px; use `--bin_size_um` to be + explicit / dataset-robust. +- **`--bin_size_um` is not yet runtime-validated.** It's the one non-trivial new + code path (reads the transform, converts). The µm→px math and the + `pixel_size_um` transform read need a first real run to confirm (config parses + clean via `viash config view`; the default path is unchanged). The 1.0-µm/px + fallback means a bad transform read degrades to "treat µm as px," which is correct + on current data. +- **Full coverage, no background.** Every pixel gets a positive label — there is no + label 0 / background. Downstream aggregation therefore counts *all* transcripts + into some pseudo-cell; there are no "unassigned" transcripts from the grid itself. +- **Whole image loaded into RAM** (`:70`, `.compute().to_numpy()`), like the other + segmenters — the reason for `midmem` despite the trivial compute. +- **Shared metadata/cell_id block** (`:89-108`) is duplicated across segmentation + scripts; fix together. +- **Only channel 0's shape** is used (`image[0]`); irrelevant here since content is + ignored, but note it if the image is ever multi-channel with differing shapes. + +## Wiring + +- `src/workflows/run_benchmark/config.vsh.yaml`: dependency + `methods_segmentation/binning` (`:160`), in the default segmentation string + `custom_segmentation:cellpose:cellposev4:binning:stardist:watershed` (`:65`), and + it is the worked example in the `--method_parameters_yaml` doc (`:125`). +- `src/workflows/run_benchmark/main.nf:99`: imported. +- Run scripts: enabled in `run_test_local.sh`, `run_full_local.sh`, + `run_full_nebius.sh`, `run_mpii_nebius.sh`, `run_test_seqeracloud.sh`, + `run_full_seqeracloud.sh`; commented in `run_test_nebius.sh` (the canonical + params-file placement example). Dedicated sweep runners added this pass: + `run_test_binning_local.sh` / `run_test_binning_nebius.sh` + the committed + `binning_params.yaml`. + +## References + +- **Config DOI (appropriate):** Marco Salas S. et al., "Optimizing Xenium In Situ + data utility by quality assessment and best practice analysis workflows", bioRxiv + **10.1101/2023.02.13.528102** (2023) — the txsim-associated Xenium + best-practices/benchmarking paper. +- txsim (`theislab/txsim@dev`) — `segment_binning` in + `txsim/preprocessing/_segmentation.py`. diff --git a/src/methods_segmentation/cellpose/NOTES.md b/src/methods_segmentation/cellpose/NOTES.md new file mode 100644 index 000000000..504b98a7d --- /dev/null +++ b/src/methods_segmentation/cellpose/NOTES.md @@ -0,0 +1,316 @@ +# Cellpose (v3, cyto/nuclei CNN) segmentation — implementation notes + +Reference for anyone (human or Claude) working on this component. Captures how it +works, why it goes through the `txsim` wrapper, why it's a *separate* component +from `cellposev4`, what its defaults really are (they are **not** speed-tuned — +see below), and where the tuning levers are. + +Links: config `links.documentation`/`links.repository` both point at this task's +own repo (not upstream). Upstream Cellpose: docs +https://cellpose.readthedocs.io/en/latest/ · repo +https://github.com/MouseLand/cellpose. + +**Citation caveat (milder than cellposev4's).** The config's `references.doi` is +`10.1038/s41592-020-01018-x` = "Cellpose: a generalist algorithm for cellular +segmentation" (Stringer et al. 2021, Nat. Methods) — the original flow-dynamics +CNN. This component's **default model is `cyto`, which IS that 2021 model**, so the +DOI is appropriate for the shipped default (unlike `cellposev4`, whose DOI pointed +at the wrong paper). Caveat only: the pinned library is Cellpose **3.x**, and the +`--model_type` sweep can select `cyto2` (Cellpose 2.0, Pachitariu & Stringer 2022, +`10.1038/s41592-022-01663-4`) and `cyto3` (Cellpose3, Stringer & Pachitariu 2025, +`10.1038/s41592-025-02595-5`), which come from later papers. If the benchmark +settles on cyto2/cyto3, add those DOIs to the `references` list. + +## What this component is + +One of the methods at the **segmentation** stage of the iST preprocessing +benchmark (`src/methods_segmentation/`). It implements the standard API +`src/api/comp_method_segmentation.yaml`: `raw_ist.zarr` in → `segmentation.zarr` +out (a 2D cell **label map** + a stub `table`). + +It runs **Cellpose v3** (`cellpose<4.0.0`, currently resolves to 3.1.1.x) via the +**txsim wrapper** `txsim.preprocessing.segment_cellpose`, not by calling cellpose +directly. That indirection is the single most important thing to understand about +this component — see the "txsim wrapper" section. The default model is `cyto` +(the original CNN cytoplasm model). + +### Why it exists separately from `cellposev4` + +There are **two** Cellpose components in this repo, and the split is deliberate. +(This table is the mirror of the one in `cellposev4/NOTES.md`.) + +| | `cellpose/` (this one) | `cellposev4/` | +|---|---|---| +| cellpose version | `cellpose<4.0.0` (pinned) | `cellpose>=4.0.0` | +| model | `cyto` (CNN) via `models.Cellpose` | `cpsam` (ViT/SAM) via `CellposeModel` | +| call path | `txsim.preprocessing.segment_cellpose(...)` wrapper | direct `model.eval(...)` | +| extra deps | needs `txsim` | no `txsim` (leaner image) | +| arg surface | ~18 args (channels, model_type, invert, do_3D, …) | 6 args (diameter, flow_threshold, cellprob_threshold, niter, min_size, resample) | +| defaults | == Cellpose library defaults (quality) | deliberately **speed-tuned** | + +History (git `-- src/methods_segmentation/cellpose/`): +- `5487edcfa` "cellpose added" (2024-11) — original component; ~17 args, `base_python`, + txsim wrapper, `models.Cellpose` (v3 API). +- `35061648c` "Support cellpose v4" (2025-06) — reshuffled args to be v4-compatible + (moved `resample`, commented out `interp`/`net_avg`, note "diameter should be None + with v4"). This is why some args are commented out in the config today. +- `8015045ab` "Use cellpose v3 instead of v4 to stay in test time limits" — pinned + `cellpose<4.0.0`, because the v4 SAM/ViT model was **too slow** to finish inside + the CI/`viash test` budget. This component stayed v3; `cellposev4` (#166) was added + as the heavier GPU sibling. +- `7e850cc84` "Add gpu for cellpose" + `702ba3fd2` — added the `gpu` label and moved + to `base_pytorch_nvidia:1`. **But see the GPU gotcha below** — the txsim v3 path + does not actually use the GPU. +- `31d57ec1c` "Add method selection arguments" — merged `setup_spatialdata_partial`. + +## How Cellpose v3 works (background) + +Cellpose predicts, per pixel, a **flow vector** toward the center of the cell that +pixel belongs to, plus a **cell probability**. At inference each pixel "follows the +flow" for `niter` iterations to a fixed point; pixels converging to the same point +become one **mask**. This flow-dynamics core is shared across Cellpose 1–4. + +v3 exposes two model classes (both still present in `cellpose<4`): +- **`models.Cellpose`** — bundles a `SizeModel` (auto-diameter estimator) with a + `CellposeModel`. Constructor default `gpu=False`, `model_type="cyto3"`. This is + what the txsim wrapper uses. +- **`models.CellposeModel`** — the mask model alone (what `cellposev4` uses). + +Builtin models (`MODEL_NAMES`, cellpose 3.1.1.1): `cyto3`, `nuclei`, `cyto2`, +`cyto` (+ specialist `*_cp3`, tissuenet, livecell, bacterial, transformer models). +`cyto`/`nuclei`/`cyto2`/`cyto3` each have a paired size model, so auto-diameter +works for them. `diam_mean` is 30 px for cyto* models, 17 px for `nuclei`. + +## txsim `segment_cellpose` wrapper — the key indirection + +`script.py` calls `tx.preprocessing.segment_cellpose(image[0], hyperparameters)`. +The wrapper lives in `theislab/txsim@dev` +(`txsim/preprocessing/_segmentation.py`, function `segment_cellpose`). For +`cellpose<4` it does, in order: + +1. Reads `hyperparams["model_type"]` (defaults to `'nuclei'` if absent — but this + component always passes `cyto`). +2. `model = models.Cellpose(model_type=model_type)` — **note `gpu=` is NOT passed**, + so it defaults to `gpu=False`. See the GPU gotcha. +3. `del hyperparams["model_type"]` — the model_type key is consumed here and does + **not** reach `eval`. +4. `res, _, _, _ = model.eval(img, channels=[0, 0], **hyperparams)` — + **`channels=[0,0]` is hardcoded** (segment `img` as a single grayscale plane, no + separate nuclear channel). Every remaining config arg is forwarded as a kwarg. + +So the contract is: **all config args except `model_type` are forwarded verbatim +into `Cellpose.eval`** (which passes the non-explicit ones through `**kwargs` into +`CellposeModel.eval`). Adding a new config arg whose name matches an `eval` kwarg +is therefore enough to wire it — no script change needed (that is exactly how the +newly added `--niter` works). + +## script.py — step by step + +1. **Build hyperparameters** (`:34-38`) — `hyperparameters = par.copy()`, then + `{k:(v if v != "None" else None)}` converts the string `"None"` (how the + string-typed args `channel_axis`, `z_axis`, `rescale`, `anisotropy` carry "unset") + back to Python `None`, then `del`s `input`/`output`. Everything else — all the + segmentation knobs — stays in the dict. +2. **Read input** (`:40-43`) — `sd.read_zarr(par["input"])`; + `sdata['image']['scale0'].image.compute().to_numpy()` pulls the full-res image + into RAM; the `scale0` transform is copied to reattach to the output. +3. **Segment** (`:44`) — `tx.preprocessing.segment_cellpose(image[0], hyperparameters)`. + `image[0]` = first channel of a `(c, y, x)` array → a single 2D plane. Returns + the label array. +4. **Downcast + parse** (`:45-48`) — `convert_to_lower_dtype` shrinks the label array + to the smallest uint that holds `max label`; wrap as `xarray.DataArray(dims=('y','x'))`, + `Labels2DModel.parse` with the copied transform, store as + `sd_output.labels['segmentation']`. +5. **Carry the metadata table** (`:50-69`) — copies a stub `table` from the input's + `tables["metadata"]`. Downstream needs a `cell_id` column; resolved in priority + order: explicit `obs["cell_id"]` → the table's `instance_key` column + (`uns["spatialdata_attrs"]["instance_key"]`) → the obs index. Optional `region` + column carried if present. **This block is duplicated verbatim in + `cellposev4/script.py`** — the instance_key fallback was added for exports (e.g. + the Xenium WTA preview behind the Atera dataset) that lack an explicit `cell_id`. + Keep the two copies in sync. +6. **Write** (`:71-74`) — `rmtree` any existing output, then `sd_output.write`. + +Note: the output `table` describes the *reference* cells from metadata, not the +newly predicted masks — segmentation output here is the **label image**; the +counting/aggregation stage downstream turns masks into a cell × gene table. + +## config.vsh.yaml — the Docker setup + +- Base image `openproblems/base_pytorch_nvidia:1` (CUDA/torch), GPU-capable — but + the v3 code path does not use the GPU (gotcha below). +- `setup`: + - `pypi: cellpose<4.0.0` — the version pin that makes this "v3". + - `docker.run`: a **build-time model download** — instantiates + `models.Cellpose(gpu=False, model_type=m)` for `m in ['cyto','nuclei','cyto2','cyto3']`, + which caches each mask model **and its size model** into the image. This was + extended (this tuning session) from caching only `cyto` to caching all four, + because the `--model_type` sweep selects among them and several concurrent tasks + fetching weights caused intermittent `HTTP 504` from the weight server. + - Merges `/src/base/setup_txsim_partial.yaml` (txsim + squidpy + rasterio) **and** + `/src/base/setup_spatialdata_partial.yaml`. Unlike `cellposev4`, this one needs + txsim (for the wrapper). +- Nextflow directives: `[ midtime, midcpu, highmem, gpuhighmem ]` — 4 h / 15 CPU / + 100 GB / high-mem GPU. The `gpuhighmem` label schedules onto a GPU node, but see + the GPU gotcha — the GPU sits idle for the v3 path. + +## Arguments + +The forwarded `Cellpose.eval` / `CellposeModel.eval` args, their config defaults, +and how they compare to Cellpose 3.1.1.1's own defaults. **Almost every default +here already equals Cellpose's library default** — the only deviation is +`model_type` (`cyto` vs library `cyto3`). So, unlike `cellposev4`, this component +is **not** shipped with speed-tuned defaults; it is essentially "stock Cellpose." + +| arg | config default | Cellpose default | notes | +|---|---|---|---| +| `--model_type` | `cyto` | `cyto3` | selects the model; consumed by the wrapper, not passed to eval. `cyto`/`nuclei`/`cyto2`/`cyto3`. **deviation from library default** | +| `--diameter` | `30.0` | `30.` (class) | fixed size assumption. `0`/`None` → auto-estimate via SizeModel (slower). biggest lever | +| `--flow_threshold` | `0.4` | `0.4` | flow-error QC; higher = keep more (more permissive), lower = stricter shape filter | +| `--cellprob_threshold` | `0.0` | `0.0` | per-pixel logit gate (~−6…+6); lower = more/dimmer cells (recall), higher = fewer (precision) | +| `--min_size` | `15` | `15` | drop ROIs below N px; `0` keeps specks | +| `--resample` | `True` | `True` | run dynamics at full resolution (smoother, slower). already the quality setting | +| `--normalize` | `True` | `True` | 1/99-percentile normalization per channel | +| `--niter` | `0` (auto) | `None` (auto) | **newly exposed.** dynamics iterations; 0/None → ∝ diameter (~200 at resample). lower for speed | +| `--augment` | `False` | `False` | test-time augmentation (TTA); quality at ~4–8× cost | +| `--batch_size` | `8` | `8` | tiles per forward pass | +| `--tile_overlap` | `0.1` | `0.1` | tile overlap fraction | +| `--invert` | `False` | `False` | invert intensities before running | +| `--rescale` | `None` | `None` | manual resize factor (only used if diameter is None) | +| `--do_3D` | `False` | `False` | 3D segmentation (input here is 2D) | +| `--anisotropy` | `None` | `None` | 3D only | +| `--stitch_threshold` | `0.0` | `0.0` | 3D stitching of 2D masks | +| `--channel_axis` | `None` | `None` | auto-detected; input is a single 2D plane | +| `--z_axis` | `None` | `None` | 3D only | + +Commented-out in the config: `--net_avg` (removed in cellpose 2.2+), `--tile` +(deprecated), `--interp` (valid in v3 — default `True` — but commented out during +the abandoned v4-compat pass; low sweep value since flipping it off only degrades +2D dynamics). + +## Optimization / tuning + +Two things to understand before tuning (both differ from the `cellposev4` story): + +1. **Defaults are already Cellpose's own defaults, i.e. quality-oriented** (flow QC + on at 0.4, min-mask filtering on at 15, resample on, normalize on). The only + non-stock default is `model_type=cyto`. So "optimize for quality" here is **not** + "walk speed-tuned defaults back to library defaults" (that was the v4 job) — it's + *exploring the model choice, the object scale, and the recall/precision dials*. +2. Grounded in the cellpose 3.1.1.1 source (`models.py`, `dynamics.py`) and docs, + not memory. Ranges tied to Xenium morphology (~0.2125 µm/px). + +**Tier 0 — the input, not a parameter.** The wrapper hardcodes `channels=[0,0]` and +the script feeds only `image[0]` — a **single grayscale plane**. iST morphology is +effectively single-channel here, but if a dataset carried a membrane/boundary stain +alongside the nuclear one, feeding both (via `channels=[cyto_ch, nuc_ch]`) would +improve whole-cell boundaries more than any threshold tweak. Not wired up (would +require changing the hardcoded `channels` in txsim, which is out of this repo). + +**Tier 1 — highest impact on quality** + +- **`model_type`** *(default `cyto`; library default `cyto3`)*. The most + distinctive v3 lever (v4 has no model choice). On a single grayscale morphology + channel the model matters a lot: `nuclei` for a nuclear-dominant stain, `cyto2` + (Cellpose 2.0) and `cyto3` (Cellpose3 generalist super-model) as improved + generalists over the 2021 `cyto`. Sweep `nuclei`/`cyto2`/`cyto3`. +- **`diameter`** *(default 30.0; `0`/`None` = auto)*. Cellpose rescales so objects + land near the model's `diam_mean` (30 px cyto, 17 px nuclei); a fixed 30 assumes + ~30 px objects. Xenium (~0.2125 µm/px): an ~8 µm nucleus ≈ 38 px, a whole cell + ≈ 60–70 px, so 30 can be materially wrong. Sweep `0` (auto), ~40 (nucleus scale), + ~60 (whole-cell scale). +- **`cellprob_threshold`** *(default 0.0, range −6…+6)*. Pure recall↔precision dial, + no speed cost. Lower (−1…−2) recovers more/dimmer cells; raise (+1…+2) suppresses + dim detections. In iST you usually want all cells → this is the most direct dial. +- **`flow_threshold`** *(default 0.4)*. Already at the Cellpose default (QC on). + Raising it (0.6/0.8) keeps cells with higher flow error (more, possibly + ill-shaped); lowering it (0.2) is a stricter shape filter. + +**Tier 2 — quality/speed trade-offs** + +- **`niter`** *(newly exposed; default 0 = auto ≈ 200 at resample)*. Lower (50) is + faster — meaningful because the v3 path is CPU-bound (see gotcha); raise for + large/elongated cells that under-converge. +- **`resample`** *(default True = quality setting)*. Only non-default is `False` + (dynamics on the downsampled grid → faster, coarser boundaries). Include to + characterize the speed cost of the current default. +- **`min_size`** *(default 15)*. `0` keeps small specks (higher recall on tiny + cells); larger (50+) drops debris (higher precision). +- **`augment`** *(default False)*. TTA — the accuracy ceiling at ~4–8× cost; rarely + worth it in a benchmark, include one `True` to bound the gain. +- **`normalize`** *(default True)*. Flip to `False` to see intensity sensitivity; + usually worse for uneven-illumination iST images. + +**Tier 3 — not exposed, worth adding for a serious sweep** + +- **`normalize` as a dict** *(percentiles, `tile_norm`, `sharpen`)*. cellpose v3 + accepts a normalization dict (see `models.normalize_default`). For iST images with + uneven illumination, tile-wise normalization / sharpening can matter; the boolean + `--normalize` can't express it. Would need a JSON/dict-typed arg. +- **`bsize`** *(224)* / **`max_size_fraction`** *(0.4)* — throughput / large-mask + culling; low value for a first sweep. + +**Suggested quality-first order:** `model_type` (nuclei/cyto2/cyto3) → `diameter` +(auto or data scale) → `cellprob_threshold` (−2…+2) → `flow_threshold` → +`min_size`. The current sweep (`scripts/run_benchmark/cellpose_params.yaml`) +covers all of these; **20 variants** total (1 default + 19). + +## Risk points / gotchas + +- **GPU is scheduled but NOT used (v3 path).** The txsim wrapper calls + `models.Cellpose(model_type=...)` without `gpu=`, so it defaults to `gpu=False` and + runs on **CPU** — even though the config uses `base_pytorch_nvidia` and the + `gpuhighmem`/`gpu` label puts it on a GPU node. As of `txsim@dev` at time of + writing, this wastes the GPU allocation and makes runs slow. Fixing it properly + means either patching txsim to pass `gpu=core.use_gpu()` (as the v4 branch of the + same wrapper does) or dropping the GPU label. Left for the user — do not silently + flip infra labels. This is also why the `--niter` speed knob matters here. +- **`--model_type` beyond `cyto` triggers weight downloads unless pre-cached.** The + image now pre-caches `cyto`/`nuclei`/`cyto2`/`cyto3` (config `docker.run`), so the + sweep is safe; if you add another model to the sweep, add it to that list too or + concurrent tasks may hit `HTTP 504` from the weight server. +- **Whole image loaded into RAM** (`:42`) — `.compute().to_numpy()` on `scale0` is + the full-res plane; big panels are why the label is `highmem`. +- **Only channel 0 is segmented, as grayscale** (`image[0]` + hardcoded + `channels=[0,0]`). Any additional stain channels are ignored. +- **`model_type` is consumed by the wrapper, not passed to eval.** If you rename or + drop it, the wrapper falls back to `'nuclei'` — a silent behaviour change. +- **String "None" convention.** `channel_axis`/`z_axis`/`rescale`/`anisotropy` are + typed `string` with default `"None"`; the script converts the literal `"None"` → + Python `None`. A new numeric knob that needs an "auto/unset" sentinel should reuse + a real numeric sentinel the tool understands (as `--niter 0` does), not the string + hack, or it will be forwarded as a string and break eval. +- **Shared metadata block** (`:50-69`) is duplicated in `cellposev4/script.py`; fix + both together. +- **Don't confuse the classes.** This component uses `models.Cellpose` (with + SizeModel); `cellposev4` uses `models.CellposeModel`. `Cellpose` still exists in + v3 but was removed in v4. + +## Wiring + +- `src/workflows/run_benchmark/config.vsh.yaml`: listed as a dependency + (`methods_segmentation/cellpose`, L158) and included in the default segmentation + method string `custom_segmentation:cellpose:cellposev4:binning:stardist:watershed` + (L65). +- `src/workflows/run_benchmark/main.nf:97`: imported alongside `cellposev4`. +- Run scripts: enabled in `run_full_local.sh`, `run_full_nebius.sh`, + `run_full_seqeracloud.sh`, `run_gpu_nebius.sh`, `run_mpii_nebius.sh`, + `run_test_seqeracloud.sh`; commented out in `run_test_local.sh` / + `run_test_nebius.sh`. Dedicated sweep runners added this session: + `scripts/run_benchmark/run_test_cellpose_local.sh` and `_nebius.sh`, driven by the + committed `scripts/run_benchmark/cellpose_params.yaml`. + +## References + +- Docs: https://cellpose.readthedocs.io/en/latest/ (settings + api pages have the + `model.eval` parameter reference). +- Repo: https://github.com/MouseLand/cellpose (defaults verified against tag + `v3.1.1.1`, `cellpose/models.py` + `cellpose/dynamics.py`). +- txsim wrapper: `theislab/txsim@dev`, `txsim/preprocessing/_segmentation.py`, + `segment_cellpose`. +- **Original Cellpose (`cyto`, the config DOI):** Stringer, Wang, Michaelos, + Pachitariu (2021), Nat. Methods, **10.1038/s41592-020-01018-x**. +- **Cellpose 2.0 (`cyto2`):** Pachitariu & Stringer (2022), Nat. Methods, + **10.1038/s41592-022-01663-4**. +- **Cellpose3 (`cyto3`, image restoration):** Stringer & Pachitariu (2025), Nat. + Methods, **10.1038/s41592-025-02595-5**. diff --git a/src/methods_segmentation/cellposev4/NOTES.md b/src/methods_segmentation/cellposev4/NOTES.md new file mode 100644 index 000000000..145be5a16 --- /dev/null +++ b/src/methods_segmentation/cellposev4/NOTES.md @@ -0,0 +1,291 @@ +# Cellpose 4 (Cellpose-SAM) segmentation — implementation notes + +Reference for anyone (human or Claude) working on this component. Captures how it +works, why it's a *separate* component from `cellpose`, what Cellpose 4 actually +is, and where the settings come from. + +Links: docs https://cellpose.readthedocs.io/en/latest/ · repo +https://github.com/MouseLand/cellpose. + +**Citation caveat:** the config's `references.doi` is `10.1038/s41592-020-01018-x`, +which is the **original Cellpose paper** (Stringer et al. 2021, Nat. Methods) — the +flow-dynamics CNN, *not* the model this component runs. The model actually used +(Cellpose-SAM / `cpsam`) is described in a **different** paper (Pachitariu, +Rariden & Stringer, bioRxiv `10.1101/2025.04.28.651001`, 2025). The config DOI is +the general Cellpose reference, not the v4-specific one; treat it accordingly. + +## What this component is + +One of the methods at the **segmentation** stage of the iST preprocessing +benchmark (`src/methods_segmentation/`). It implements the standard API +`src/api/comp_method_segmentation.yaml`: `raw_ist.zarr` in → `segmentation.zarr` +out (a 2D cell **label map** + a stub `table`). + +It runs **Cellpose 4** (a.k.a. **Cellpose-SAM**, the `cpsam` model) directly via +`cellpose.models.CellposeModel` + `model.eval`. Adapted from +`openproblems-bio/task_spatial_segmentation` (the sibling OpenProblems task) — +the two scripts are nearly identical. + +### Why it exists separately from `cellpose` + +There are **two** Cellpose components in this repo, and this split is deliberate: + +| | `cellpose/` | `cellposev4/` (this one) | +|---|---|---| +| cellpose version | `cellpose<4.0.0` (pinned) | `cellpose>=4.0.0` | +| model | `cyto` (CNN) via `Cellpose` | `cpsam` (ViT/SAM) via `CellposeModel` | +| call path | `txsim.preprocessing.segment_cellpose(...)` wrapper | direct `model.eval(...)` | +| extra deps | needs `txsim` | no `txsim` (leaner image) | +| arg surface | ~17 args (channels, model_type, invert, do_3D, …) | 5 args (diameter, flow_threshold, niter, min_size, resample) | + +History (git): +- `35061648c` "Support cellpose v4" — first attempt to make the *original* + `cellpose` component v4-compatible (dropped `interp`, moved `resample`, noted + `diameter` "should be None with v4"). +- `8015045ab` "Use cellpose v3 instead of v4 to stay in test time limits" — + reverted: pinned `cellpose<4.0.0`, because the v4 SAM/ViT model was **too slow** + to finish inside the CI/`viash test` time budget on a lean/CPU runner. +- `041ab3e23` "Cellposev4 (#166)" — instead of forcing one component to straddle + both, added **this** dedicated `cellposev4` component with a speed-tuned default + param set (see Arguments) and registered it in the benchmark. + +So `cellpose` stays as the fast-enough v3 CNN baseline; `cellposev4` is the +heavier, more-generalizing transformer model run on GPU. + +## What Cellpose 4 / Cellpose-SAM is (background) + +Cellpose predicts, per pixel, a **flow vector** pointing toward the center of the +cell that pixel belongs to, plus a **cell probability**. At inference each pixel +"follows the flow" for a number of iterations to a fixed point; pixels converging +to the same point become one **mask**. (This flow-dynamics core is unchanged from +Cellpose 1–3.) + +**What's new in v4 (Cellpose-SAM, `cpsam`)** — from the paper itself (Pachitariu, +Rariden & Stringer, "Cellpose-SAM: superhuman generalization for cellular +segmentation", bioRxiv `10.1101/2025.04.28.651001`, v1 2025-05-01, corresponding +author C. Stringer, HHMI Janelia; CC-BY-NC). Verified against the bioRxiv abstract: +- They **adapt the pretrained transformer backbone of a foundation model (SAM) + into the Cellpose framework** — i.e. keep Cellpose's flow-dynamics + mask-reconstruction, swap the U-Net CNN backbone for SAM's ViT. The flow → + fixed-point → mask machinery is unchanged from Cellpose 1–3. +- **Headline claim:** the model "substantially outperforms inter-human agreement + and approaches the human-consensus bound" — the paper frames prior methods as + matching inter-human agreement, and argues a human *consensus* could roughly + halve error rates; Cellpose-SAM pushes toward that. Hence "superhuman." +- Generalization robustness was **explicitly trained in**, to: **channel + shuffling** (order-invariance), **cell size**, **shot noise**, **downsampling**, + and **isotropic + anisotropic blur**. This is why in practice you don't need to + set `diameter` or pick channels. +- Positioned as a **foundation model** that drops into the existing Cellpose + ecosystem: finetuning, human-in-the-loop training, image restoration, and 3D + segmentation. + +Consequences in the library API (from the docs / release notes, consistent with +the paper's channel- and size-invariance): +- **Trained around a mean object diameter of 30 px** (range ~7.5–120 px) and + largely **size-invariant** → `diameter` is optional. +- **`channels` is gone** — uses the first ≤3 channels; no channel/model-type + selection. +- **API consolidated:** the old `models.Cellpose` class (which bundled a + `SizeModel` for auto-diameter) and `models.SizeModel` were **removed**; only + `models.CellposeModel` remains, and `CellposeModel()` loads `cpsam` by default. + This is why the script uses `CellposeModel`, not `Cellpose`. +- Newer 4.2+ point releases add `cpsam_v2` / DINOv3 (`cpdino`) variants, but this + component just pins `cellpose>=4.0.0` and takes whatever default ships. + +## script.py — step by step + +1. **Device pick** (`:11-13`) — `torch.device('cuda' if available else 'cpu')`, + printed. Runs on CPU if no GPU, but see the speed caveat below. +2. **Read input** (`:46-49`) — `sd.read_zarr(par["input"])`, then + `sdata['image']['scale0'].image.compute().to_numpy()` pulls the full-res image + into memory, and the `scale0` transform is copied to reattach to the output. +3. **Init model** (`:51-52`) — `CellposeModel(gpu=torch.cuda.is_available())`. + No `model_type`/`pretrained_model` given → loads the default `cpsam` weights. +4. **Build eval params** (`:54`) — collects the 6 tunables + (`diameter, flow_threshold, cellprob_threshold, niter, min_size, resample`) from + `par`, dropping any that are `None`. Note `min_size: -1`, `flow_threshold: 0.0` + and `cellprob_threshold: 0.0` are **passed through** (they're not `None`) — see + Arguments for what those values mean. +5. **Segment** (`:56`) — `model.eval(image[0], progress=True, **eval_params)`. + `image[0]` = first channel of a `(c, y, x)` array → a single 2D plane. Returns + `(masks, flows, styles)`; only `masks` is kept. +6. **Post-process** (`:58-65`) — `convert_to_lower_dtype` downcasts the label + array to the smallest uint that holds `max label` (uint8/16/32/64) to shrink the + zarr; wrap as an `xarray.DataArray(dims=('y','x'))`, `Labels2DModel.parse` with + the copied transform, store as `sd_output.labels['segmentation']`. +7. **Carry the metadata table** (`:67-86`) — copies a stub `table` from the input's + `tables["metadata"]`. Downstream needs a `cell_id` column; it's resolved in + priority order: explicit `obs["cell_id"]` → the table's `instance_key` column + (`uns["spatialdata_attrs"]["instance_key"]`) → the obs index. Optional `region` + column is carried if present. **This exact block is shared verbatim with + `cellpose/script.py`** — the instance_key fallback was added for exports (e.g. + the Xenium WTA preview behind the Atera dataset) that lack an explicit + `cell_id`. Keep the two copies in sync if you touch it. +8. **Write** (`:88-91`) — `rmtree` any existing output, then `sd_output.write`. + +Note: the output `table` describes the *reference* cells from metadata, not the +newly predicted masks — segmentation output here is the **label image**; the +counting/aggregation stage downstream is what turns masks into a cell × gene table. + +## config.vsh.yaml — the Docker setup + +- Base image `openproblems/base_pytorch_nvidia:1` (CUDA/torch) — GPU-capable. +- `setup`: + - `pypi: cellpose>=4.0.0` — the version pin that makes this "v4". + - `script: from cellpose.models import CellposeModel; model = CellposeModel()` — + a **build-time model download**. Instantiating `CellposeModel()` fetches the + `cpsam` weights into the image so runtime tasks never hit the cellpose model + server. (The v3 `cellpose` component does the same thing via a `docker run` + line, for the same reason: several concurrent segmentation tasks all fetching + weights caused intermittent `HTTP 504` from the weight server.) + - Merges `/src/base/setup_spatialdata_partial.yaml` (spatialdata stack). Unlike + `cellpose`, it does **not** merge `setup_txsim_partial.yaml` — no txsim needed. +- Nextflow directives: `[ midtime, midcpu, highmem, gpuhighmem ]` — 4 h / 15 CPU / + 100 GB / high-mem GPU. GPU label is what schedules it onto GPU nodes; on CPU the + SAM model is very slow (the reason v4 was pulled from the CPU test path). + +## Arguments (defaults are speed-tuned) + +The defaults deliberately trade a little accuracy for speed so the transformer +model finishes in budget: + +- `--diameter` (default **30.0**) — expected cell diameter in px. 30 == the model's + training mean, so it's effectively "no rescaling." Left unset, v4 would run a + slower size estimate; fixing it at 30 skips that. Bump it for genuinely large + cells (downsamples → faster + better convergence). +- `--flow_threshold` (default **0.0**) — flow-error QC threshold. Cellpose's own + default is 0.4. **Setting it to 0 disables the flow-consistency check**, which + skips a recompute step → faster, at the cost of possibly keeping some ill-shaped + ROIs. Raise toward 0.4 if you see bad masks. +- `--cellprob_threshold` (default **0.0**) — cell-probability threshold; the + network emits a per-pixel logit in roughly **−6…+6** and only pixels **above** + this value seed masks. This is the Cellpose default (0.0). **Lower it** (e.g. −2) + to recover more / dimmer / low-contrast cells (higher recall); **raise it** + (e.g. +1…+2) to suppress detections in dim regions (higher precision). Unlike the + other knobs it's a pure recall↔precision dial, not a speed dial. Exposed here so + it can be swept per dataset. +- `--niter` (default **10**) — number of flow-dynamics iterations. Cellpose default + is `None` (≈ proportional to diameter, often ~200). **10 is aggressively low** → + fast, but pixels for large/elongated cells may not fully converge. Increase if + cells look under-segmented/split. +- `--min_size` (default **-1**) — minimum pixels per mask. `-1` **disables** + small-mask removal (Cellpose treats `min_size<0` as "off"); skips a filtering + pass. Set a positive value (v3 default was 15) to drop specks. +- `--resample` (default **false**) — whether to run dynamics at the original image + resolution. `false` runs them on the downsampled grid → faster, slightly coarser + boundaries. `true` gives smoother/more precise masks but is slower. + +All six are simply forwarded into `model.eval`. `flow_threshold=0`, +`cellprob_threshold=0` and `min_size=-1` are meaningful values (not "unset") and +*are* passed through — only `None` values get dropped in the eval-params +comprehension (`:54`). + +## Optimization / tuning + +Two things to understand before tuning: +1. **Every shipped default is biased toward speed**, not accuracy — the component + exists precisely because the v4 model is slow (see history). So "optimizing for + quality" here mostly means *walking those defaults back* toward Cellpose's own + defaults, spending the time budget you can afford on GPU. +2. Grounded in the Cellpose-SAM paper + docs (defaults quoted from the docs). The + levers below are ranked by expected impact on iST segmentation quality. + +**Tier 1 — highest impact on quality** + +- **`diameter`** *(exposed; default 30, Cellpose default = auto)*. The biggest + lever. Cellpose rescales the image so objects land near the ~30 px training mean; + `30` therefore means "assume objects are already ~30 px / no rescaling." For + Xenium morphology (~0.2125 µm/px) an ~8 µm nucleus ≈ 35–40 px and a whole cell + ≈ 60–70 px, so a fixed 30 can be materially wrong. v4 is size-*robust* but not + size-*invariant* — tune per dataset, or set `None`/`0` to auto-estimate (slower). +- **`cellprob_threshold`** *(now exposed; default 0.0, range −6…+6)*. Pure + recall↔precision dial. Lower (e.g. −2) → recover more / dimmer / low-contrast + cells; raise (e.g. +1…+2) → suppress dim-region detections. In iST you usually + want to capture *all* cells, so sweeping this (≈ −2 … +2) is the most direct way + to trade missed cells against false ones. No speed cost. +- **`flow_threshold`** *(exposed; default 0.0 here, Cellpose default 0.4)*. `0.0` + **disables** the flow-consistency QC (fast, keeps ill-shaped masks). Restore + ~0.4 to filter malformed ROIs; raise above 0.4 to recover more. + +**Tier 2 — quality/speed trade-offs (already exposed)** + +- **`niter`** *(default 10; Cellpose default None ≈ ∝ diameter, often ~200)*. `10` + is aggressively low → large/elongated cells may not converge → fragmentation. + Raise (or set `None`) for non-round cells. +- **`resample`** *(default False)*. `True` runs dynamics at full resolution → + smoother, more precise boundaries, slower. Turn on when boundary accuracy matters. +- **`min_size`** *(default −1 = off; Cellpose default 15)*. Set positive (≈ 15–50) + to drop debris/specks → higher precision. + +**Tier 3 — not exposed, worth adding for a serious sweep** + +- **`normalize`** *(Cellpose default True, 1/99-percentile)*. Accepts a dict + (percentile bounds, tile-wise normalization, sharpening). For iST images with + uneven illumination / low contrast, the normalization scheme can matter; expose + at least the percentiles if results look intensity-sensitive. +- **`batch_size` / `bsize`** *(tile size, 256 for cpsam)*. Throughput knobs — raise + `batch_size` to trade GPU memory for speed. +- **`augment` / TTA** — test-time augmentation is the accuracy ceiling at ~4–8× + cost; rarely worth it inside a benchmark. + +**Tier 0 — the biggest lever is the input, not a parameter** + +The script segments **`image[0]` — a single channel only** (`:56`). Cellpose-SAM +dropped the `channels` arg *because* it's trained to use up to 3 channels +(cytoplasm + nuclear, any order). If a dataset carries a membrane/boundary stain +alongside the nuclear channel, feeding **both** as a 2-channel stack would improve +*whole-cell* boundaries more than any threshold tweak — the current code discards +that signal. This is the highest-leverage change when input images are +multi-channel (but see the "only channel 0" gotcha below — it's not wired up). + +**Suggested quality-first sweep order:** `diameter` (per dataset or auto) → +`cellprob_threshold` (−2…+2) → `flow_threshold` → 0.4 → `niter` ↑ / `resample` → +True → `min_size` → ~15–50. + +## Risk points / gotchas + +- **Speed / GPU.** cpsam is a ViT — on CPU it can blow the `viash test`/CI time + limit (that's literally why the v3 pin exists on the sibling component). Run this + one on GPU (`gpuhighmem`). If someone flips the test path to include it on a + CPU-only runner, expect timeouts. +- **Whole image loaded into RAM** (`:48`) — `.compute().to_numpy()` on `scale0` is + the full-res plane; big panels are why the label is `highmem`. No tiling/chunking + is done here beyond what cellpose does internally. +- **Only channel 0 is segmented** (`image[0]`). iST morphology images here are + effectively single-channel; if a multi-channel image ever arrives, the other + channels are ignored (v4 could use up to 3 — not wired up). +- **Aggressive defaults can under-segment.** `niter=10`, `flow_threshold=0`, + `min_size=-1`, `resample=false` are all tuned for throughput. If a dataset gives + poor masks, the first knobs to relax are `niter` ↑, `flow_threshold` → 0.4, + `resample` → true. +- **Shared metadata block** (`:67-86`) is duplicated in `cellpose/script.py`; fix + both together. +- **Don't downgrade to the `Cellpose` class.** It no longer exists in v4; only + `CellposeModel` is valid. Likewise there is no `channels`/`model_type` in v4. + +## Wiring + +- `src/workflows/run_benchmark/config.vsh.yaml`: listed as a dependency + (`methods_segmentation/cellposev4`) and included in the default method string + `custom_segmentation:cellpose:cellposev4:binning:stardist:watershed`. +- `src/workflows/run_benchmark/main.nf:98`: imported alongside `cellpose`. +- Run scripts: enabled in the GPU/full/MPII/seqera run configs + (`run_gpu_nebius.sh`, `run_full_nebius.sh`, `run_mpii_nebius.sh`, + `run_full_seqeracloud.sh`); commented out in `run_test_local.sh` / + `run_test_nebius.sh` (GPU + time cost too high for the quick test path). + +## References + +- Docs: https://cellpose.readthedocs.io/en/latest/ (settings, api pages have the + `model.eval` parameter reference). +- Repo: https://github.com/MouseLand/cellpose +- **Cellpose-SAM (the model this component runs):** Pachitariu M., Rariden M., + Stringer C., "Cellpose-SAM: superhuman generalization for cellular + segmentation", bioRxiv **10.1101/2025.04.28.651001**, v1, 2025-05-01 + (https://www.biorxiv.org/content/10.1101/2025.04.28.651001v1). Not yet a + peer-reviewed journal version at time of writing. +- **Original Cellpose (the DOI actually in `config.vsh.yaml`):** Stringer et al. + (2021), Nat. Methods, DOI 10.1038/s41592-020-01018-x — the flow-dynamics CNN, + i.e. the *framework*, not the v4 model. See the "Citation caveat" up top. +- Adapted from `openproblems-bio/task_spatial_segmentation` (cellpose method). diff --git a/src/methods_segmentation/stardist/NOTES.md b/src/methods_segmentation/stardist/NOTES.md new file mode 100644 index 000000000..0121111b1 --- /dev/null +++ b/src/methods_segmentation/stardist/NOTES.md @@ -0,0 +1,228 @@ +# StarDist2D segmentation — implementation notes + +Reference for anyone (human or Claude) working on this component. Captures how it +works, why the Docker/version pins are the way they are, what the tunable knobs are, +and where it can break. + +Links: docs/repo https://github.com/stardist/stardist · +DOI 10.48550/arXiv.1806.03535. + +**Citation caveat (minor — the DOI is correct):** `references.doi` is +`10.48550/arXiv.1806.03535` = Schmidt, Weigert, Broaddus & Myers, *"Cell Detection +with Star-convex Polygons"*, MICCAI 2018. That **is** the StarDist**2D** method paper, +and this component runs `StarDist2D`, so — unlike `cellposev4` — the DOI matches the +model actually used. Only nuance: the pretrained weights (`2D_versatile_fluo`) were +released later with the library, and there is a separate 3D follow-up (Weigert et al., +*"Star-convex Polyhedra…"*, WACV 2020) that does **not** apply here. No fix needed. + +## What this component is + +One of the methods at the **segmentation** stage of the iST preprocessing benchmark +(`src/methods_segmentation/`). It implements the standard API +`src/api/comp_method_segmentation.yaml`: `raw_ist.zarr` in → `segmentation.zarr` out +(a 2D cell **label map** + a stub `table`). + +It runs **StarDist2D** — a CNN that, for every pixel, predicts a **star-convex +polygon** (a set of radial distances to the object boundary) plus an object +probability, then reconstructs instances by non-maximum suppression over those +polygons. Because objects are represented as star-convex polygons centered on each +nucleus, it excels at **round/blob-like nuclei in crowded fluorescence images** — +the DAPI-like nuclear morphology channel of iST data. It loads a **pretrained** model +(`StarDist2D.from_pretrained`), so there is no training step here. + +Unlike Cellpose-SAM (up to 3 channels), StarDist2D is a **single-channel nuclear +detector**: the script feeds it `image[0]` only (see Tier 0 below). + +## script.py — step by step + +1. **numpy-alias shim** (`:5-15`) — restores the deprecated `np.bool`/`np.int`/… + aliases that numpy≥1.24 removed but stardist/csbdeep still reference. Load-bearing + (see Setup); must run **before** `import stardist`. +2. **Read input** (`:52-54`) — `sd.read_zarr(par["input"])`, then + `sdata['image']['scale0'].image.compute().to_numpy()` pulls the full-res image + into RAM as a `(c, y, x)` array; the `scale0` transform is copied to reattach to + the output. (NB: the *original* 2025-07 version read `morphology_mip`; the current + raw_ist contract exposes the morphology plane as `image` — do not revert.) +3. **Load model** (`:59`) — `StarDist2D.from_pretrained(par['model'])`. Default + `2D_versatile_fluo`. This also loads the model's optimized `thresholds.json` + (prob=0.479071, nms=0.3 for that model) as the fallback thresholds. +4. **Percentile normalizer** (`:64-79`) — a csbdeep `Normalizer` subclass that min-max + scales the image to its **1st / 99.8th percentiles** (`normalize_mi_ma`), the + recommended StarDist preprocessing but with fixed percentile bounds. `block_size` + and `offset` (block overlap/context) are derived from the image width so a large + panel is processed in tiles. +5. **Build eval-params** (`:85-89`) — collects the newly exposed tunables + (`prob_thresh, nms_thresh, scale`) from `par`, **dropping any that are `None`**. + A dropped key ⇒ `predict_instances` uses the model's own optimized value (so the + no-args call is byte-for-byte the pre-tuning behaviour). Mirrors the cellposev4 + eval-params pattern. +6. **Segment** (`:92-95`) — `model.predict_instances_big(image[0], axes='YX', + block_size=…, min_overlap=…, context=…, normalizer=…, **eval_params)`. + `predict_instances_big` splits the image into `block_size` blocks, calls + `predict_instances` on each (forwarding `**eval_params` unchanged — it only + overrides `axes/overlap_label/return_labels/return_predict`), and reassembles the + labels into global coordinates. `image[0]` = first channel → a single 2D plane. +7. **Post-process** (`:100-104`) — `convert_to_lower_dtype` downcasts the label array + to the smallest uint that holds `max label`; wrap as an `xarray.DataArray`, + `Labels2DModel.parse` with the copied transform, store as + `sd_output.labels['segmentation']`. +8. **Carry the metadata table** (`:106-125`) — copies a stub `table` from the input's + `tables["metadata"]`; resolves a required `cell_id` in priority order (explicit + `obs["cell_id"]` → the table's `instance_key` column → the obs index) and carries + an optional `region`. **This block is duplicated verbatim in `cellpose/`, + `cellposev4/` — keep them in sync.** +9. **Write** (`:127-131`) — `rmtree` any existing output, then `sd_output.write`. + +Note: the output `table` describes the *reference* cells from metadata, not the newly +predicted masks — segmentation output here is the **label image**; downstream +aggregation turns masks into a cell × gene table. + +## config.vsh.yaml — the Docker setup (the highest-value section) + +Base image `openproblems/base_tensorflow_nvidia:1` (GPU-capable TF). The pip pins are +**fought-over and load-bearing** — see the history before changing them: + +- **`tensorflow==2.18.0`** + **`tf_keras==2.18.0`** + **`numpy>=2.0.0,<2.2.0`** + + **`scipy<1.15.0`**. The knot: `anndata`/`spatialdata` reference `np.bool` (re-added + in numpy **2.0**), so `numpy<2.0` makes `import anndata` crash; but TF **2.17** caps + `numpy<2.0`. Resolution = bump TF to **2.18** (allows numpy≥2.0), and because + csbdeep/stardist use the **Keras-2 API via `tf_keras`**, `tf_keras` must match the TF + version. `scipy<1.15.0` is pinned for a stardist/csbdeep compatibility break. +- Even with numpy≥2.0, stardist/csbdeep still reference the **removed** scalar aliases + (`np.bool`, `np.long`, …) at import → hence the runtime shim at the top of the script + (`:5-15`). Both halves (the TF/numpy pin **and** the shim) are required together. +- History of this pin (git): `06d998545` add (numpy<2.0, tf loose) → `5058466a7` + add `scipy<1.15.0` → `aa12d73fc` drop the numpy/scipy pins → `15a214eb3` (#167) the + current tf 2.18 / tf_keras 2.18 / numpy 2.0-2.2 / scipy pin + the alias shim. + Don't "simplify" these back. +- Dev note (in-config): on **macOS** the TF install fails; develop in a conda env + (`conda install -c conda-forge tensorflow`), test the Docker via gh-actions. +- Nextflow directives: `[ hightime, midcpu, highmem, gpu ]` — 8 h / 15 CPU / 100 GB / + GPU. TF uses the GPU when present; on CPU it still runs (much lighter than + cellposev4's SAM ViT) but slower. + +## Arguments + +| arg | type | default | maps to | +|-----|------|---------|---------| +| `--model` | string | `2D_versatile_fluo` | `StarDist2D.from_pretrained(model)` | +| `--prob_thresh` | double | *(unset → model 0.479071)* | `predict_instances(prob_thresh=)` | +| `--nms_thresh` | double | *(unset → model 0.3)* | `predict_instances(nms_thresh=)` | +| `--scale` | double | *(unset → no rescale)* | `predict_instances(scale=)` | + +`prob_thresh`, `nms_thresh`, `scale` were **added during tuning** (see below). They are +**optional with no static default on purpose**: leaving them unset makes StarDist use +each model's own optimized `thresholds.json` — hard-coding e.g. `0.479071` would be +wrong for a different `--model` whose optimized thresholds differ. **They need +`viash ns build` + a container rebuild to take effect** (see `check-component`). + +Not exposed: +- `--n_tiles` — a pure GPU-memory tiling knob. `predict_instances_big` **already** tiles + the image via `block_size` (≤4096 px), so `n_tiles` would only sub-tile each block for + the forward pass; it has **no effect on segmentation quality**, only on peak memory. + Deliberately left out of the sweep (would just pad it). +- The normalization percentiles (`1, 99.8`) are hardcoded in the script (`:76`); could + be exposed if results look intensity-sensitive, but not a priority (see Tier 2). + +## Optimization / tuning + +Two things to understand before tuning: +1. Unlike `cellposev4`, the shipped defaults here are **not** aggressively speed-tuned — + `predict_instances` runs with each model's paper-optimized thresholds. So "optimize + for quality" here is mostly **fitting StarDist to iST nuclei** (object size + the + recall/precision thresholds), not walking back throttled knobs. +2. Ranges are grounded in the StarDist paper/docs and the pixel geometry of the data. + The pretrained thresholds quoted (prob 0.479071 / nms 0.3) are those of + `2D_versatile_fluo`. + +**Tier 0 — the input, not a parameter.** The script segments **`image[0]` — a single +channel only** (`:93`). StarDist2D is a single-channel **nuclear** detector (no +membrane/multi-channel mode like Cellpose), so the lever here is **which stain is fed** +and its **normalization** — feeding the crisp nuclear (DAPI-like) plane and getting the +1/99.8-percentile normalization right matters more than any threshold tweak. Its output +is **nuclei**, not whole cells; whole-cell boundaries come from downstream expansion. + +**Tier 1 — highest impact on quality (all newly exposed)** + +- **`prob_thresh`** *(now exposed; default = model 0.479071, range ~0..1)*. Pure + recall↔precision dial and the most direct knob. **Lower** (e.g. 0.3–0.4) → recover + more / dimmer nuclei (higher recall); **raise** (e.g. 0.6–0.7) → keep only confident + detections (higher precision). In iST you usually want to capture all nuclei, so this + is the first axis to sweep. No speed cost. +- **`scale`** *(now exposed; default = none/1.0)*. StarDist's analogue of Cellpose's + `diameter`: rescales the image so nuclei land near the model's trained size. + `2D_versatile_fluo` was trained on a DSB2018 subset (varied, roughly ~20–40 px nuclei); + Xenium morphology (~0.2125 µm/px) makes an ~8 µm nucleus ≈ **38 px**, so a mild + down/upscale can matter. `>1` upscales (small nuclei appear larger), `<1` downscales. + Sweep both sides (≈ 0.5 … 2.0). +- **`model`** *(already exposed; default `2D_versatile_fluo`)*. The other **fluorescence** + pretrained model is `2D_paper_dsb2018`. `2D_versatile_he` is a **brightfield H&E RGB** + model — wrong for single-channel fluorescence — so it is NOT swept. + +**Tier 1/2 — merge control** + +- **`nms_thresh`** *(now exposed; default = model 0.3, range ~0..1)*. Non-max-suppression + IoU for overlapping polygon candidates. **Lower** → more aggressive suppression + (touching nuclei more likely merged/dropped); **raise** → keep more overlapping + detections in crowded tissue. Secondary to `prob_thresh`/`scale`; sweep a few values + (≈ 0.2 … 0.5). + +**Tier 3 — not exposed, worth adding only for a deeper sweep** + +- **Normalization percentiles** (`pmin, pmax`, currently `1, 99.8`). For panels with + uneven illumination / low contrast the percentile bounds change what counts as + background; expose `pmin`/`pmax` if results look intensity-sensitive. +- **`n_tiles`** — memory only, not quality (see Arguments). Not worth a sweep axis. + +**Suggested quality-first sweep order:** `prob_thresh` (0.3…0.7) → `scale` +(0.5…2.0, from pixel size) → `nms_thresh` (0.2…0.5) → `model` → `2D_paper_dsb2018`. +That is exactly the sweep encoded in `scripts/run_benchmark/stardist_params.yaml` +(**13 variants** = 1 default + 4 + 4 + 3 + 1). + +## Wiring + +- `src/workflows/run_benchmark/config.vsh.yaml`: listed as a dependency + (`methods_segmentation/stardist`, `:161`) and included in the default method string + `custom_segmentation:cellpose:cellposev4:binning:stardist:watershed` (`:65`). +- `src/workflows/run_benchmark/main.nf:100`: imported alongside the other seg methods. +- Sweep run scripts (this tuning): `scripts/run_benchmark/run_test_stardist_local.sh` + and `run_test_stardist_nebius.sh`, sharing the committed + `scripts/run_benchmark/stardist_params.yaml` (local reads it directly; Nebius reads + it from the GitHub raw URL — so it must be **pushed** before launching). +- Already enabled in the standing run configs: `run_gpu_nebius.sh`, `run_full_nebius.sh`, + `run_mpii_nebius.sh`, `run_test_seqeracloud.sh`, `run_full_seqeracloud.sh`; commented + out in `run_test_local.sh` / `run_test_nebius.sh` (GPU/time cost for the quick path). + +## Risk points / gotchas + +- **The TF/tf_keras/numpy/scipy pin quartet + the alias shim are a matched set.** Change + one and imports break (see Setup). This is the single most fragile thing here. +- **New args need a rebuild.** `prob_thresh`/`nms_thresh`/`scale` only exist after + `viash ns build` + a container rebuild; a stale image silently ignores them. The sweep + has **not yet been run end-to-end** with the new args — validated only by + `viash config view` + a script `ast.parse`. +- **`scale` through `predict_instances_big`.** It is forwarded per-block via `**kwargs`; + block geometry (`block_size`/`context`) stays in original-pixel units, so extreme + `scale` values interact with the block overlap. Sanity-check masks at the block seams + if using large `scale`. +- **Only channel 0 is segmented** (`image[0]`). Fine for single-channel iST morphology; + StarDist2D has no multi-channel mode anyway. +- **Whole image loaded into RAM** (`:53`) — full-res plane; big panels are why the label + is `highmem`. Tiling beyond that is `predict_instances_big`'s `block_size`. +- **Shared metadata block** (`:106-125`) is duplicated across the cellpose components; + fix all together. +- **StarDist detects nuclei, not whole cells.** Expect nuclear-sized masks; downstream + volume/expansion stages handle cell extent. + +## References + +- **StarDist2D (the model this component runs, = the config DOI):** Schmidt U., Weigert + M., Broaddus C., Myers G., *"Cell Detection with Star-convex Polygons"*, MICCAI 2018, + arXiv **1806.03535** (DOI 10.48550/arXiv.1806.03535). +- 3D follow-up (not used here): Weigert M., Schmidt U., et al., *"Star-convex Polyhedra + for 3D Object Detection and Segmentation in Microscopy"*, WACV 2020. +- Docs/repo: https://github.com/stardist/stardist (see `examples/other2D/ + predict_big_data.ipynb`, the source of the `predict_instances_big` + custom-normalizer + pattern used here). +- `2D_versatile_fluo` optimized thresholds (`thresholds.json`): prob 0.479071, nms 0.3; + trained on a subset of the DSB 2018 nuclei-segmentation dataset. diff --git a/src/methods_segmentation/watershed/NOTES.md b/src/methods_segmentation/watershed/NOTES.md new file mode 100644 index 000000000..56ceeb697 --- /dev/null +++ b/src/methods_segmentation/watershed/NOTES.md @@ -0,0 +1,238 @@ +# Watershed segmentation — implementation notes + +Reference for anyone (human or Claude) working on this component. Captures how the +classic watershed pipeline works, which of its ~40 exposed knobs actually matter, +and where the sharp edges are. + +Links: docs (this repo) https://github.com/openproblems-bio/task_ist_preprocessing · +implementation https://github.com/theislab/txsim (`txsim.preprocessing.segment_watershed`). + +**Citation caveat:** `references.doi` is `10.1109/34.87344` — Vincent & Soille (1991), +"Watersheds in digital spaces: an efficient algorithm based on immersion simulations", +IEEE TPAMI. That is a legitimate *foundational* watershed reference, so this is **not** a +wrong-paper mismatch like cellposev4. Minor nuance: the code uses +`skimage.segmentation.watershed`, which is the **marker-controlled / priority-flood** +variant (Meyer's flooding, plus Neubert & Protzel for the `compactness` option), not the +Vincent-Soille immersion algorithm per se. Also of historical note: the `repository` link +once (commits `d7c5ae0a0`..`5885bdac8`) pointed at `BennyStrobes/Watershed` — a completely +**unrelated genomics** method — before being corrected to `theislab/txsim` in `5885bdac8`. +The DOI is fine; leave it. + +## What this component is + +One of the methods at the **segmentation** stage of the iST preprocessing benchmark +(`src/methods_segmentation/`). It implements the standard API +`src/api/comp_method_segmentation.yaml`: `raw_ist.zarr` in → `segmentation.zarr` out (a 2D +cell **label map** + a stub `table`). + +Unlike the deep-learning segmenters (cellpose/cellposev4/stardist), this is a **classic, +CPU-only image-processing pipeline** with **no learned model** — a chain of skimage +operations wrapped by `txsim.preprocessing.segment_watershed`. Its distinguishing feature +is an unusually **large, fully function-selectable parameter surface** (~40 args): each +pipeline stage's algorithm is chosen by a `*_func` string, and each stage's parameters are +individually exposed. Most of that surface is low-value plumbing; see "Optimization / +tuning" for the handful of knobs that move the result. + +## How the watershed pipeline works (`segment_watershed`) + +`img[0]` (first channel, a single 2D plane) flows through this fixed chain in txsim +(`txsim/preprocessing/_segmentation.py:382`); each stage is toggled by whether its +`*_func` string maps to a known function: + +1. **Normalize** (`normalize_func` → `adjust_gamma`|`adjust_log`|`adjust_sigmoid`). Default + `gamma` with gamma=1, gain=1 → effectively identity. +2. **Contrast** (`contrast_adjustment_func` → `equalize_adapthist`|`equalize_hist`| + `rescale_intensity`). Default `equalize_adapthist` (CLAHE, clip_limit 0.01). +3. **Blur** (`blur_func` → `gaussian`|`median`). Default `gaussian`, `blur_sigma=1`. +4. **Threshold → binary nuclei mask** (`threshold_func` → `otsu`|`triangle`|`local_otsu`). + Image is first cast to ubyte (`skimage.util.img_as_ubyte`; the module-level `import + skimage` at `_segmentation.py:8` is what makes that name resolve). `local_otsu` + (`rank.otsu`, default) is **adaptive** over a footprint (`threshold_footprint`=square of + size `threshold_footprint_size`=50) → `nuclei = img >= local_threshold`. Global `otsu`/ + `triangle` pick one scalar threshold → `nuclei = img > threshold`. +5. **Mask post-processing** (loop over `post_processing_func_{i}`): default + `remove_small_objects` (min_size 64), then `remove_small_holes` (area_threshold 64), + applied to the **binary mask** before the distance transform. +6. **Distance transform** (`distance_transform_edt`) of the binary mask. +7. **Local-maxima markers** (`find_local_maxima` → `peak_local_max` with + `local_maxima_min_distance`, then `label`). Each maximum becomes one watershed seed → + one cell. +8. **Watershed** — `watershed(-distance, markers, mask=nuclei, watershed_line=True, + **watershed_params)`. Floods basins from the markers within the nuclei mask. +9. **Background-intensity filter** (`filter_cells_based_on_local_background_intensity`, + `_segmentation.py:330`) — if `bg_intensity_filter_bg_factor > 0` (default 0.3), drops + cells whose mean intensity is below `bg_factor ×` local background and reindexes labels. + +## script.py — step by step + +1. **Build hyperparams** (`:34-38`) — `par.copy()`, convert the string `"None"` → Python + `None` (viash passes unset optionals as the literal string `"None"`), drop `input`/ + `output`. Every remaining key is forwarded by name; `segment_watershed` reads what it + recognises via `hyperparams.get(...)`. +2. **Lift compactness into `watershed_params`** (`:40-46`) — see "Optimization"; pops the + exposed `--watershed_compactness` and nests it as `watershed_params={"compactness": …}`, + the only dict `segment_watershed` forwards to `skimage.watershed`. Default 0.0 == + standard watershed (unchanged behaviour). +3. **Read input** (`:48-52`) — `sd.read_zarr`, `sdata['image']['scale0']` pulled full-res + into memory (`.compute().to_numpy()`), and the `scale0` transform copied to reattach. +4. **Segment** (`:53`) — `tx.preprocessing.segment_watershed(image[0], hyperparameters)`. + `image[0]` = first channel only. +5. **Post-process** (`:54-57`) — `convert_to_lower_dtype` downcasts the label array to the + smallest uint that holds `max label`; wrap as `xarray.DataArray(dims=('y','x'))`, + `Labels2DModel.parse` with the copied transform, store as `labels['segmentation']`. +6. **Carry the metadata table** (`:59-78`) — copies a stub `table` from + `tables["metadata"]`; resolves `cell_id` in priority order explicit `obs["cell_id"]` → + `instance_key` column → obs index; carries optional `region`. **This block is shared + near-verbatim with the other segmentation components** (cellpose/cellposev4) — keep in + sync if you touch it. +7. **Write** (`:80-83`) — `rmtree` any existing output, then `sd_output.write`. + +## config.vsh.yaml — Docker setup + +- Base image `openproblems/base_python:1` (plain CPU Python — no CUDA/torch; this method + never uses a GPU). +- `setup` merges `/src/base/setup_txsim_partial.yaml` (needed — the whole method is + `txsim.preprocessing.segment_watershed`) and `/src/base/setup_spatialdata_partial.yaml`. + Commit `050fa15e1` reordered the installs ("Change order of installs in cellpose and + watershed") — the txsim/spatialdata install order is load-bearing for a clean env. +- Nextflow directives: `[ hightime, midcpu, midmem ]` — 8 h / 15 CPU / 50 GB. Note the + **hightime** budget: the classic pipeline (esp. `local_otsu` rank filter with a 50 px + footprint and the O(image) background-intensity filter over large windows) is slow on + big panels even though it is CPU-light per pixel. + +## Arguments (the surface is large; most of it is inert) + +~40 args across 8 stages. Two facts to internalise before reading the list: + +- **`*_func` args are function selectors with tiny, closed mappings.** A value **not** in + the mapping silently sets that stage's function to `None` (skips the stage) — and for + `threshold_func`/`local_maxima_func` that means downstream names (`nuclei`, + `distance_transform`, `local_maxima`) are never defined → the script **crashes** with a + `NameError`. Valid values: `normalize_func` ∈ {gamma, log, sigmoid}; `contrast_ + adjustment_func` ∈ {equalize_adapthist, equalize_hist, rescale_intensity}; `blur_func` ∈ + {gaussian, median}; `threshold_func` ∈ {otsu, triangle, local_otsu}; `local_maxima_func` + ∈ {find_local_maxima}; `post_processing_func_{i}` ∈ {remove_small_objects, + remove_small_holes}. +- **`filter_params` drops any param a chosen function doesn't accept.** So many exposed + args are no-ops for the default function (e.g. the `distance_transform_*` return/indices + args, the `threshold_shift_*`/`hist`/`out`/`mask` args). Don't expect them to do + anything unless you also switch the corresponding `*_func`. + +The knobs that actually move the output (defaults in **bold**): + +| Arg | Default | Effect | +|---|---|---| +| `--threshold_func` | **local_otsu** | foreground/background mask algorithm; see Tier 1 | +| `--threshold_footprint_size` | **50** | neighbourhood (px) for `local_otsu` only | +| `--blur_sigma` | **1** | gaussian pre-smoothing; higher = fewer spurious maxima | +| `--local_maxima_min_distance` | **5** | min px between watershed seeds = merge↔split dial | +| `--post_processing_min_size_1` | **64** | `remove_small_objects` min mask area (px) | +| `--post_processing_area_threshold_2` | **64** | `remove_small_holes` fill area (px) | +| `--bg_intensity_filter_bg_factor` | **0.3** | drop cells dimmer than 0.3× local bg; 0 disables | +| `--watershed_compactness` | **0.0** | NEW; compact-watershed shape regularity (Tier 3) | + +## Optimization / tuning + +Grounded in the txsim implementation and Xenium morphology geometry: at ~0.2125 µm/px an +~8 µm nucleus ≈ **37 px diameter, ~18 px radius, ~1000 px² area** — these anchor the +ranges below. Unlike cellposev4 the defaults here are not obviously *speed*-tuned; they +are a specific (and somewhat arbitrary) pipeline config, and the default `min_distance=5` +in particular tends to **over-split** real nuclei. + +**Tier 0 — the input, not a parameter.** Only `image[0]` (a single channel) is segmented +(`script.py:53`). watershed here targets **nuclei**; whole-cell boundaries are not +recoverable from a nuclear channel alone. If a membrane/boundary stain exists, no threshold +tweak substitutes for it. + +**Tier 1 — highest impact on quality (these are the swept knobs):** + +- **`local_maxima_min_distance`** *(default 5)*. The dominant lever — it sets how close two + watershed seeds may be, i.e. the merge↔split dial. 5 px is far below the nucleus radius + (~18 px), so single nuclei get multiple seeds → over-segmentation. Sweep **[10, 15, 20]** + (walking toward the radius) to merge fragments back into one cell. +- **`threshold_func`** *(default local_otsu)*. Decides *what is foreground* at all. + `local_otsu` is adaptive (good under uneven illumination but slow, footprint 50 px); + global **otsu**/**triangle** use one threshold (fast; `triangle` suits a dominant + background peak, typical of fluorescence). Categorical — sweep the two alternatives. +- **`blur_sigma`** *(default 1)*. Heavier gaussian smoothing removes spurious distance- + transform maxima (fewer over-split cells) but blurs small/dim nuclei. Sweep **[2, 3, 4]** + px (stays sub-nucleus). +- **`post_processing_min_size_1`** *(default 64)*. `remove_small_objects` on the binary mask + — the precision dial against debris/partial nuclei. Sweep **[128, 256, 512]**, all well + below a full nucleus area (~1000 px²) so real nuclei survive. + +**Tier 2 — quality knobs, exposed but not swept:** + +- **`threshold_footprint_size`** *(default 50)* — only active when `threshold_func= + local_otsu`; sets the local-Otsu neighbourhood. Couple it with any `threshold_func` + change. +- **`bg_intensity_filter_bg_factor`** *(default 0.3)* — a real recall↔precision post-filter + (drops dim cells); set 0 to disable, raise to prune harder. Its `window_size`/`bg_size` + (1000/2000) are large and drive much of the runtime. +- **`post_processing_area_threshold_2`** *(default 64)* — `remove_small_holes`; minor. +- **`contrast_adjustment_clip_limit`** *(default 0.01)* — CLAHE aggressiveness; can help + low-contrast panels but interacts unpredictably with thresholding. + +**Tier 3 — high-value knob newly exposed by this component:** + +- **`watershed_compactness`** *(NEW; default 0.0)*. `skimage.watershed`'s compactness was + previously unreachable (only settable via the nested `watershed_params` dict that txsim + forwards but the component never populated). Now exposed and lifted into that dict + (`script.py:40-46`). 0.0 = standard watershed; small positive values (try 1e-3…1e-1) + enable **compact watershed** (Neubert & Protzel) for rounder, more regular cell shapes — + useful when watershed produces jagged/leaky basins. Left out of the default sweep to keep + it at 12 variants and honour the Tier-1 focus. **Requires `viash ns build` + a container + rebuild to take effect** (see "Risk points"). + +**Not worth touching for a benchmark:** the normalize/contrast function *choices*, all the +`distance_transform_*` return/indices/sampling args, the `threshold_shift_*`/`hist`/`out`/ +`mask` args, `blur_mode`/`cval`/`preserve_range`/`truncate`, `*_connectivity_*`, `*_out_*`. +They are either inert under the default functions (dropped by `filter_params`) or pure +plumbing. + +**Suggested quality-first order:** `local_maxima_min_distance` ↑ (fix over-splitting) → +`threshold_func` / `threshold_footprint_size` → `blur_sigma` ↑ → `post_processing_min_size_1` +↑ → `bg_intensity_filter_bg_factor` → `watershed_compactness` for shape. + +## Risk points / gotchas + +- **Newly exposed arg needs a rebuild.** `--watershed_compactness` was added to + `config.vsh.yaml` and threaded in `script.py`; it has **no effect until** `viash ns build` + regenerates the target and the Docker container is rebuilt (see the `check-component` + skill). The rest of the sweep uses pre-existing args and works against the current image. +- **Unmapped `*_func` string = silent skip or crash.** Only sweep `threshold_func` among + its three valid values; never point a selector at a name outside its mapping (Arguments + §). `threshold_func`/`local_maxima_func` set to an unmapped value → `NameError` because + `nuclei`/`local_maxima` never get defined. +- **`min_distance` too small over-splits; too large merges cells.** The default 5 errs + toward over-segmentation — this is the first knob to raise if masks look fragmented. +- **Whole image loaded into RAM** (`script.py:51`) — full-res plane; part of why the label + is `midmem` and the stage is `hightime`. +- **Only channel 0 is segmented** (`image[0]`) — other channels ignored. +- **Shared metadata block** (`script.py:59-78`) is duplicated across the segmentation + components; fix together. +- **Runtime is threshold/filter-bound**, not model-bound: `local_otsu` (rank filter, + footprint 50) and the background-intensity filter (windows 1000/2000) dominate. + +## Wiring + +- `src/workflows/run_benchmark/config.vsh.yaml`: listed as a dependency + (`methods_segmentation/watershed`, `:162`) and included in the default segmentation + method string `custom_segmentation:cellpose:cellposev4:binning:stardist:watershed` + (`:65`). +- `src/workflows/run_benchmark/main.nf:101`: imported. +- Run scripts (this tuning task): `scripts/run_benchmark/run_test_watershed_local.sh` and + `run_test_watershed_nebius.sh`, driven by the committed sweep + `scripts/run_benchmark/watershed_params.yaml` (single source of truth; local reads it via + `$REPO_ROOT`, Nebius via its raw GitHub URL — so it must be pushed before a Nebius + launch). Otherwise watershed is commented out in the standard `run_test_*` scripts. + +## References + +- **Watershed transform (the DOI in config):** Vincent L. & Soille P. (1991), "Watersheds + in digital spaces: an efficient algorithm based on immersion simulations", IEEE TPAMI + 13(6):583-598, DOI **10.1109/34.87344**. +- Implementation: `theislab/txsim` (`dev`), `txsim/preprocessing/_segmentation.py:382` + `segment_watershed`. +- Underlying ops: `skimage.segmentation.watershed` (marker-controlled; `compactness` per + Neubert & Protzel), `skimage.feature.peak_local_max`, `skimage.filters.rank.otsu`. diff --git a/src/methods_transcript_assignment/fastreseg/NOTES.md b/src/methods_transcript_assignment/fastreseg/NOTES.md new file mode 100644 index 000000000..e57e325a8 --- /dev/null +++ b/src/methods_transcript_assignment/fastreseg/NOTES.md @@ -0,0 +1,139 @@ +# fastReseg — transcript assignment + +## What this component is + +fastReseg is a **transcript-assignment** method (API: `src/api/comp_method_transcript_assignment.yaml`; +inputs `raw_ist.zarr` + `segmentation.zarr` + `scrnaseq_reference.h5ad`, output +`transcript_assignments.zarr`). It corrects an initial image-based segmentation using the +spatial profile of transcripts. Wraps the R package +[`Nanostring-Biostats/FastReseg`](https://github.com/Nanostring-Biostats/FastReseg) +(DOI 10.1038/s41598-025-08733-5). + +**What's unusual: it is the only multi-script, bash-orchestrated component in the repo.** +It has four resources and the *first* (`orchestrator.sh`, a `bash_script`) is the Viash +entrypoint. It chains three sub-scripts through TSV/CSV files in a temp dir: + +``` +orchestrator.sh + → input.py (SpatialData zarr → counts.tsv, transcripts.tsv, cell_types.tsv) + → script.R (FastReseg::fastReseg_full_pipeline → cell_ids.csv, gene_names.csv, transcripts_out.csv) + → output.py (CSVs → transcript_assignments.zarr) +``` + +## orchestrator.sh — control flow + +- `set -eo pipefail` (line 6) — **load-bearing.** Without it, a crash in any sub-step is + swallowed (the script's last line is `echo $(date)` → exit 0) and Viash reports only the + generic *"Required output file is missing. Expression: par.required"*, hiding the real + Python/R traceback. This misdirection cost a whole debugging round; keep it. +- Sub-scripts are invoked as `"$meta_resources_dir/