From 818540daecd6633bcbb326fa1872466d0b5e8367 Mon Sep 17 00:00:00 2001 From: Fredric Johansson Date: Fri, 17 Jul 2026 12:45:25 +0200 Subject: [PATCH 1/2] Make a package of the project --- .gitignore | 14 +++ MANIFEST.in | 15 +++ README.md | 25 +++- hpacellseg/.gitignore | 40 ------ hpacellsegmentator/VERSION | 1 + hpacellsegmentator/__init__.py | 6 + .../generate_cell_crops.py | 2 +- .../generate_masks.py | 2 +- .../hpacellseg}/VERSION | 0 .../hpacellseg}/__init__.py | 0 .../hpacellseg}/__main__.py | 6 +- .../hpacellseg}/cellsegmentator.py | 6 +- .../hpacellseg}/constants.py | 0 .../hpacellseg}/utils.py | 0 .../image_utils.py | 0 hpacellsegmentator/process.py | 114 ++++++++++++++++++ process.py | 112 ----------------- pyproject.toml | 43 +++++++ 18 files changed, 220 insertions(+), 166 deletions(-) create mode 100644 .gitignore create mode 100644 MANIFEST.in delete mode 100644 hpacellseg/.gitignore create mode 100644 hpacellsegmentator/VERSION create mode 100644 hpacellsegmentator/__init__.py rename generate_cell_crops.py => hpacellsegmentator/generate_cell_crops.py (99%) rename generate_masks.py => hpacellsegmentator/generate_masks.py (93%) rename {hpacellseg => hpacellsegmentator/hpacellseg}/VERSION (100%) rename {hpacellseg => hpacellsegmentator/hpacellseg}/__init__.py (100%) rename {hpacellseg => hpacellsegmentator/hpacellseg}/__main__.py (84%) rename {hpacellseg => hpacellsegmentator/hpacellseg}/cellsegmentator.py (98%) rename {hpacellseg => hpacellsegmentator/hpacellseg}/constants.py (100%) rename {hpacellseg => hpacellsegmentator/hpacellseg}/utils.py (100%) rename image_utils.py => hpacellsegmentator/image_utils.py (100%) create mode 100644 hpacellsegmentator/process.py delete mode 100644 process.py create mode 100644 pyproject.toml diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..cdf5541 --- /dev/null +++ b/.gitignore @@ -0,0 +1,14 @@ +*.png +log.txt +crop_info.csv + +*.pyc +*.pyo +*.pth +__pycache__/ + +# Distribution / packaging +build/ +dist/ +*.egg-info/ +.eggs/ diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..f05d48b --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,15 @@ +include README.md +include requirements.txt +include config.yaml +include hpacellseg/VERSION + +exclude path_list.csv +exclude log.txt +exclude crop_info.csv + +recursive-exclude models * +recursive-exclude test * +recursive-exclude * __pycache__ +recursive-exclude * *.py[co] +recursive-exclude * *.pth +recursive-exclude * *.tif \ No newline at end of file diff --git a/README.md b/README.md index 219d324..1ce3ab0 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,17 @@ Just a repackage of the "HPA Cell Segmentator" repository with updated libraries Installation ------------ -If you use any Python IDE (VSCode, PyCharm, Spyder, etc...), just: +**As a package (recommended).** From the repository root: + +- `pip install .` to install it, or `pip install -e .` for an editable/development install. + +This installs three importable top-level packages (`hpacellsegmentator`, `hpacellseg`, `pytorch_zoo`) and three commands: + +- `hpacellsegmentator`: segment a single set of images (see *Running the code*). +- `hpacellsegmentator-batch`: process many image sets via `path_list.csv`. +- `hpacellsegmentator-service`: run the HTTP segmentation service. + +**Manual/IDE setup.** If you use any Python IDE (VSCode, PyCharm, Spyder, etc...), just: - Either import the project into your IDE through git/github OR Create a new project and download all the repository code from github into it. - Create a virtual environment for that project. - Install the project requirements through your IDE. Make sure the packages versions match, as IDEs try to be too smart some times. @@ -67,13 +77,16 @@ images/CACO-2_2047_C3_6_red.png,images/CACO-2_2047_C3_6_yellow.png,images/CACO-2 images/U-215MG792_H7_2_red.png,images/U-215MG792_H7_2_yellow.png,images/U-215MG792_H7_2_blue.png,,output,,U-215MG792_H7_2_ ``` -Once you have prepared your `path_list.csv` you are ready to run the `process.py` script. You can choose between 3 different running approaches, depending on your personal preferences: +Once you have prepared your `path_list.csv` you are ready to run the batch script (`hpacellsegmentator/batch.py`, formerly `process.py`). It reads `path_list.csv` and `config.yaml` from the current working directory, so run it from the directory that holds them. You can choose between 3 different running approaches, depending on your personal preferences: -- Edit directly the constants located in the `process.py` script: +- Edit directly the constants located in the `hpacellsegmentator/batch.py` script: - Probably the least versatile, but useful if you are always running HPACellSegmentatorPortable with the same settings. - Just change the values under for the following section of code: `# If you want to use constants with your script, add them here` . - - Simply call `python process.py`. + - Simply call `hpacellsegmentator-batch`. +- Call the batch script with arguments: + - You can get a list of available parameters (and their default values) using `-help` or `-?` argument. + - Example call: `hpacellsegmentator-batch -c True -cs 684`. - Call `process.py` script with arguments: - You can get a list of available parameters using the `-h` or `--help` argument. - Boolean options are toggled with a flag: use `-c`/`--crop_cells` to enable or `--no-crop_cells` to disable (likewise `-cm`/`--crop_mask` and `-mc`/`--mask_cell`). @@ -82,7 +95,9 @@ Once you have prepared your `path_list.csv` you are ready to run the `process.py - Edit the `config.yaml` file: - Just change the contents of the file with your desired values. - - Simply call `python process.py`. + - Simply call `hpacellsegmentator-batch`. + +If you have not installed the package, `python -m hpacellsegmentator.batch` works the same way from the repository root. These approaches can be combined; when a parameter is set in more than one place, command line arguments take precedence over `config.yaml`, which in turn takes precedence over the constants in `process.py`. diff --git a/hpacellseg/.gitignore b/hpacellseg/.gitignore deleted file mode 100644 index e6f70dd..0000000 --- a/hpacellseg/.gitignore +++ /dev/null @@ -1,40 +0,0 @@ -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class - -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -pip-wheel-metadata/ -share/python-wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Environments -.env -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ \ No newline at end of file diff --git a/hpacellsegmentator/VERSION b/hpacellsegmentator/VERSION new file mode 100644 index 0000000..1cc5f65 --- /dev/null +++ b/hpacellsegmentator/VERSION @@ -0,0 +1 @@ +1.1.0 \ No newline at end of file diff --git a/hpacellsegmentator/__init__.py b/hpacellsegmentator/__init__.py new file mode 100644 index 0000000..1d2e191 --- /dev/null +++ b/hpacellsegmentator/__init__.py @@ -0,0 +1,6 @@ +"""HPA Cell Segmentator Portable: a repackaging of HPA Cell Segmentator with +updated libraries and simplified usage.""" + +from pathlib import Path + +__version__ = (Path(__file__).parent / "VERSION").read_text().strip() \ No newline at end of file diff --git a/generate_cell_crops.py b/hpacellsegmentator/generate_cell_crops.py similarity index 99% rename from generate_cell_crops.py rename to hpacellsegmentator/generate_cell_crops.py index e4ad4b0..43517ed 100644 --- a/generate_cell_crops.py +++ b/hpacellsegmentator/generate_cell_crops.py @@ -4,7 +4,7 @@ from skimage.measure import regionprops from scipy.ndimage import grey_dilation import pandas as pd -import image_utils +from hpacellsegmentator import image_utils warnings.simplefilter(action="ignore", category=FutureWarning) diff --git a/generate_masks.py b/hpacellsegmentator/generate_masks.py similarity index 93% rename from generate_masks.py rename to hpacellsegmentator/generate_masks.py index 4ed10d8..8452943 100644 --- a/generate_masks.py +++ b/hpacellsegmentator/generate_masks.py @@ -1,7 +1,7 @@ import os import warnings import cv2 -from hpacellseg.utils import label_cell +from hpacellsegmentator.hpacellseg.utils import label_cell warnings.simplefilter(action="ignore", category=FutureWarning) diff --git a/hpacellseg/VERSION b/hpacellsegmentator/hpacellseg/VERSION similarity index 100% rename from hpacellseg/VERSION rename to hpacellsegmentator/hpacellseg/VERSION diff --git a/hpacellseg/__init__.py b/hpacellsegmentator/hpacellseg/__init__.py similarity index 100% rename from hpacellseg/__init__.py rename to hpacellsegmentator/hpacellseg/__init__.py diff --git a/hpacellseg/__main__.py b/hpacellsegmentator/hpacellseg/__main__.py similarity index 84% rename from hpacellseg/__main__.py rename to hpacellsegmentator/hpacellseg/__main__.py index ef0341a..33cbd74 100644 --- a/hpacellseg/__main__.py +++ b/hpacellsegmentator/hpacellseg/__main__.py @@ -1,7 +1,5 @@ -import click -from hpacellseg import __version__ -from hpacellseg.cellsegmentator import CellSegmentator -from hpacellseg.utils import label_nuclei, label_cell +from hpacellsegmentator.hpacellseg.cellsegmentator import CellSegmentator +from hpacellsegmentator.hpacellseg.utils import label_nuclei, label_cell def main(images=None): diff --git a/hpacellseg/cellsegmentator.py b/hpacellsegmentator/hpacellseg/cellsegmentator.py similarity index 98% rename from hpacellseg/cellsegmentator.py rename to hpacellsegmentator/hpacellseg/cellsegmentator.py index f18b60b..3c9dc36 100644 --- a/hpacellseg/cellsegmentator.py +++ b/hpacellsegmentator/hpacellseg/cellsegmentator.py @@ -10,9 +10,9 @@ import torch.nn.functional as F from skimage import transform, util -from hpacellseg.constants import (MULTI_CHANNEL_CELL_MODEL_URL, - NUCLEI_MODEL_URL, TWO_CHANNEL_CELL_MODEL_URL) -from hpacellseg.utils import download_with_url +from hpacellsegmentator.hpacellseg.constants import (MULTI_CHANNEL_CELL_MODEL_URL, + NUCLEI_MODEL_URL, TWO_CHANNEL_CELL_MODEL_URL) +from hpacellsegmentator.hpacellseg.utils import download_with_url NORMALIZE = {"mean": [124 / 255, 117 / 255, 104 / 255], "std": [1 / (0.0167 * 255)] * 3} diff --git a/hpacellseg/constants.py b/hpacellsegmentator/hpacellseg/constants.py similarity index 100% rename from hpacellseg/constants.py rename to hpacellsegmentator/hpacellseg/constants.py diff --git a/hpacellseg/utils.py b/hpacellsegmentator/hpacellseg/utils.py similarity index 100% rename from hpacellseg/utils.py rename to hpacellsegmentator/hpacellseg/utils.py diff --git a/image_utils.py b/hpacellsegmentator/image_utils.py similarity index 100% rename from image_utils.py rename to hpacellsegmentator/image_utils.py diff --git a/hpacellsegmentator/process.py b/hpacellsegmentator/process.py new file mode 100644 index 0000000..c58abed --- /dev/null +++ b/hpacellsegmentator/process.py @@ -0,0 +1,114 @@ +import argparse +import datetime +import logging +import os +import yaml +import pandas as pd + +from hpacellsegmentator import generate_masks, generate_cell_crops, image_utils +import hpacellsegmentator.hpacellseg.cellsegmentator as cellsegmentator + + +def main(): + # This is the log configuration. It will log everything to a file AND the console + logging.basicConfig( + filename="log.txt", + encoding="utf-8", + format="%(levelname)s: %(message)s", + filemode="w", + level=logging.INFO, + ) + console = logging.StreamHandler() + logging.getLogger().addHandler(console) + logger = logging.getLogger("HPACellSegmentatorPortable") + + # This is the general configuration variable. We are going to use the special key "log" in the dictionary to use the log in our code + config = {"log": logger} + + # If you want to use constants with your script, add them here + config["crop_cells"] = True + config["crop_size"] = 1024 + config["crop_bitdepth"] = 8 + config["crop_mask"] = True + config["mask_cell"] = True + + # If you want to use a configuration file with your script, add it here + with open("config.yaml", "r") as file: + config_contents = yaml.safe_load(file) + if config_contents: + config = config | config_contents + + # If you want to use command line parameters with your script, add them here. + # Do NOT set defaults here: the defaults live in the constants/config.yaml layers above. + # We only merge the arguments the user actually passed (value is not None), so the + # priority ends up being constants < config.yaml < command line arguments. + argparser = argparse.ArgumentParser(description="Please input the following parameters") + argparser.add_argument("-c", "--crop_cells", help="if you want to generate the crops of the cells detected in the segmentation", action=argparse.BooleanOptionalAction) + argparser.add_argument("-cs", "--crop_size", help="the cell crop size", type=int) + argparser.add_argument("-cb", "--crop_bitdepth", help="the cell crop bitdepth", type=int) + argparser.add_argument("-cm", "--crop_mask", help="if you want to also generate the crop binary mask from the segmentation", action=argparse.BooleanOptionalAction) + argparser.add_argument("-mc", "--mask_cell", help="if you want additional crops with only the segmented cell area", action=argparse.BooleanOptionalAction) + args = argparser.parse_args() + config = config | {key: value for key, value in vars(args).items() if value is not None} + + # Log the start time and the final configuration so you can keep track of what you did + config["log"].info("Start: " + datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")) + config["log"].info("Parameters used:") + config["log"].info(config) + config["log"].info("----------") + + # We load the CellSegmentator model. A failure here aborts the whole run, as there + # is nothing we can process without the model. + try: + segmentator = cellsegmentator.CellSegmentator( + "./models/dpn_unet_nuclei_v1.pth", + "./models/dpn_unet_cell_3ch_v1.pth", + device="cuda", padding=True, multi_channel_model=True) + except Exception as e: + config["log"].error("- Could not load the segmentation model: " + str(e)) + raise + + # if we want to generate the crops, we are going to keep their information in a CSV file + if config["crop_cells"]: + df = pd.DataFrame(columns=['id', 'cell', 'x1', 'y1', 'x2', 'y2']) + + # We iterate over each set of images to process. Each set is processed inside its own + # try/except so a single failing FOV is logged and skipped instead of aborting the batch. + if os.path.exists("./path_list.csv"): + with open("./path_list.csv", "r") as path_list: + for curr_set in path_list: + + if curr_set.strip() != "" and not curr_set.startswith("#"): + try: + curr_set_arr = curr_set.split(",") + # We create the output folder + os.makedirs(curr_set_arr[4].strip(), exist_ok=True) + # We load the images as numpy arrays + image_stack = [] + image_stack.append([image_utils.read_grayscale_image(curr_set_arr[0].strip())]) + image_stack.append([image_utils.read_grayscale_image(curr_set_arr[1].strip())]) + image_stack.append([image_utils.read_grayscale_image(curr_set_arr[2].strip())]) + + # We run the model + nuclei_mask, cell_mask = generate_masks.create_masks(segmentator, image_stack, curr_set_arr[4].strip(), curr_set_arr[6].strip()) + # Single cell crops + if config["crop_cells"]: + os.makedirs(curr_set_arr[5].strip(), exist_ok=True) + image_stack.append([image_utils.read_grayscale_image(curr_set_arr[3].strip())]) + cell_bbox_df = generate_cell_crops.generate_crops(image_stack, cell_mask, nuclei_mask, config["crop_size"], config["crop_bitdepth"], config["crop_mask"], config["mask_cell"], curr_set_arr[5].strip(), curr_set_arr[6].strip()) + df = pd.concat([df, cell_bbox_df], ignore_index=True) + + config["log"].info("- Saved results for " + curr_set_arr[6].strip()) + except Exception as e: + config["log"].error("- Failed to process '" + curr_set.strip() + "': " + str(e)) + continue + + # We store the cell crops bboxes and ids for easy localization + if config["crop_cells"]: + df.to_csv("crop_info.csv", index=False) + + config["log"].info("----------") + config["log"].info("End: " + datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/process.py b/process.py deleted file mode 100644 index d59c1d7..0000000 --- a/process.py +++ /dev/null @@ -1,112 +0,0 @@ -import argparse -import datetime -import logging -import os -import yaml -import pandas as pd - -import generate_masks -import generate_cell_crops -import image_utils -import hpacellseg.cellsegmentator as cellsegmentator - - -# This is the log configuration. It will log everything to a file AND the console -logging.basicConfig( - filename="log.txt", - encoding="utf-8", - format="%(levelname)s: %(message)s", - filemode="w", - level=logging.INFO, -) -console = logging.StreamHandler() -logging.getLogger().addHandler(console) -logger = logging.getLogger("HPACellSegmentatorPortable") - -# This is the general configuration variable. We are going to use the special key "log" in the dictionary to use the log in our code -config = {"log": logger} - -# If you want to use constants with your script, add them here -config["crop_cells"] = True -config["crop_size"] = 1024 -config["crop_bitdepth"] = 8 -config["crop_mask"] = True -config["mask_cell"] = True - -# If you want to use a configuration file with your script, add it here -with open("config.yaml", "r") as file: - config_contents = yaml.safe_load(file) - if config_contents: - config = config | config_contents - -# If you want to use command line parameters with your script, add them here. -# Do NOT set defaults here: the defaults live in the constants/config.yaml layers above. -# We only merge the arguments the user actually passed (value is not None), so the -# priority ends up being constants < config.yaml < command line arguments. -argparser = argparse.ArgumentParser(description="Please input the following parameters") -argparser.add_argument("-c", "--crop_cells", help="if you want to generate the crops of the cells detected in the segmentation", action=argparse.BooleanOptionalAction) -argparser.add_argument("-cs", "--crop_size", help="the cell crop size", type=int) -argparser.add_argument("-cb", "--crop_bitdepth", help="the cell crop bitdepth", type=int) -argparser.add_argument("-cm", "--crop_mask", help="if you want to also generate the crop binary mask from the segmentation", action=argparse.BooleanOptionalAction) -argparser.add_argument("-mc", "--mask_cell", help="if you want additional crops with only the segmented cell area", action=argparse.BooleanOptionalAction) -args = argparser.parse_args() -config = config | {key: value for key, value in vars(args).items() if value is not None} - -# Log the start time and the final configuration so you can keep track of what you did -config["log"].info("Start: " + datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")) -config["log"].info("Parameters used:") -config["log"].info(config) -config["log"].info("----------") - -# We load the CellSegmentator model. A failure here aborts the whole run, as there -# is nothing we can process without the model. -try: - segmentator = cellsegmentator.CellSegmentator( - "./models/dpn_unet_nuclei_v1.pth", - "./models/dpn_unet_cell_3ch_v1.pth", - device="cuda", padding=True, multi_channel_model=True) -except Exception as e: - config["log"].error("- Could not load the segmentation model: " + str(e)) - raise - -# if we want to generate the crops, we are going to keep their information in a CSV file -if config["crop_cells"]: - df = pd.DataFrame(columns=['id', 'cell', 'x1', 'y1', 'x2', 'y2']) - -# We iterate over each set of images to process. Each set is processed inside its own -# try/except so a single failing FOV is logged and skipped instead of aborting the batch. -if os.path.exists("./path_list.csv"): - with open("./path_list.csv", "r") as path_list: - for curr_set in path_list: - - if curr_set.strip() != "" and not curr_set.startswith("#"): - try: - curr_set_arr = curr_set.split(",") - # We create the output folder - os.makedirs(curr_set_arr[4].strip(), exist_ok=True) - # We load the images as numpy arrays - image_stack = [] - image_stack.append([image_utils.read_grayscale_image(curr_set_arr[0].strip())]) - image_stack.append([image_utils.read_grayscale_image(curr_set_arr[1].strip())]) - image_stack.append([image_utils.read_grayscale_image(curr_set_arr[2].strip())]) - - # We run the model - nuclei_mask, cell_mask = generate_masks.create_masks(segmentator, image_stack, curr_set_arr[4].strip(), curr_set_arr[6].strip()) - # Single cell crops - if config["crop_cells"]: - os.makedirs(curr_set_arr[5].strip(), exist_ok=True) - image_stack.append([image_utils.read_grayscale_image(curr_set_arr[3].strip())]) - cell_bbox_df = generate_cell_crops.generate_crops(image_stack, cell_mask, nuclei_mask, config["crop_size"], config["crop_bitdepth"], config["crop_mask"], config["mask_cell"], curr_set_arr[5].strip(), curr_set_arr[6].strip()) - df = pd.concat([df, cell_bbox_df], ignore_index=True) - - config["log"].info("- Saved results for " + curr_set_arr[6].strip()) - except Exception as e: - config["log"].error("- Failed to process '" + curr_set.strip() + "': " + str(e)) - continue - - # We store the cell crops bboxes and ids for easy localization - if config["crop_cells"]: - df.to_csv("crop_info.csv", index=False) - -config["log"].info("----------") -config["log"].info("End: " + datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..8c831b4 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,43 @@ +[build-system] +requires = ["setuptools>=77"] +build-backend = "setuptools.build_meta" + +[project] +name = "hpacellsegmentatorportable" +dynamic = ["version"] +description = "A repackage of HPA Cell Segmentator with updated libraries and simplified usage." +readme = "README.md" +requires-python = ">=3.12" +classifiers = [ + "Programming Language :: Python :: 3", + "Topic :: Scientific/Engineering :: Image Processing", + "Topic :: Scientific/Engineering :: Bio-Informatics", +] + +dependencies = [ + "imageio==2.34.0", + "numpy==1.26.4", + "opencv-python==4.9.0.80", + "scikit-image==0.22.0", + "torch==2.2.0", + "torchvision==0.17.0", + "PyYAML==6.0.1", + "pandas==2.2.3", + "imagecodecs==2024.12.30", +] + +[project.scripts] +hpacellsegmentator-batch = "hpacellsegmentator.process:main" + +[tool.setuptools.dynamic] +version = {attr = "hpacellsegmentator.__version__"} + +# pytorch_zoo must stay a top-level package: the .pth model files are pickled +# whole objects that record their classes as `pytorch_zoo.*`, so torch.load +# (weights_only=False) resolves those import paths at load time. +[tool.setuptools.packages.find] +include = ["hpacellsegmentator*", "pytorch_zoo*"] +namespaces = false + +[tool.setuptools.package-data] +hpacellseg = ["VERSION"] \ No newline at end of file From 999b9975077399b6b93f5371488a7f8638b9f023 Mon Sep 17 00:00:00 2001 From: Fredric Johansson Date: Fri, 17 Jul 2026 14:36:59 +0200 Subject: [PATCH 2/2] Add missing hpacellseg/VERSION --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 8c831b4..f542aee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,4 +40,4 @@ include = ["hpacellsegmentator*", "pytorch_zoo*"] namespaces = false [tool.setuptools.package-data] -hpacellseg = ["VERSION"] \ No newline at end of file +hpacellsegmentator = ["VERSION", "hpacellseg/VERSION"] \ No newline at end of file