Make a package of the project

.gitignore

@@ -0,0 +1,14 @@
+*.png
+log.txt
+crop_info.csv
+
+*.pyc
+*.pyo
+*.pth
+__pycache__/
+
+# Distribution / packaging
+build/
+dist/
+*.egg-info/
+.eggs/

MANIFEST.in

@@ -0,0 +1,15 @@
+include README.md
+include requirements.txt
+include config.yaml
+include hpacellseg/VERSION
+
+exclude path_list.csv
+exclude log.txt
+exclude crop_info.csv
+
+recursive-exclude models *
+recursive-exclude test *
+recursive-exclude * __pycache__
+recursive-exclude * *.py[co]
+recursive-exclude * *.pth
+recursive-exclude * *.tif

README.md

@@ -7,7 +7,17 @@ Just a repackage of the "HPA Cell Segmentator" repository with updated libraries
 Installation
 ------------
 
-If you use any Python IDE (VSCode, PyCharm, Spyder, etc...), just:
+**As a package (recommended).** From the repository root:
+
+- `pip install .` to install it, or `pip install -e .` for an editable/development install.
+
+This installs three importable top-level packages (`hpacellsegmentator`, `hpacellseg`, `pytorch_zoo`) and three commands:
+
+- `hpacellsegmentator`: segment a single set of images (see *Running the code*).
+- `hpacellsegmentator-batch`: process many image sets via `path_list.csv`.
+- `hpacellsegmentator-service`: run the HTTP segmentation service.
+
+**Manual/IDE setup.** If you use any Python IDE (VSCode, PyCharm, Spyder, etc...), just:
 - Either import the project into your IDE through git/github OR Create a new project and download all the repository code from github into it.
 - Create a virtual environment for that project.
 - Install the project requirements through your IDE. Make sure the packages versions match, as IDEs try to be too smart some times.
@@ -67,13 +77,16 @@ images/CACO-2_2047_C3_6_red.png,images/CACO-2_2047_C3_6_yellow.png,images/CACO-2
 images/U-215MG792_H7_2_red.png,images/U-215MG792_H7_2_yellow.png,images/U-215MG792_H7_2_blue.png,,output,,U-215MG792_H7_2_
 ```
 
-Once you have prepared your `path_list.csv` you are ready to run the `process.py` script. You can choose between 3 different running approaches, depending on your personal preferences:
+Once you have prepared your `path_list.csv` you are ready to run the batch script (`hpacellsegmentator/batch.py`, formerly `process.py`). It reads `path_list.csv` and `config.yaml` from the current working directory, so run it from the directory that holds them. You can choose between 3 different running approaches, depending on your personal preferences:
 
-- Edit directly the constants located in the `process.py` script:
+- Edit directly the constants located in the `hpacellsegmentator/batch.py` script:
   - Probably the least versatile, but useful if you are always running HPACellSegmentatorPortable with the same settings.
   - Just change the values under for the following section of code: `# If you want to use constants with your script, add them here` .
-  - Simply call `python process.py`.
+  - Simply call `hpacellsegmentator-batch`.
 
+- Call the batch script with arguments:
+  - You can get a list of available parameters (and their default values) using `-help` or `-?` argument.
+  - Example call: `hpacellsegmentator-batch -c True -cs 684`.
 - Call `process.py` script with arguments:
   - You can get a list of available parameters using the `-h` or `--help` argument.
   - Boolean options are toggled with a flag: use `-c`/`--crop_cells` to enable or `--no-crop_cells` to disable (likewise `-cm`/`--crop_mask` and `-mc`/`--mask_cell`).
@@ -82,7 +95,9 @@ Once you have prepared your `path_list.csv` you are ready to run the `process.py
 
 - Edit the `config.yaml` file:
   - Just change the contents of the file with your desired values.
-  - Simply call `python process.py`.
+  - Simply call `hpacellsegmentator-batch`.
+
+If you have not installed the package, `python -m hpacellsegmentator.batch` works the same way from the repository root.
 
 These approaches can be combined; when a parameter is set in more than one place, command line arguments take precedence over `config.yaml`, which in turn takes precedence over the constants in `process.py`.
 

hpacellsegmentator/VERSION

@@ -0,0 +1 @@
+1.1.0

hpacellsegmentator/__init__.py

@@ -0,0 +1,6 @@
+"""HPA Cell Segmentator Portable: a repackaging of HPA Cell Segmentator with
+updated libraries and simplified usage."""
+
+from pathlib import Path
+
+__version__ = (Path(__file__).parent / "VERSION").read_text().strip()

generate_cell_crops.py → hpacellsegmentator/generate_cell_crops.py

@@ -4,7 +4,7 @@
 from skimage.measure import regionprops
 from scipy.ndimage import grey_dilation
 import pandas as pd
-import image_utils
+from hpacellsegmentator import image_utils
 
 
 warnings.simplefilter(action="ignore", category=FutureWarning)

generate_masks.py → hpacellsegmentator/generate_masks.py

@@ -1,7 +1,7 @@
 import os
 import warnings
 import cv2
-from hpacellseg.utils import label_cell
+from hpacellsegmentator.hpacellseg.utils import label_cell
 
 
 warnings.simplefilter(action="ignore", category=FutureWarning)

hpacellseg/__main__.py → hpacellsegmentator/hpacellseg/__main__.py

@@ -1,7 +1,5 @@
-import click
-from hpacellseg import __version__
-from hpacellseg.cellsegmentator import CellSegmentator
-from hpacellseg.utils import label_nuclei, label_cell
+from hpacellsegmentator.hpacellseg.cellsegmentator import CellSegmentator
+from hpacellsegmentator.hpacellseg.utils import label_nuclei, label_cell
 
 
 def main(images=None):

hpacellseg/cellsegmentator.py → hpacellsegmentator/hpacellseg/cellsegmentator.py

@@ -10,9 +10,9 @@
 import torch.nn.functional as F
 from skimage import transform, util
 
-from hpacellseg.constants import (MULTI_CHANNEL_CELL_MODEL_URL,
-                                  NUCLEI_MODEL_URL, TWO_CHANNEL_CELL_MODEL_URL)
-from hpacellseg.utils import download_with_url
+from hpacellsegmentator.hpacellseg.constants import (MULTI_CHANNEL_CELL_MODEL_URL,
+                                                     NUCLEI_MODEL_URL, TWO_CHANNEL_CELL_MODEL_URL)
+from hpacellsegmentator.hpacellseg.utils import download_with_url
 
 NORMALIZE = {"mean": [124 / 255, 117 / 255, 104 / 255], "std": [1 / (0.0167 * 255)] * 3}
 

hpacellsegmentator/process.py

@@ -0,0 +1,114 @@
+import argparse
+import datetime
+import logging
+import os
+import yaml
+import pandas as pd
+
+from hpacellsegmentator import generate_masks, generate_cell_crops, image_utils
+import hpacellsegmentator.hpacellseg.cellsegmentator as cellsegmentator
+
+
+def main():
+    # This is the log configuration. It will log everything to a file AND the console
+    logging.basicConfig(
+        filename="log.txt",
+        encoding="utf-8",
+        format="%(levelname)s: %(message)s",
+        filemode="w",
+        level=logging.INFO,
+    )
+    console = logging.StreamHandler()
+    logging.getLogger().addHandler(console)
+    logger = logging.getLogger("HPACellSegmentatorPortable")
+
+    # This is the general configuration variable. We are going to use the special key "log" in the dictionary to use the log in our code
+    config = {"log": logger}
+
+    # If you want to use constants with your script, add them here
+    config["crop_cells"] = True
+    config["crop_size"] = 1024
+    config["crop_bitdepth"] = 8
+    config["crop_mask"] = True
+    config["mask_cell"] = True
+
+    # If you want to use a configuration file with your script, add it here
+    with open("config.yaml", "r") as file:
+        config_contents = yaml.safe_load(file)
+        if config_contents:
+            config = config | config_contents
+
+    # If you want to use command line parameters with your script, add them here.
+    # Do NOT set defaults here: the defaults live in the constants/config.yaml layers above.
+    # We only merge the arguments the user actually passed (value is not None), so the
+    # priority ends up being constants < config.yaml < command line arguments.
+    argparser = argparse.ArgumentParser(description="Please input the following parameters")
+    argparser.add_argument("-c", "--crop_cells", help="if you want to generate the crops of the cells detected in the segmentation", action=argparse.BooleanOptionalAction)
+    argparser.add_argument("-cs", "--crop_size", help="the cell crop size", type=int)
+    argparser.add_argument("-cb", "--crop_bitdepth", help="the cell crop bitdepth", type=int)
+    argparser.add_argument("-cm", "--crop_mask", help="if you want to also generate the crop binary mask from the segmentation", action=argparse.BooleanOptionalAction)
+    argparser.add_argument("-mc", "--mask_cell", help="if you want additional crops with only the segmented cell area", action=argparse.BooleanOptionalAction)
+    args = argparser.parse_args()
+    config = config | {key: value for key, value in vars(args).items() if value is not None}
+
+    # Log the start time and the final configuration so you can keep track of what you did
+    config["log"].info("Start: " + datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S"))
+    config["log"].info("Parameters used:")
+    config["log"].info(config)
+    config["log"].info("----------")
+
+    # We load the CellSegmentator model. A failure here aborts the whole run, as there
+    # is nothing we can process without the model.
+    try:
+        segmentator = cellsegmentator.CellSegmentator(
+            "./models/dpn_unet_nuclei_v1.pth",
+            "./models/dpn_unet_cell_3ch_v1.pth",
+            device="cuda", padding=True, multi_channel_model=True)
+    except Exception as e:
+        config["log"].error("- Could not load the segmentation model: " + str(e))
+        raise
+
+    # if we want to generate the crops, we are going to keep their information in a CSV file
+    if config["crop_cells"]:
+        df = pd.DataFrame(columns=['id', 'cell', 'x1', 'y1', 'x2', 'y2'])
+
+    # We iterate over each set of images to process. Each set is processed inside its own
+    # try/except so a single failing FOV is logged and skipped instead of aborting the batch.
+    if os.path.exists("./path_list.csv"):
+        with open("./path_list.csv", "r") as path_list:
+            for curr_set in path_list:
+
+                if curr_set.strip() != "" and not curr_set.startswith("#"):
+                    try:
+                        curr_set_arr = curr_set.split(",")
+                        # We create the output folder
+                        os.makedirs(curr_set_arr[4].strip(), exist_ok=True)
+                        # We load the images as numpy arrays
+                        image_stack = []
+                        image_stack.append([image_utils.read_grayscale_image(curr_set_arr[0].strip())])
+                        image_stack.append([image_utils.read_grayscale_image(curr_set_arr[1].strip())])
+                        image_stack.append([image_utils.read_grayscale_image(curr_set_arr[2].strip())])
+
+                        # We run the model
+                        nuclei_mask, cell_mask = generate_masks.create_masks(segmentator, image_stack, curr_set_arr[4].strip(), curr_set_arr[6].strip())
+                        # Single cell crops
+                        if config["crop_cells"]:
+                            os.makedirs(curr_set_arr[5].strip(), exist_ok=True)
+                            image_stack.append([image_utils.read_grayscale_image(curr_set_arr[3].strip())])
+                            cell_bbox_df = generate_cell_crops.generate_crops(image_stack, cell_mask, nuclei_mask, config["crop_size"], config["crop_bitdepth"], config["crop_mask"], config["mask_cell"], curr_set_arr[5].strip(), curr_set_arr[6].strip())
+                            df = pd.concat([df, cell_bbox_df], ignore_index=True)
+
+                        config["log"].info("- Saved results for " + curr_set_arr[6].strip())
+                    except Exception as e:
+                        config["log"].error("- Failed to process '" + curr_set.strip() + "': " + str(e))
+                        continue
+
+        # We store the cell crops bboxes and ids for easy localization
+        if config["crop_cells"]:
+            df.to_csv("crop_info.csv", index=False)
+
+    config["log"].info("----------")
+    config["log"].info("End: " + datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S"))
+
+if __name__ == "__main__":
+    main()