From 4f4b1731ef1355f6d17346baca2b23a687da5fee Mon Sep 17 00:00:00 2001 From: sev89498 Date: Thu, 23 Feb 2023 13:39:03 +0000 Subject: [PATCH 1/4] Containerization support --- 3d_cnn/environment.yaml | 18 ++++++++++-------- Dockerfile-2d_cnn | 25 +++++++++++++++++++++++++ Dockerfile-3d_cnn | 25 +++++++++++++++++++++++++ 3 files changed, 60 insertions(+), 8 deletions(-) create mode 100644 Dockerfile-2d_cnn create mode 100644 Dockerfile-3d_cnn diff --git a/3d_cnn/environment.yaml b/3d_cnn/environment.yaml index 3a9b5ec..a04acb1 100644 --- a/3d_cnn/environment.yaml +++ b/3d_cnn/environment.yaml @@ -1,25 +1,27 @@ name: 3d-unet channels: + - pytorch - bioconda - - anaconda - conda-forge + - anaconda - default dependencies: - python=3.7 - jupyter - imageio - - pytorch - - torchvision - scikit-learn - scikit-image - scipy - numpy - - tensorboardX - tqdm - pandas - pyaml - pip - - pip: - - h5py==2.10.0 - - mrcfile==1.1.2 - - monai + - h5py==2.10.0 + - mrcfile==1.1.2 + - monai + - pytorch==1.12.1 + - torchvision==0.13.1 + - cudatoolkit=11.6 + - tensorboardx + - crc32c diff --git a/Dockerfile-2d_cnn b/Dockerfile-2d_cnn new file mode 100644 index 0000000..97625ed --- /dev/null +++ b/Dockerfile-2d_cnn @@ -0,0 +1,25 @@ +FROM nvidia/cuda:10.0-devel-ubuntu18.04 + +WORKDIR /app/DeePiCt +COPY . . + +# Install necessary packages +RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections && \ + apt-get update -y && apt-get install -y dialog apt-utils && \ + apt-get install -y build-essential git wget software-properties-common && \ + apt-get update && rm -rf /var/lib/apt/lists/* + +# Install miniconda +ENV PATH="/usr/local/miniconda3/bin:${PATH}" +ARG PATH="/usr/local/miniconda3/bin:${PATH}" +RUN cd /tmp \ + && wget \ + https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O Miniconda3.sh\ + && bash Miniconda3.sh -b -p /usr/local/miniconda3 \ + && rm -f Miniconda3.sh + + +# Create environment +RUN conda init bash && \ + conda install -c conda-forge mamba --yes && \ + mamba env create -f /app/DeePiCt/2d_cnn/envs/keras-env.yaml diff --git a/Dockerfile-3d_cnn b/Dockerfile-3d_cnn new file mode 100644 index 0000000..d6a4957 --- /dev/null +++ b/Dockerfile-3d_cnn @@ -0,0 +1,25 @@ +FROM nvidia/cuda:11.6.2-devel-ubuntu20.04 + +WORKDIR /app/DeePiCt +COPY . . + +# Install necessary packages +RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections && \ + apt-get update -y && apt-get install -y dialog apt-utils && \ + apt-get install -y build-essential git wget software-properties-common && \ + apt-get update && rm -rf /var/lib/apt/lists/* + +# Install miniconda +ENV PATH="/usr/local/miniconda3/bin:${PATH}" +ARG PATH="/usr/local/miniconda3/bin:${PATH}" +RUN cd /tmp \ + && wget \ + https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O Miniconda3.sh\ + && bash Miniconda3.sh -b -p /usr/local/miniconda3 \ + && rm -f Miniconda3.sh + + +# Create environment +RUN conda init bash && \ + conda install -c conda-forge mamba --yes && \ + mamba env create -f /app/DeePiCt/3d_cnn/environment.yaml \ From b0673aed77965ebb7c334b69cc35967e1bdd395d Mon Sep 17 00:00:00 2001 From: sev89498 Date: Thu, 23 Feb 2023 13:56:05 +0000 Subject: [PATCH 2/4] Containerization support --- Dockerfile-3d_cnn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile-3d_cnn b/Dockerfile-3d_cnn index d6a4957..115e809 100644 --- a/Dockerfile-3d_cnn +++ b/Dockerfile-3d_cnn @@ -22,4 +22,4 @@ RUN cd /tmp \ # Create environment RUN conda init bash && \ conda install -c conda-forge mamba --yes && \ - mamba env create -f /app/DeePiCt/3d_cnn/environment.yaml \ + mamba env create -f /app/DeePiCt/3d_cnn/environment.yaml From fe2ed85b5bd810cb0d573ba469c70c98ab422573 Mon Sep 17 00:00:00 2001 From: sev89498 Date: Mon, 27 Feb 2023 11:40:59 +0000 Subject: [PATCH 3/4] Inclusion of snakemake --- 3d_cnn/environment.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/3d_cnn/environment.yaml b/3d_cnn/environment.yaml index a04acb1..0f4faf4 100644 --- a/3d_cnn/environment.yaml +++ b/3d_cnn/environment.yaml @@ -25,3 +25,4 @@ dependencies: - cudatoolkit=11.6 - tensorboardx - crc32c + - snakemake From dcd156b5cefc05d6b800742b5f12ef0ca1f1a99d Mon Sep 17 00:00:00 2001 From: sev89498 Date: Wed, 1 Mar 2023 16:06:26 +0000 Subject: [PATCH 4/4] Conda environment fix --- 2d_cnn/deploy_config.sh | 5 +++++ 2d_cnn/deploy_local_2D_container.sh | 23 +++++++++++++++++++++++ 3d_cnn/deploy_config.sh | 5 +++++ 3d_cnn/deploy_local_3D_container.sh | 24 ++++++++++++++++++++++++ 3d_cnn/environment.yaml | 2 +- Dockerfile-2d_cnn | 10 +++++++++- Dockerfile-3d_cnn | 12 +++++++++++- 7 files changed, 78 insertions(+), 3 deletions(-) create mode 100755 2d_cnn/deploy_config.sh create mode 100755 2d_cnn/deploy_local_2D_container.sh create mode 100755 3d_cnn/deploy_config.sh create mode 100755 3d_cnn/deploy_local_3D_container.sh diff --git a/2d_cnn/deploy_config.sh b/2d_cnn/deploy_config.sh new file mode 100755 index 0000000..33c5446 --- /dev/null +++ b/2d_cnn/deploy_config.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +srcdir=$(dirname ${BASH_SOURCE[0]}) +config_file=$1 +cp "${srcdir}/config.yaml" $(pwd)/"${config_file}" diff --git a/2d_cnn/deploy_local_2D_container.sh b/2d_cnn/deploy_local_2D_container.sh new file mode 100755 index 0000000..2c41f98 --- /dev/null +++ b/2d_cnn/deploy_local_2D_container.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +srcdir=$(dirname ${BASH_SOURCE[0]}) +config_file=$1 +export CUDA_VISIBLE_DEVICES= +for i in $(seq 1 1 $(nvidia-smi --list-gpus | wc -l)) +do + if [ $i -eq 1 ] + then + export CUDA_VISIBLE_DEVICES=$(expr $i - 1) + else + export CUDA_VISIBLE_DEVICES=$CUDA_VISIBLE_DEVICES,$(expr $i - 1) + fi +done +echo $CUDA_VISIBLE_DEVICES + +snakemake \ + --snakefile "${srcdir}/snakefile.py" \ + --config config="${config_file}" \ + --forceall \ + --printshellcmds \ + --cores $(nvidia-smi --list-gpus | wc -l) \ + --resources gpu=$(nvidia-smi --list-gpus | wc -l) \ No newline at end of file diff --git a/3d_cnn/deploy_config.sh b/3d_cnn/deploy_config.sh new file mode 100755 index 0000000..33c5446 --- /dev/null +++ b/3d_cnn/deploy_config.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +srcdir=$(dirname ${BASH_SOURCE[0]}) +config_file=$1 +cp "${srcdir}/config.yaml" $(pwd)/"${config_file}" diff --git a/3d_cnn/deploy_local_3D_container.sh b/3d_cnn/deploy_local_3D_container.sh new file mode 100755 index 0000000..bec1a62 --- /dev/null +++ b/3d_cnn/deploy_local_3D_container.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +export srcdir="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" +config_file=$1 +export CUDA_VISIBLE_DEVICES= +for i in $(seq 1 1 $(nvidia-smi --list-gpus | wc -l)) +do + if [ $i -eq 1 ] + then + export CUDA_VISIBLE_DEVICES=$(expr $i - 1) + else + export CUDA_VISIBLE_DEVICES=$CUDA_VISIBLE_DEVICES,$(expr $i - 1) + fi +done +echo $CUDA_VISIBLE_DEVICES +export PYTHONPATH=${srcdir}/src +echo PYTHONPATH=$PYTHONPATH + +snakemake \ + --snakefile "${srcdir}/snakefile" \ + --config config="${config_file}" gpu=$CUDA_VISIBLE_DEVICES \ + --printshellcmds \ + --cores $(nvidia-smi --list-gpus | wc -l) \ + --resources gpu=$(nvidia-smi --list-gpus | wc -l) diff --git a/3d_cnn/environment.yaml b/3d_cnn/environment.yaml index 0f4faf4..696fde0 100644 --- a/3d_cnn/environment.yaml +++ b/3d_cnn/environment.yaml @@ -25,4 +25,4 @@ dependencies: - cudatoolkit=11.6 - tensorboardx - crc32c - - snakemake + - snakemake==5.13.0 diff --git a/Dockerfile-2d_cnn b/Dockerfile-2d_cnn index 97625ed..662c823 100644 --- a/Dockerfile-2d_cnn +++ b/Dockerfile-2d_cnn @@ -1,7 +1,8 @@ FROM nvidia/cuda:10.0-devel-ubuntu18.04 +COPY DeePiCt /app/DeePiCt +COPY deploy_local_2D_container.sh /app/DeePiCt/2d_cnn WORKDIR /app/DeePiCt -COPY . . # Install necessary packages RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections && \ @@ -23,3 +24,10 @@ RUN cd /tmp \ RUN conda init bash && \ conda install -c conda-forge mamba --yes && \ mamba env create -f /app/DeePiCt/2d_cnn/envs/keras-env.yaml + +RUN cd 2d_cnn && \ + ln -s deploy_local_2D_container.sh deepict2D && \ + ln -s deploy_config.sh deepict2D_config + +ENV PATH="${PATH}:/app/DeePiCt/2d_cnn" +ARG PATH="${PATH}:/app/DeePiCt/2d_cnn" diff --git a/Dockerfile-3d_cnn b/Dockerfile-3d_cnn index 115e809..8bce8f7 100644 --- a/Dockerfile-3d_cnn +++ b/Dockerfile-3d_cnn @@ -1,7 +1,10 @@ FROM nvidia/cuda:11.6.2-devel-ubuntu20.04 +COPY DeePiCt /app/DeePiCt +COPY environment.yaml /app/DeePiCt/3d_cnn +COPY deploy_config.sh /app/DeePiCt/3d_cnn +COPY deploy_local_3D_container.sh /app/DeePiCt/3d_cnn WORKDIR /app/DeePiCt -COPY . . # Install necessary packages RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections && \ @@ -23,3 +26,10 @@ RUN cd /tmp \ RUN conda init bash && \ conda install -c conda-forge mamba --yes && \ mamba env create -f /app/DeePiCt/3d_cnn/environment.yaml + +RUN cd 3d_cnn && \ + ln -s deploy_local_3D_container.sh deepict3D && \ + ln -s deploy_config.sh deepict3D_config + +ENV PATH="${PATH}:/app/DeePiCt/3d_cnn" +ARG PATH="${PATH}:/app/DeePiCt/3d_cnn"