From d45222d54b592e8595abd722e0862558061570f3 Mon Sep 17 00:00:00 2001 From: hamad45 Date: Fri, 27 Jan 2023 13:22:37 -0800 Subject: [PATCH 01/58] DXT Explorer v2.0 --- .github/workflows/explorer.yml | 70 +- Dockerfile | 26 +- MANIFEST.in | 8 +- README.md | 4 + explorer/plots/__init__.py => __init__.py | 0 docs/build/html/building.html | 14 +- docs/build/html/citation.html | 12 +- docs/build/html/copyright.html | 12 +- docs/build/html/exploring.html | 16 +- docs/build/html/genindex.html | 12 +- docs/build/html/index.html | 19 +- docs/build/html/instructions.html | 14 +- docs/build/html/license.html | 14 +- docs/build/html/search.html | 10 +- docs/source/building.rst | 4 +- docs/source/copyright.rst | 2 +- docs/source/exploring.rst | 77 +- docs/source/index.rst | 12 +- docs/source/license.rst | 2 +- explorer/dependencies.R | 25 - explorer/dxt.py | 1531 +++++++++++++++------ explorer/insights.py | 401 ++++++ explorer/plots/dxt-explorer.png | Bin 18275 -> 0 bytes explorer/plots/io_phase.py | 189 +++ explorer/plots/lustre.R | 151 -- explorer/plots/operation.R | 623 --------- explorer/plots/operation.py | 902 ++++++++++++ explorer/plots/ost_usage_operation.py | 200 +++ explorer/plots/ost_usage_transfer.py | 214 +++ explorer/plots/overlap.R | 186 --- explorer/plots/overview.R | 142 -- explorer/plots/space-time.R | 137 -- explorer/plots/spatiality.R | 256 ---- explorer/plots/spatiality.py | 196 +++ explorer/plots/transfer.R | 617 --------- explorer/plots/transfer.py | 347 +++++ explorer/version.py | 4 +- install-r-libraries.R | 15 - requirements.txt | 5 + sample.json | 1 + setup.py | 35 +- 41 files changed, 3849 insertions(+), 2656 deletions(-) rename explorer/plots/__init__.py => __init__.py (100%) delete mode 100644 explorer/dependencies.R mode change 100755 => 100644 explorer/dxt.py create mode 100644 explorer/insights.py delete mode 100644 explorer/plots/dxt-explorer.png create mode 100644 explorer/plots/io_phase.py delete mode 100755 explorer/plots/lustre.R delete mode 100755 explorer/plots/operation.R create mode 100644 explorer/plots/operation.py create mode 100644 explorer/plots/ost_usage_operation.py create mode 100644 explorer/plots/ost_usage_transfer.py delete mode 100644 explorer/plots/overlap.R delete mode 100755 explorer/plots/overview.R delete mode 100755 explorer/plots/space-time.R delete mode 100755 explorer/plots/spatiality.R create mode 100644 explorer/plots/spatiality.py delete mode 100755 explorer/plots/transfer.R create mode 100644 explorer/plots/transfer.py delete mode 100644 install-r-libraries.R create mode 100644 sample.json diff --git a/.github/workflows/explorer.yml b/.github/workflows/explorer.yml index 551b33e..7993493 100644 --- a/.github/workflows/explorer.yml +++ b/.github/workflows/explorer.yml @@ -42,10 +42,6 @@ jobs: pip install -r requirements.txt pip install . - - name: Install R Dependencies - run: | - Rscript --save explorer/dependencies.R - - name: Run DXT Explorer (help) run: | export PATH=/opt/darshan/bin:$PATH @@ -76,6 +72,72 @@ jobs: dxt-explorer --debug --spatiality sample/jeanbez_8_benchmark_parallel_id45195555_8-6-50011-11681279140261054765_1628283217.darshan + - name: Run DXT Explorer (io phase) + run: | + export PATH=/opt/darshan/bin:$PATH + + dxt-explorer --debug --io_phase sample/jeanbez_8_benchmark_parallel_id45195555_8-6-50011-11681279140261054765_1628283217.darshan + + - name: Run DXT Explorer (ost usage operation) + run: | + export PATH=/opt/darshan/bin:$PATH + + dxt-explorer --debug --ost_usage_operation sample/jeanbez_8_benchmark_parallel_id45195555_8-6-50011-11681279140261054765_1628283217.darshan + + - name: Run DXT Explorer (ost usage transfer) + run: | + export PATH=/opt/darshan/bin:$PATH + + dxt-explorer --debug --ost_usage_transfer sample/jeanbez_8_benchmark_parallel_id45195555_8-6-50011-11681279140261054765_1628283217.darshan + + - name: Run DXT Explorer (rank zero workload) + run: | + export PATH=/opt/darshan/bin:$PATH + + dxt-explorer --debug --rank_zero_workload sample/jeanbez_8_benchmark_parallel_id45195555_8-6-50011-11681279140261054765_1628283217.darshan + + - name: Run DXT Explorer (unbalanced workload) + run: | + export PATH=/opt/darshan/bin:$PATH + + dxt-explorer --debug --unbalanced_workload sample/jeanbez_8_benchmark_parallel_id45195555_8-6-50011-11681279140261054765_1628283217.darshan + + - name: Run DXT Explorer (stragglers) + run: | + export PATH=/opt/darshan/bin:$PATH + + dxt-explorer --debug --stragglers sample/jeanbez_8_benchmark_parallel_id45195555_8-6-50011-11681279140261054765_1628283217.darshan + + - name: Run DXT Explorer (Truncate Runtime) + run: | + export PATH=/opt/darshan/bin:$PATH + + dxt-explorer --debug --start 3.7 sample/jeanbez_8_benchmark_parallel_id45195555_8-6-50011-11681279140261054765_1628283217.darshan + + - name: Run DXT Explorer (Truncate Runtime) + run: | + export PATH=/opt/darshan/bin:$PATH + + dxt-explorer --debug --end 3.9 sample/jeanbez_8_benchmark_parallel_id45195555_8-6-50011-11681279140261054765_1628283217.darshan + + - name: Run DXT Explorer (Truncate Rank) + run: | + export PATH=/opt/darshan/bin:$PATH + + dxt-explorer --debug --from 1 sample/jeanbez_8_benchmark_parallel_id45195555_8-6-50011-11681279140261054765_1628283217.darshan + + - name: Run DXT Explorer (Truncate Rank) + run: | + export PATH=/opt/darshan/bin:$PATH + + dxt-explorer --debug --to 100 sample/jeanbez_8_benchmark_parallel_id45195555_8-6-50011-11681279140261054765_1628283217.darshan + + - name: Run DXT Explorer (Truncate Runtime & Rank) + run: | + export PATH=/opt/darshan/bin:$PATH + + dxt-explorer --debug --start 3.7 --end 3.9 --from 1 --to 100 sample/jeanbez_8_benchmark_parallel_id45195555_8-6-50011-11681279140261054765_1628283217.darshan + - name: Upload Artifact if: always() uses: actions/upload-artifact@v2 diff --git a/Dockerfile b/Dockerfile index 24aedd5..335ec32 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,31 +9,31 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get install -y \ wget \ libssl-dev \ libcurl4-openssl-dev \ - r-base \ - r-cran-curl \ - r-cran-openssl \ - r-cran-httr \ - r-cran-plotly \ - r-cran-cairo + autoconf \ + zlib1g \ + libtool \ + r-base RUN rm -rf /var/lib/apt/lists/* +RUN pip install matplotlib==3.5.1 -RUN wget ftp://ftp.mcs.anl.gov/pub/darshan/releases/darshan-3.3.1.tar.gz -RUN tar zxvf darshan-3.3.1.tar.gz -WORKDIR /darshan-3.3.1/darshan-util/ +RUN wget ftp://ftp.mcs.anl.gov/pub/darshan/releases/darshan-3.4.0.tar.gz +RUN tar zxvf darshan-3.4.0.tar.gz +WORKDIR /darshan-3.4.0/ +RUN ./prepare.sh -RUN ./configure +WORKDIR /darshan-3.4.0/darshan-util/ +RUN ./configure --enable-pydarshan RUN make RUN make install WORKDIR / -RUN git clone https://github.com/hpc-io/dxt-explorer +RUN git clone https://github.com/hpc-io/dxt-explorer-2 -WORKDIR /dxt-explorer +WORKDIR /dxt-explorer-2 RUN pip install --upgrade pip RUN pip install -r requirements.txt -RUN Rscript install-r-libraries.R RUN pip install . ENTRYPOINT ["dxt-explorer"] diff --git a/MANIFEST.in b/MANIFEST.in index af8b142..81a2677 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,6 +1,4 @@ include requirements.txt -include dxt-explorer.png -include explorer/*.* -include explorer/plots/*.R -include explorer/plots/*.png -include explorer/plots/*.html \ No newline at end of file +include explorer/plots/*.py +include explorer/plots/*.html +include explorer/plots/*.png \ No newline at end of file diff --git a/README.md b/README.md index d5b4a56..558da89 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,7 @@ +```code +Notice that is a private repository for the ungoing work to upgrade DXT Explorer. +``` +

DXT Explorer

diff --git a/explorer/plots/__init__.py b/__init__.py similarity index 100% rename from explorer/plots/__init__.py rename to __init__.py diff --git a/docs/build/html/building.html b/docs/build/html/building.html index 7ac3105..5d3b804 100644 --- a/docs/build/html/building.html +++ b/docs/build/html/building.html @@ -43,7 +43,7 @@ - + @@ -96,7 +96,17 @@
  • Docker Image
    • +
  • Interactive Plots
  • Exploring
    • +
  • Operation Plot
    • +
  • Transfer Plot
    • +
  • Spatiality Plot
    • +
  • Rank Zero Heavy Workload Plot
    • +
  • Unbalanced Workload Plot
    • +
  • Stragglers Plot
    • +
  • IO Phase Plot
    • +
  • OST Usage Operation Plot
    • +
  • OST Usage Transfer Plot

    • Contribute