diff --git a/auto_examples/auto_examples_jupyter.zip b/auto_examples/auto_examples_jupyter.zip index 9ffd7b62..4b8bd7bc 100644 Binary files a/auto_examples/auto_examples_jupyter.zip and b/auto_examples/auto_examples_jupyter.zip differ diff --git a/auto_examples/auto_examples_python.zip b/auto_examples/auto_examples_python.zip index ff0f6bed..1f33f110 100644 Binary files a/auto_examples/auto_examples_python.zip and b/auto_examples/auto_examples_python.zip differ diff --git a/auto_examples/example_job_submit_wait.ipynb b/auto_examples/example_job_submit_wait.ipynb new file mode 100644 index 00000000..b52ebfa3 --- /dev/null +++ b/auto_examples/example_job_submit_wait.ipynb @@ -0,0 +1,126 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "%matplotlib inline" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n# Introductory example - Job Submit and Wait\n\nTo programatically create and check the status for a group\nof jobs using the flux.job.FluxExecutor.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "import os\nimport concurrent.futures\nfrom flux.job import JobspecV1\nfrom flux.job import JobspecV1, FluxExecutor" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Instead of directly creating a flux handle by importing flux and doing flux.Flux(),\nthis time we are going to use the flux.job.FluxExecutor. This will allow us to submit\njobs and then asynchronously wait for them to finish. As we did before, let's start\nwith a jobspec for a sleep job. We are fairly certain this will run with a return\ncode of 0 to indicate success.\n\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "jobspec = JobspecV1.from_command(\n command=[\"sleep\", \"1\"], num_tasks=2, num_nodes=1, cores_per_task=1\n)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's again set the working directory and current environment.\n\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "jobspec.cwd = os.getcwd()\njobspec.environment = dict(os.environ)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To mix things up a bit, let's run a command that we know will fail. The false\ncommand always returns a value of 1. This is a \"bad\" jobspec that will fail!\n\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "bad_jobspec = JobspecV1.from_command([\"/bin/false\"])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now we will demonstrate using the FluxExecutor (via a context) to submit both good\nand bad jobs, and wait for them to finish. We call a job that is marked as completed\na \"future\" and can inspect error code and exceptions to see details about the results!\n\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "# create an executor to submit jobs\nwith FluxExecutor() as executor:\n\n # we will capture and keep each job future\n futures = []\n\n # submit half successful jobs\n for _ in range(5):\n futures.append(executor.submit(jobspec))\n print(f\"submit: {id(futures[-1])} (good) jobspec\")\n\n # and half failure jobs!\n for _ in range(5):\n futures.append(executor.submit(bad_jobspec))\n print(f\"submit: {id(futures[-1])} (bad) jobspec\")\n\n # We can now check on our job futures\n for future in concurrent.futures.as_completed(futures):\n\n # There was an exception!\n if future.exception() is not None:\n print(f\"\u26a0\ufe0f wait: {id(future)} Error: job raised error {future.exception()}\")\n\n # Successful result, return code is zero\n elif future.result() == 0:\n print(f\"\ud83c\udfc6\ufe0f wait: {id(future)} Success\")\n\n # Some other result\n else:\n print(f\"\u274c\ufe0f wait: {id(future)} Error: job returned exit code {future.result()}\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.10" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} \ No newline at end of file diff --git a/auto_examples/example_job_submit_wait.py b/auto_examples/example_job_submit_wait.py new file mode 100644 index 00000000..c5766911 --- /dev/null +++ b/auto_examples/example_job_submit_wait.py @@ -0,0 +1,72 @@ +# -*- coding: utf-8 -*- +""" +Introductory example - Job Submit and Wait +========================================== + +To programatically create and check the status for a group +of jobs using the flux.job.FluxExecutor. +""" + + +import os +import concurrent.futures +from flux.job import JobspecV1 +from flux.job import JobspecV1, FluxExecutor + + +#%% +# Instead of directly creating a flux handle by importing flux and doing flux.Flux(), +# this time we are going to use the flux.job.FluxExecutor. This will allow us to submit +# jobs and then asynchronously wait for them to finish. As we did before, let's start +# with a jobspec for a sleep job. We are fairly certain this will run with a return +# code of 0 to indicate success. + +jobspec = JobspecV1.from_command( + command=["sleep", "1"], num_tasks=2, num_nodes=1, cores_per_task=1 +) + +#%% +# Let's again set the working directory and current environment. +jobspec.cwd = os.getcwd() +jobspec.environment = dict(os.environ) + +#%% +# To mix things up a bit, let's run a command that we know will fail. The false +# command always returns a value of 1. This is a "bad" jobspec that will fail! +bad_jobspec = JobspecV1.from_command(["/bin/false"]) + +#%% +# Now we will demonstrate using the FluxExecutor (via a context) to submit both good +# and bad jobs, and wait for them to finish. We call a job that is marked as completed +# a "future" and can inspect error code and exceptions to see details about the results! + +# create an executor to submit jobs +with FluxExecutor() as executor: + + # we will capture and keep each job future + futures = [] + + # submit half successful jobs + for _ in range(5): + futures.append(executor.submit(jobspec)) + print(f"submit: {id(futures[-1])} (good) jobspec") + + # and half failure jobs! + for _ in range(5): + futures.append(executor.submit(bad_jobspec)) + print(f"submit: {id(futures[-1])} (bad) jobspec") + + # We can now check on our job futures + for future in concurrent.futures.as_completed(futures): + + # There was an exception! + if future.exception() is not None: + print(f"⚠️ wait: {id(future)} Error: job raised error {future.exception()}") + + # Successful result, return code is zero + elif future.result() == 0: + print(f"🏆️ wait: {id(future)} Success") + + # Some other result + else: + print(f"❌️ wait: {id(future)} Error: job returned exit code {future.result()}") \ No newline at end of file diff --git a/auto_examples/example_job_submit_wait.py.md5 b/auto_examples/example_job_submit_wait.py.md5 new file mode 100644 index 00000000..3eaf5cda --- /dev/null +++ b/auto_examples/example_job_submit_wait.py.md5 @@ -0,0 +1 @@ +a34c7c5d594deb1a871f9b331d920cd3 \ No newline at end of file diff --git a/auto_examples/example_job_submit_wait.rst b/auto_examples/example_job_submit_wait.rst new file mode 100644 index 00000000..d37e7ca2 --- /dev/null +++ b/auto_examples/example_job_submit_wait.rst @@ -0,0 +1,204 @@ + +.. DO NOT EDIT. +.. THIS FILE WAS AUTOMATICALLY GENERATED BY SPHINX-GALLERY. +.. TO MAKE CHANGES, EDIT THE SOURCE PYTHON FILE: +.. "auto_examples/example_job_submit_wait.py" +.. LINE NUMBERS ARE GIVEN BELOW. + +.. only:: html + + .. note:: + :class: sphx-glr-download-link-note + + Click :ref:`here ` + to download the full example code + +.. rst-class:: sphx-glr-example-title + +.. _sphx_glr_auto_examples_example_job_submit_wait.py: + + +Introductory example - Job Submit and Wait +========================================== + +To programatically create and check the status for a group +of jobs using the flux.job.FluxExecutor. + +.. GENERATED FROM PYTHON SOURCE LINES 9-17 + +.. code-block:: default + + + + import os + import concurrent.futures + from flux.job import JobspecV1 + from flux.job import JobspecV1, FluxExecutor + + + + + + + + + +.. GENERATED FROM PYTHON SOURCE LINES 18-23 + +Instead of directly creating a flux handle by importing flux and doing flux.Flux(), +this time we are going to use the flux.job.FluxExecutor. This will allow us to submit +jobs and then asynchronously wait for them to finish. As we did before, let's start +with a jobspec for a sleep job. We are fairly certain this will run with a return +code of 0 to indicate success. + +.. GENERATED FROM PYTHON SOURCE LINES 23-28 + +.. code-block:: default + + + jobspec = JobspecV1.from_command( + command=["sleep", "1"], num_tasks=2, num_nodes=1, cores_per_task=1 + ) + + + + + + + + +.. GENERATED FROM PYTHON SOURCE LINES 29-30 + +Let's again set the working directory and current environment. + +.. GENERATED FROM PYTHON SOURCE LINES 30-33 + +.. code-block:: default + + jobspec.cwd = os.getcwd() + jobspec.environment = dict(os.environ) + + + + + + + + +.. GENERATED FROM PYTHON SOURCE LINES 34-36 + +To mix things up a bit, let's run a command that we know will fail. The false +command always returns a value of 1. This is a "bad" jobspec that will fail! + +.. GENERATED FROM PYTHON SOURCE LINES 36-38 + +.. code-block:: default + + bad_jobspec = JobspecV1.from_command(["/bin/false"]) + + + + + + + + +.. GENERATED FROM PYTHON SOURCE LINES 39-42 + +Now we will demonstrate using the FluxExecutor (via a context) to submit both good +and bad jobs, and wait for them to finish. We call a job that is marked as completed +a "future" and can inspect error code and exceptions to see details about the results! + +.. GENERATED FROM PYTHON SOURCE LINES 42-72 + +.. code-block:: default + + + # create an executor to submit jobs + with FluxExecutor() as executor: + + # we will capture and keep each job future + futures = [] + + # submit half successful jobs + for _ in range(5): + futures.append(executor.submit(jobspec)) + print(f"submit: {id(futures[-1])} (good) jobspec") + + # and half failure jobs! + for _ in range(5): + futures.append(executor.submit(bad_jobspec)) + print(f"submit: {id(futures[-1])} (bad) jobspec") + + # We can now check on our job futures + for future in concurrent.futures.as_completed(futures): + + # There was an exception! + if future.exception() is not None: + print(f"⚠️ wait: {id(future)} Error: job raised error {future.exception()}") + + # Successful result, return code is zero + elif future.result() == 0: + print(f"🏆️ wait: {id(future)} Success") + + # Some other result + else: + print(f"❌️ wait: {id(future)} Error: job returned exit code {future.result()}") + + + +.. rst-class:: sphx-glr-script-out + + .. code-block:: none + + submit: 140663441081680 (good) jobspec + submit: 140663441084272 (good) jobspec + submit: 140663441133824 (good) jobspec + submit: 140663441134112 (good) jobspec + submit: 140663441134400 (good) jobspec + submit: 140663441081584 (bad) jobspec + submit: 140663441135072 (bad) jobspec + submit: 140663441135408 (bad) jobspec + submit: 140663441135744 (bad) jobspec + submit: 140663441136080 (bad) jobspec + 🏆️ wait: 140663441081680 Success + 🏆️ wait: 140663441084272 Success + 🏆️ wait: 140663441133824 Success + 🏆️ wait: 140663441134112 Success + ❌️ wait: 140663441081584 Error: job returned exit code 1 + ❌️ wait: 140663441135072 Error: job returned exit code 1 + ❌️ wait: 140663441135408 Error: job returned exit code 1 + ❌️ wait: 140663441135744 Error: job returned exit code 1 + ❌️ wait: 140663441136080 Error: job returned exit code 1 + 🏆️ wait: 140663441134400 Success + + + + + +.. rst-class:: sphx-glr-timing + + **Total running time of the script:** ( 0 minutes 3.621 seconds) + + +.. _sphx_glr_download_auto_examples_example_job_submit_wait.py: + +.. only:: html + + .. container:: sphx-glr-footer sphx-glr-footer-example + + + .. container:: sphx-glr-download sphx-glr-download-python + + :download:`Download Python source code: example_job_submit_wait.py ` + + .. container:: sphx-glr-download sphx-glr-download-jupyter + + :download:`Download Jupyter notebook: example_job_submit_wait.ipynb ` + + +.. only:: html + + .. rst-class:: sphx-glr-signature + + `Gallery generated by Sphinx-Gallery `_ diff --git a/auto_examples/example_job_submit_wait_codeobj.pickle b/auto_examples/example_job_submit_wait_codeobj.pickle new file mode 100644 index 00000000..666b5ed5 Binary files /dev/null and b/auto_examples/example_job_submit_wait_codeobj.pickle differ diff --git a/auto_examples/images/sphx_glr_example_job_submit_wait_001.png b/auto_examples/images/sphx_glr_example_job_submit_wait_001.png new file mode 100644 index 00000000..7b1fd156 Binary files /dev/null and b/auto_examples/images/sphx_glr_example_job_submit_wait_001.png differ diff --git a/auto_examples/images/thumb/sphx_glr_example_job_submit_wait_thumb.png b/auto_examples/images/thumb/sphx_glr_example_job_submit_wait_thumb.png new file mode 100644 index 00000000..d1854d39 Binary files /dev/null and b/auto_examples/images/thumb/sphx_glr_example_job_submit_wait_thumb.png differ diff --git a/auto_examples/index.rst b/auto_examples/index.rst index 62e560bb..ebdaf746 100644 --- a/auto_examples/index.rst +++ b/auto_examples/index.rst @@ -40,6 +40,23 @@ of using the Flux Python API. +.. raw:: html + +
+ +.. only:: html + + .. image:: /auto_examples/images/thumb/sphx_glr_example_job_submit_wait_thumb.png + :alt: Introductory example - Job Submit and Wait + + :ref:`sphx_glr_auto_examples_example_job_submit_wait.py` + +.. raw:: html + +
Introductory example - Job Submit and Wait
+
+ + .. raw:: html @@ -49,6 +66,7 @@ of using the Flux Python API. :hidden: /auto_examples/example_job_submit_api + /auto_examples/example_job_submit_wait .. only:: html diff --git a/auto_examples/sg_execution_times.rst b/auto_examples/sg_execution_times.rst index 57290d63..53480639 100644 --- a/auto_examples/sg_execution_times.rst +++ b/auto_examples/sg_execution_times.rst @@ -5,8 +5,10 @@ Computation times ================= -**00:00.770** total execution time for **auto_examples** files: +**00:03.621** total execution time for **auto_examples** files: -+-----------------------------------------------------------------------------------------+-----------+--------+ -| :ref:`sphx_glr_auto_examples_example_job_submit_api.py` (``example_job_submit_api.py``) | 00:00.770 | 0.0 MB | -+-----------------------------------------------------------------------------------------+-----------+--------+ ++-------------------------------------------------------------------------------------------+-----------+--------+ +| :ref:`sphx_glr_auto_examples_example_job_submit_wait.py` (``example_job_submit_wait.py``) | 00:03.621 | 0.0 MB | ++-------------------------------------------------------------------------------------------+-----------+--------+ +| :ref:`sphx_glr_auto_examples_example_job_submit_api.py` (``example_job_submit_api.py``) | 00:00.000 | 0.0 MB | ++-------------------------------------------------------------------------------------------+-----------+--------+ diff --git a/examples/example_job_submit_wait.py b/examples/example_job_submit_wait.py new file mode 100644 index 00000000..c5766911 --- /dev/null +++ b/examples/example_job_submit_wait.py @@ -0,0 +1,72 @@ +# -*- coding: utf-8 -*- +""" +Introductory example - Job Submit and Wait +========================================== + +To programatically create and check the status for a group +of jobs using the flux.job.FluxExecutor. +""" + + +import os +import concurrent.futures +from flux.job import JobspecV1 +from flux.job import JobspecV1, FluxExecutor + + +#%% +# Instead of directly creating a flux handle by importing flux and doing flux.Flux(), +# this time we are going to use the flux.job.FluxExecutor. This will allow us to submit +# jobs and then asynchronously wait for them to finish. As we did before, let's start +# with a jobspec for a sleep job. We are fairly certain this will run with a return +# code of 0 to indicate success. + +jobspec = JobspecV1.from_command( + command=["sleep", "1"], num_tasks=2, num_nodes=1, cores_per_task=1 +) + +#%% +# Let's again set the working directory and current environment. +jobspec.cwd = os.getcwd() +jobspec.environment = dict(os.environ) + +#%% +# To mix things up a bit, let's run a command that we know will fail. The false +# command always returns a value of 1. This is a "bad" jobspec that will fail! +bad_jobspec = JobspecV1.from_command(["/bin/false"]) + +#%% +# Now we will demonstrate using the FluxExecutor (via a context) to submit both good +# and bad jobs, and wait for them to finish. We call a job that is marked as completed +# a "future" and can inspect error code and exceptions to see details about the results! + +# create an executor to submit jobs +with FluxExecutor() as executor: + + # we will capture and keep each job future + futures = [] + + # submit half successful jobs + for _ in range(5): + futures.append(executor.submit(jobspec)) + print(f"submit: {id(futures[-1])} (good) jobspec") + + # and half failure jobs! + for _ in range(5): + futures.append(executor.submit(bad_jobspec)) + print(f"submit: {id(futures[-1])} (bad) jobspec") + + # We can now check on our job futures + for future in concurrent.futures.as_completed(futures): + + # There was an exception! + if future.exception() is not None: + print(f"⚠️ wait: {id(future)} Error: job raised error {future.exception()}") + + # Successful result, return code is zero + elif future.result() == 0: + print(f"🏆️ wait: {id(future)} Success") + + # Some other result + else: + print(f"❌️ wait: {id(future)} Error: job returned exit code {future.result()}") \ No newline at end of file