Merge remote-tracking branch 'origin/master'

Author: felix5572

.github/workflows/mirror_gitee.yml

@@ -0,0 +1,19 @@
+name: Mirror to Gitee Repo
+
+on: [ push, delete, create ]
+
+# Ensures that only one mirror task will run at a time.
+concurrency:
+  group: git-mirror
+
+jobs:
+  git-mirror:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: wearerequired/git-mirror-action@v1
+        env:
+          ORGANIZATION: deepmodeling
+          SSH_PRIVATE_KEY: ${{ secrets.SYNC_GITEE_PRIVATE_KEY }}
+        with:
+          source-repo: "https://github.com/deepmodeling/dpdispatcher.git"
+          destination-repo: "git@gitee.com:deepmodeling/dpdispatcher.git"

doc/api.rst

@@ -10,15 +10,16 @@
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 #
-# import os
-# import sys
+import os
+import sys
+from datetime import date
 # sys.path.insert(0, os.path.abspath('.'))
 
 
 # -- Project information -----------------------------------------------------
 
 project = 'DPDispatcher'
-copyright = '2020, Deep Modeling'
+copyright = '2020-%d, Deep Modeling' % date.today().year
 author = 'Deep Modeling'
 
 
@@ -30,6 +31,9 @@
 extensions = [
     'recommonmark',
     "sphinx_rtd_theme",
+    'sphinx.ext.viewcode',
+    'sphinx.ext.intersphinx',
+    'numpydoc',
     'sphinx.ext.autosummary'
 ]
 
@@ -58,3 +62,18 @@
 autodoc_default_flags = ['members']
 autosummary_generate = True
 master_doc = 'index'
+
+def run_apidoc(_):
+    from sphinx.ext.apidoc import main
+    sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+    cur_dir = os.path.abspath(os.path.dirname(__file__))
+    module = os.path.join(cur_dir, "..", "dpdispatcher")
+    main(['-M', '--tocfile', 'api', '-H', 'DPDispatcher API', '-o', os.path.join(cur_dir, "api"), module, '--force'])
+
+def setup(app):
+    app.connect('builder-inited', run_apidoc)
+
+
+intersphinx_mapping = {
+    "python": ("https://docs.python.org/", None),
+}

doc/conf.py

@@ -20,7 +20,7 @@ DPDispatcher will monitor (poke) until these jobs finish and download the result
    machine
    resources
    task
-   api
+   api/api
 
 
 Indices and tables

doc/index.rst

@@ -42,7 +42,7 @@ def do_submit(self, job, retry=0, max_retry=3):
         ret, stdin, stdout, stderr = self.context.block_call('cd %s && %s %s' % (self.context.remote_root, 'sbatch', script_file_name))
         if ret != 0:
             err_str = stderr.read().decode('utf-8')
-            if "Socket timed out on send/recv operation" in err_str:
+            if "Socket timed out on send/recv operation" in err_str or "Unable to contact slurm controller" in err_str:
                 # server network error, retry 3 times
                 if retry < max_retry:
                     dlog.warning("Get error code %d in submitting through ssh with job: %s . message: %s" %
@@ -81,7 +81,7 @@ def check_status(self, job, retry=0, max_retry=3):
                     return JobStatus.finished
                 else :
                     return JobStatus.terminated
-            elif "Socket timed out on send/recv operation" in err_str:
+            elif "Socket timed out on send/recv operation" in err_str or "Unable to contact slurm controller" in err_str:
                 # retry 3 times
                 if retry < max_retry:
                     dlog.warning("Get error code %d in checking status through ssh with job: %s . message: %s" %

dpdispatcher/slurm.py

@@ -526,18 +526,18 @@ def _get_files(self,
         per_nfile = 100
         ntar = len(files) // per_nfile + 1
         if ntar <= 1:
-            self.block_checkcall('tar czf %s %s' % (of, " ".join(files)))
+            self.block_checkcall('tar czfh %s %s' % (of, " ".join(files)))
         else:
             of_tar = self.submission.submission_hash + '.tar'
             for ii in range(ntar):
                 ff = files[per_nfile * ii : per_nfile * (ii+1)]
                 if ii == 0:
                     # tar cf for the first time
-                    self.block_checkcall('tar cf %s %s' % (of_tar, " ".join(ff)))
+                    self.block_checkcall('tar cfh %s %s' % (of_tar, " ".join(ff)))
                 else:
                     # append using tar rf
                     # -r, --append append files to the end of an archive
-                    self.block_checkcall('tar rf %s %s' % (of_tar, " ".join(ff)))
+                    self.block_checkcall('tar rfh %s %s' % (of_tar, " ".join(ff)))
             # compress the tar file using gzip, and will get a tar.gz file
             # overwrite considering dpgen may stop and restart
             # -f, --force force overwrite of output file and compress links

dpdispatcher/ssh_context.py

@@ -554,11 +554,12 @@ def handle_unexpected_job_state(self):
             self.handle_unexpected_job_state()
 
         if job_state == JobStatus.unsubmitted:
-            dlog.info(f"job: {self.job_hash} unsubmitted; submit it")
+            dlog.debug(f"job: {self.job_hash} unsubmitted; submit it")
             # if self.fail_count > 3:
             #     raise RuntimeError("job:job {job} failed 3 times".format(job=self))
             self.submit_job()
-            dlog.info("job: {job_hash} submit; job_id is {job_id}".format(job_hash=self.job_hash, job_id=self.job_id))
+            if self.job_state != JobStatus.unsubmitted:
+                dlog.info("job: {job_hash} submit; job_id is {job_id}".format(job_hash=self.job_hash, job_id=self.job_id))
             # self.get_job_state()
 
     def get_hash(self):

dpdispatcher/submission.py

@@ -40,7 +40,7 @@
     keywords='deep potential generator active learning deepmd-kit',
     install_requires=install_requires,    
     extras_require={
-        'docs': ['sphinx', 'recommonmark', 'sphinx_rtd_theme'],
+        'docs': ['sphinx', 'recommonmark', 'sphinx_rtd_theme>=1.0.0rc1', 'numpydoc'],
         "cloudserver": ["oss2"],
         ":python_version<'3.7'": ["typing_extensions"],
     },