Merge pull request #131 from njzjz/slurm

add a sitatuation to retry slurm cmd

Author: njzjz

dpdispatcher/slurm.py

@@ -42,7 +42,7 @@ def do_submit(self, job, retry=0, max_retry=3):
         ret, stdin, stdout, stderr = self.context.block_call('cd %s && %s %s' % (self.context.remote_root, 'sbatch', script_file_name))
         if ret != 0:
             err_str = stderr.read().decode('utf-8')
-            if "Socket timed out on send/recv operation" in err_str:
+            if "Socket timed out on send/recv operation" in err_str or "Unable to contact slurm controller" in err_str:
                 # server network error, retry 3 times
                 if retry < max_retry:
                     dlog.warning("Get error code %d in submitting through ssh with job: %s . message: %s" %
@@ -81,7 +81,7 @@ def check_status(self, job, retry=0, max_retry=3):
                     return JobStatus.finished
                 else :
                     return JobStatus.terminated
-            elif "Socket timed out on send/recv operation" in err_str:
+            elif "Socket timed out on send/recv operation" in err_str or "Unable to contact slurm controller" in err_str:
                 # retry 3 times
                 if retry < max_retry:
                     dlog.warning("Get error code %d in checking status through ssh with job: %s . message: %s" %