Merge branch 'deepmodeling:master' into for_pbs_torque

Author: saltball

.github/workflows/publish_conda.yml

@@ -0,0 +1,18 @@
+name: publish_conda
+
+on:
+  push:
+    tags:
+      - 'v*'
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: publish-to-conda
+      uses: felix5572/conda-publish-action@v1.9
+      with:
+        subdir: 'conda'
+        anacondatoken: ${{ secrets.ANACONDA_TOKEN }}
+        platforms: 'noarch'

.gitignore

@@ -40,6 +40,7 @@ pydispatcher.egg-info
 .coverage
 dbconfig.json
 .vscode/*
+.idea
 */_version.py
 */_date.py
 *.egg

conda/conda_build_config.yaml

@@ -0,0 +1,5 @@
+channel_sources:
+  - defaults
+  - conda-forge
+channel_targets:
+  - deepmodeling

conda/meta.yaml

@@ -0,0 +1,49 @@
+{% set name = "dpdispatcher" %}
+{% set version = environ.get('GIT_DESCRIBE_TAG').lstrip('v') %}
+
+package:
+  name: {{ name|lower }}
+  version: {{ version }}
+
+source:
+  git_url: https://github.com/deepmodeling/dpdispatcher
+  # git_rev: {{ version }}
+
+build:
+  number: 0
+  noarch: python
+  script: python -m pip install --no-deps --ignore-installed .
+
+requirements:
+  build:
+    - git
+  host:
+    - git
+    - python >=3.6
+    - pip
+    - setuptools_scm
+    - dargs
+    - paramiko
+    - requests
+
+  run:
+    - python >=3.6
+    - dargs
+    - paramiko
+    - requests
+
+test:
+  imports:
+    - dpdispatcher
+
+about:
+  home: https://github.com/deepmodeling/dpdispatcher
+  license: LGPL-3.0
+  license_family: LGPL
+  license_file: LICENSE
+  doc_url: https://github.com/deepmodeling/dpdispatcher
+  dev_url: https://github.com/deepmodeling/dpdispatcher
+
+extra:
+  recipe-maintainers:
+    - felix5572

doc/machine-auto.rst

@@ -57,23 +57,29 @@ machine:
             ssh connection port.
 
         key_filename: 
-            | type: ``NoneType`` | ``str``, optional, default: ``None``
+            | type: ``str`` | ``NoneType``, optional, default: ``None``
             | argument path: ``machine/remote_profile/key_filename``
 
-            key_filename used by ssh connection
+            key filename used by ssh connection. If left None, find key in ~/.ssh or use password for login
 
         passphrase: 
-            | type: ``NoneType`` | ``str``, optional, default: ``None``
+            | type: ``str`` | ``NoneType``, optional, default: ``None``
             | argument path: ``machine/remote_profile/passphrase``
 
-            passphrase used by ssh connection
+            passphrase of key used by ssh connection
 
         timeout: 
             | type: ``int``, optional, default: ``10``
             | argument path: ``machine/remote_profile/timeout``
 
             timeout of ssh connection
 
+        totp_secret: 
+            | type: ``str`` | ``NoneType``, optional, default: ``None``
+            | argument path: ``machine/remote_profile/totp_secret``
+
+            Time-based one time password secret. It should be a base32-encoded string extracted from the 2D code.
+
     clean_asynchronously: 
         | type: ``bool``, optional, default: ``False``
         | argument path: ``machine/clean_asynchronously``

doc/task-auto.rst

@@ -27,13 +27,13 @@ task:
         The files to be download to local_root in task_work_path after the task finished
 
     outlog: 
-        | type: ``NoneType`` | ``str``
+        | type: ``str`` | ``NoneType``
         | argument path: ``task/outlog``
 
         The out log file name. redirect from stdout
 
     errlog: 
-        | type: ``NoneType`` | ``str``
+        | type: ``str`` | ``NoneType``
         | argument path: ``task/errlog``
 
         The err log file name. redirect from stderr

dpdispatcher/__init__.py

@@ -1,11 +1,17 @@
 import logging
 import os, sys
-
+import warnings
 
 ROOT_PATH=__path__[0]
 dlog = logging.getLogger(__name__)
+dlog.propagate = False
 dlog.setLevel(logging.INFO)
-dlogf = logging.FileHandler(os.getcwd()+os.sep+'dpdispatcher'+'.log')
+try:
+    dlogf = logging.FileHandler(os.getcwd()+os.sep+'dpdispatcher'+'.log')
+except PermissionError:
+    warnings.warn(f"dpdispatcher.log meet permission error. redirect the log to ~/dpdispatcher.log")
+    dlogf = logging.FileHandler(os.path.join(os.path.expanduser('~'),'dpdispatcher.log'))
+
 # dlogf = logging.FileHandler('./'+os.sep+SHORT_CMD+'.log')
 # dlogf = logging.FileHandler(os.path.join(os.environ['HOME'], SHORT_CMD+'.log'))
 # dlogf = logging.FileHandler(os.path.join(os.path.expanduser('~'), SHORT_CMD+'.log'))

dpdispatcher/dp_cloud_server.py

@@ -3,6 +3,7 @@
 from dpdispatcher.machine import Machine
 from dpdispatcher.dpcloudserver import api
 from dpdispatcher.dpcloudserver.config import ALI_OSS_BUCKET_URL
+import time
 
 shell_script_header_template="""
 #!/bin/bash -l
@@ -45,7 +46,8 @@ def do_submit(self, job):
         job_id = api.job_create(
             job_type=input_data['job_type'],
             oss_path=input_data['job_resources'],
-            input_data=input_data
+            input_data=input_data,
+            program_id=self.context.remote_profile.get('program_id', None)
         )
 
         job.job_id = job_id
@@ -56,10 +58,19 @@ def check_status(self, job):
         if job.job_id == '':
             return JobStatus.unsubmitted
         dlog.debug(f"debug: check_status; job.job_id:{job.job_id}; job.job_hash:{job.job_hash}")
+
+        check_return = api.get_tasks(job.job_id)
         try:
-            dp_job_status = api.get_tasks(job.job_id)[0]["status"]
+            dp_job_status = check_return[0]["status"]
         except IndexError as e:
-            raise RuntimeError(f"cannot find job information in dpcloudserver's database for job {job.job_id}")
+            dlog.error(f"cannot find job information in check_return. job {job.job_id}. check_return:{check_return}; retry one more time after 60 seconds")
+            time.sleep(60)
+            retry_return = api.get_tasks(job.job_id)
+            try:
+                dp_job_status = retry_return[0]["status"]
+            except IndexError as e:
+                raise RuntimeError(f"cannot find job information in dpcloudserver's database for job {job.job_id} {check_return} {retry_return}")
+
         job_state = self.map_dp_job_state(dp_job_status)
         return job_state
 

dpdispatcher/dp_cloud_server_context.py

@@ -26,10 +26,11 @@ def __init__ (self,
     ):
         self.temp_local_root = os.path.abspath(local_root)
         self.remote_profile = remote_profile
-        username = remote_profile['username']
+        email = remote_profile.get("email", None)
+        username = remote_profile.get('username', None)
         password = remote_profile['password']
 
-        api.login(username=username, password=password)
+        api.login(username=username, email=email, password=password)
 
         os.makedirs(DP_CLOUD_SERVER_HOME_DIR, exist_ok=True)
 

dpdispatcher/dpcloudserver/api.py

@@ -47,11 +47,18 @@ def post(url, params):
     return ret['data']
 
 
-def login(username, password):
+def login(password, email=None, username=None):
     global token
+    post_data = {"password": password}
+    if email is None and username is None:
+        raise ValueError(f"Error: can not find username or email from remote_profile")
+    if email is not None:
+        post_data['email'] = email
+    if username is not None:
+        post_data['username'] = username
     ret = post(
             '/account/login',
-            {"username": username, "password": password}
+            post_data
             )
     dlog.debug(f"debug: login ret:{ret}")
     token = ret['token']
@@ -99,14 +106,15 @@ def upload(oss_task_zip, zip_task_file, endpoint, bucket_name):
     return result
 
 
-def job_create(job_type, oss_path, input_data):
-    ret = post(
-            '/data/insert_job',
-            {
+def job_create(job_type, oss_path, input_data, program_id=None):
+    post_data = {
                 'job_type': job_type,
                 'oss_path': oss_path,
                 'input_data': input_data,
-            })
+            }
+    if program_id is not None:
+        post_data["program_id"] = program_id
+    ret = post('/data/insert_job', post_data)
     return ret['job_id']