fix copying directory (#429)

Fix #428.

---------

Signed-off-by: Jinzhe Zeng <jinzhe.zeng@rutgers.edu>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: njzjz

dpdispatcher/contexts/local_context.py

@@ -208,7 +208,10 @@ def download(
                             shutil.rmtree(lfile, ignore_errors=True)
                         elif os.path.isfile(lfile) or os.path.islink(lfile):
                             os.remove(lfile)
-                        shutil.copyfile(rfile, lfile)
+                        if not os.path.islink(rfile):
+                            shutil.move(rfile, lfile)
+                        else:
+                            shutil.copyfile(rfile, lfile)
                     else:
                         raise RuntimeError("should not reach here!")
                 else:
@@ -275,7 +278,10 @@ def download(
                         shutil.rmtree(lfile, ignore_errors=True)
                     elif os.path.isfile(lfile) or os.path.islink(lfile):
                         os.remove(lfile)
-                    shutil.copyfile(rfile, lfile)
+                    if not os.path.islink(rfile):
+                        shutil.move(rfile, lfile)
+                    else:
+                        shutil.copyfile(rfile, lfile)
                 else:
                     raise RuntimeError("should not reach here!")
             else:

tests/test_run_submission.py

@@ -3,6 +3,7 @@
 import random
 import shutil
 import sys
+import tempfile
 import traceback
 
 sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
@@ -72,6 +73,16 @@ def test_run_submission(self):
             )
             task_list.append(task)
 
+        for ii in range(2):
+            task = Task(
+                command=f"mkdir -p out_dir{ii} && touch out_dir{ii}/out{ii}",
+                task_work_path="./",
+                forward_files=[],
+                backward_files=[f"out_dir{ii}"],
+                outlog=f"out_dir{ii}.txt",
+            )
+            task_list.append(task)
+
         # test space in file name
         task_list.append(
             Task(
@@ -90,6 +101,11 @@ def test_run_submission(self):
             backward_common_files=[],
             task_list=task_list,
         )
+        # test override directory
+        os.makedirs(
+            os.path.join(self.machine_dict["local_root"], "test_dir", "out_dir1"),
+            exist_ok=True,
+        )
         submission.run_submission(check_interval=2)
 
         for ii in range(4):
@@ -251,6 +267,23 @@ def test_async_run_submission(self):
         return super().test_async_run_submission()
 
 
+@unittest.skipIf(sys.platform == "win32", "Shell is not supported on Windows")
+class TestLocalContext(RunSubmission, unittest.TestCase):
+    def setUp(self):
+        super().setUp()
+        self.temp_dir = tempfile.TemporaryDirectory()
+        self.machine_dict["context_type"] = "LocalContext"
+        self.machine_dict["remote_root"] = self.temp_dir.name
+
+    def tearDown(self):
+        super().tearDown()
+        self.temp_dir.cleanup()
+
+    @unittest.skipIf(sys.platform == "darwin", "TODO: Error on macos")
+    def test_async_run_submission(self):
+        return super().test_async_run_submission()
+
+
 @unittest.skipIf(sys.platform == "win32", "Shell is not supported on Windows")
 class TestLazyLocalContext(RunSubmission, unittest.TestCase):
     def setUp(self):