diff --git a/tools/test_verify_output.py b/tools/test_verify_output.py new file mode 100644 index 0000000..454cae2 --- /dev/null +++ b/tools/test_verify_output.py @@ -0,0 +1,24 @@ +import tempfile +import unittest +from pathlib import Path + +from verify_lib import find_last_output + + +class FindLastOutputTest(unittest.TestCase): + def test_combines_csv_files_without_cat(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + for rank in (0, 1): + for step in (1, 2): + path = root / f"output-2D-{rank}-0000{step}.csv" + path.write_text(f"rank{rank}\n") + + output, combined = find_last_output(dir=directory) + + self.assertTrue(combined) + self.assertEqual(Path(output).read_text(), "rank0\nrank1\n") + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/verify_lib.py b/tools/verify_lib.py index f54801f..d284f14 100644 --- a/tools/verify_lib.py +++ b/tools/verify_lib.py @@ -1,7 +1,7 @@ import argparse import glob import os -import subprocess +import shutil from collections.abc import Callable from importlib import util as imputil from typing import Any @@ -216,8 +216,10 @@ def find_last_output( # Combine the files new_file = combine[-1].replace(f"D-{mm}-", f"D-{mm + 1}-") - with open(new_file, "w") as fwrite: - subprocess.run(["cat"] + combine, stdout=fwrite) + with open(new_file, "wb") as fwrite: + for file in combine: + with open(file, "rb") as fread: + shutil.copyfileobj(fread, fwrite, length=1024 * 1024) return new_file, True