ensure atomic writes when saving a file

Right now, if a program crashes in the middle of saving, you lose
all your data.

Right now, if a program crashes in the middle of saving, you lose
all your data. This ensures that the old file is first moved, then
the new file is saved, and only then the old file is removed.

Author: basnijholt

adaptive/utils.py

@@ -7,6 +7,8 @@
 from contextlib import contextmanager
 from itertools import product
 
+from atomicwrites import AtomicWriter
+
 
 def named_product(**items):
     names = items.keys()
@@ -44,9 +46,13 @@ def save(fname, data, compress=True):
     dirname = os.path.dirname(fname)
     if dirname:
         os.makedirs(dirname, exist_ok=True)
-    _open = gzip.open if compress else open
-    with _open(fname, "wb") as f:
-        pickle.dump(data, f, protocol=pickle.HIGHEST_PROTOCOL)
+
+    blob = pickle.dumps(data, protocol=pickle.HIGHEST_PROTOCOL)
+    if compress:
+        blob = gzip.compress(blob)
+
+    with AtomicWriter(fname, "wb", overwrite=True).open() as f:
+        f.write(blob)
 
 
 def load(fname, compress=True):

docs/environment.yml

@@ -12,6 +12,7 @@ dependencies:
   - bokeh=1.0.4
   - plotly=3.9.0
   - ipywidgets=7.4.2
+  - atomicwrites=1.3.0
   - pip:
     - git+https://github.com/basnijholt/jupyter-sphinx.git@widgets_execute
     - sphinx_fontawesome==0.0.6

environment.yml

@@ -16,3 +16,4 @@ dependencies:
   - ipywidgets
   - scikit-optimize
   - plotly
+  - atomicwrites

setup.py

@@ -24,7 +24,12 @@ def get_version_and_cmdclass(package_name):
 version, cmdclass = get_version_and_cmdclass("adaptive")
 
 
-install_requires = ["scipy", "sortedcollections >= 1.1", "sortedcontainers >= 2.0"]
+install_requires = [
+    "scipy",
+    "sortedcollections >= 1.1",
+    "sortedcontainers >= 2.0",
+    "atomicwrites",
+]
 
 extras_require = {
     "notebook": [