Merge pull request #200 from KernelTuner/fix-json-errors

Fix json errors

Author: benvanwerkhoven

kernel_tuner/file_utils.py

@@ -131,7 +131,7 @@ def store_output_file(output_filename, results, tune_params, objective="time"):
     version, _ = output_file_schema("results")
     output_json = dict(results=output_data, schema_version=version)
     with open(output_filename, 'w+') as fh:
-        json.dump(output_json, fh)
+        json.dump(output_json, fh, cls=util.NpEncoder)
 
 
 def get_dependencies(package='kernel_tuner'):
@@ -184,15 +184,19 @@ def store_metadata_file(metadata_filename):
     metadata_filename = filename_ensure_json_extension(metadata_filename)
     metadata = {}
 
-    # lshw only works on Linux, this intentionally raises a FileNotFoundError when ran on systems that do not have it
-    lshw_out = subprocess.run(["lshw", "-json"], capture_output=True)
+    # lshw only works on Linux
+    try:
+        lshw_out = subprocess.run(["lshw", "-json"], capture_output=True)
 
-    # sometimes lshw outputs a list of length 1, sometimes just as a dict, schema wants a list
-    lshw_string = lshw_out.stdout.decode('utf-8').strip()
-    if lshw_string[0] == '{' and lshw_string[-1] == '}':
-        lshw_string = '[' + lshw_string + ']'
+        # sometimes lshw outputs a list of length 1, sometimes just as a dict, schema wants a list
+        lshw_string = lshw_out.stdout.decode('utf-8').strip()
+        if lshw_string[0] == '{' and lshw_string[-1] == '}':
+            lshw_string = '[' + lshw_string + ']'
+        hardware_desc = dict(lshw=json.loads(lshw_string))
+    except:
+        hardware_desc = dict(lshw=["lshw error"])
 
-    metadata["hardware"] = dict(lshw=json.loads(lshw_string))
+    metadata["hardware"] = hardware_desc
 
     # attempts to use nvidia-smi or rocm-smi if present
     device_query = {}

kernel_tuner/util.py

@@ -49,6 +49,19 @@ class RuntimeFailedConfig(ErrorConfig):
     pass
 
 
+class NpEncoder(json.JSONEncoder):
+    """ Class we use for dumping Numpy objects to JSON """
+
+    def default(self, obj):
+        if isinstance(obj, np.integer):
+            return int(obj)
+        if isinstance(obj, np.floating):
+            return float(obj)
+        if isinstance(obj, np.ndarray):
+            return obj.tolist()
+        return super(NpEncoder, self).default(obj)
+
+
 class TorchPlaceHolder():
 
     def __init__(self):
@@ -725,18 +738,6 @@ def compile_restrictions(restrictions: list, tune_params: dict):
     return func
 
 
-class NpEncoder(json.JSONEncoder):
-
-    def default(self, obj):
-        if isinstance(obj, np.integer):
-            return int(obj)
-        if isinstance(obj, np.floating):
-            return float(obj)
-        if isinstance(obj, np.ndarray):
-            return obj.tolist()
-        return super(NpEncoder, self).default(obj)
-
-
 def process_cache(cache, kernel_options, tuning_options, runner):
     """cache file for storing tuned configurations
 
@@ -871,16 +872,6 @@ def close_cache(cache):
 def store_cache(key, params, tuning_options):
     """ stores a new entry (key, params) to the cachefile """
 
-    # create converter for dumping numpy objects to JSON
-    def JSONconverter(obj):
-        if isinstance(obj, np.integer):
-            return int(obj)
-        if isinstance(obj, np.floating):
-            return float(obj)
-        if isinstance(obj, np.ndarray):
-            return obj.tolist()
-        return obj.__str__()
-
     #logging.debug('store_cache called, cache=%s, cachefile=%s' % (tuning_options.cache, tuning_options.cachefile))
     if isinstance(tuning_options.cache, dict):
         if not key in tuning_options.cache:
@@ -894,7 +885,7 @@ def JSONconverter(obj):
 
             if tuning_options.cachefile:
                 with open(tuning_options.cachefile, "a") as cachefile:
-                    cachefile.write("\n" + json.dumps({ key: output_params }, default=JSONconverter)[1:-1] + ",")
+                    cachefile.write("\n" + json.dumps({ key: output_params }, cls=NpEncoder)[1:-1] + ",")
 
 
 def dump_cache(obj: str, tuning_options):