Added .csv output option to data_parser.py

TARS/.gitignore

@@ -3,3 +3,4 @@
 .vscode/c_cpp_properties.json
 .vscode/launch.json
 .vscode/ipch
+sdlog.dat

TARS/data_parser.py

@@ -9,13 +9,18 @@
     - There are numerous reasons this might not work out of the box. Some reasons I can think of right now include
       not having python on your path, having the wrong python on your path, not being in the right working directory,
       or not having installed the right lark.
+    - Outputting to a `.csv` file will automatically trigger a conversion from JSON to CSV at the end of the process.
+
+Authors:
+    Magilan Sendhil
+    Michael Karpov
 """
 
 from __future__ import annotations
 
 import json
 from abc import ABC, abstractmethod
-from typing import Callable
+from typing import Callable, Any, NamedTuple, cast
 
 import lark
 from lark import Lark
@@ -599,7 +604,13 @@ def render_bytes(n: int) -> str:
     return f"{n:#.4g}{endings[end_idx]}"
 
 
-def get_arguments():
+class Arguments(NamedTuple):
+    header: pathlib.Path
+    raw: pathlib.Path
+    out: pathlib.Path
+
+
+def get_arguments() -> Arguments:
     """
     A helper function which parses the arguments passed to the file.
 
@@ -617,17 +628,60 @@ def get_arguments():
     if args.out is None:
         raw_path: pathlib.Path = args.raw
         args.out = raw_path.with_suffix(".json")
-    return args
+    return cast(Arguments, args)
 
 
 def preprocess(t: str) -> str:
+    """
+    A very basic and very wrong C++ preprocessor, it just deletes any line which has a preprocessor directive
+    """
     l = []
     for line in t.splitlines():
         if not line.startswith("#"):
             l.append(line)
     return "\n".join(l)
 
 
+def construct_header_string(jsonobject: dict[str, Any], path: str = "") -> str:
+    """
+    Given a json object that is representative of the whole file, recursively create a header string containing every
+    key in the json as a period-separated path.
+
+    :param jsonobject: A JSON object to construct a header string for
+    :param path: The path this JSON object is at
+    :return: A string for the keys in this json object
+    """
+    items: list[str] = []
+    for key, value in jsonobject.items():
+        if isinstance(value, dict):
+            # If it's a dictionary, continue recursively but including the trace to the datatype
+            items.append(construct_header_string(value, f"{path}.{key}"))
+        else:
+            items.append(f"{path}.{key}")
+    return ",".join(items)
+
+
+def construct_data_string(jsonobject: dict[str, Any]):
+    """
+    Construct a string turning a single JSON object into csv columns
+
+    :param jsonobject: A JSON object to convert
+    :return: a string for the values in this json object as csv rows
+    """
+    items: list[str] = []
+    for key, value in jsonobject.items():
+        if isinstance(value, dict):
+            items.append(construct_data_string(value))
+        elif isinstance(value, list):
+            # If it's a list, take only the first value
+            # (Lists are only used for FSM, anyway, someone could alter this behavior though if they wish)
+            # todo make this less fragile, but it should be good for now
+            items.append(str(value[0]))
+        else:
+            items.append(value)
+    return ",".join(items)
+
+
 def main():
     args = get_arguments()
 
@@ -651,6 +705,7 @@ def main():
         all_data: bytes = f_da.read()
 
     idx = 0  # a counter which keeps track of how far into the raw data we are
+
     # open up the output file, which we write to incrementally
     with args.out.open("w") as f_out:
         f_out.write("[")
@@ -676,6 +731,19 @@ def main():
             print(f"\rParsed {render_bytes(idx)}/{render_bytes(len(all_data))} bytes", end="", flush=True)
         f_out.write("\n]")
 
+    # If file_type is .csv, we will convert it from JSON (It is easier to convert from JSON than from a binary format.)
+    if args.out.suffix == ".csv":
+        print("\nConverting .json output to .csv")
+        with args.out.open("r") as f_in:
+            data_json = json.load(f_in)
+
+        header_string = construct_header_string(data_json[0])
+        with args.out.open("w") as f_out:
+            # Now we can overwrite the data, since we have the json loaded.
+            f_out.write(header_string + "\n")
+            for data_object in data_json:
+                f_out.write(construct_data_string(data_object) + "\n")
+
 
 if __name__ == '__main__':
-    main()
+    main()