From bcecd183c4eafe675586dc01c2d9e78841f2b99f Mon Sep 17 00:00:00 2001 From: Anton Kargin Date: Wed, 7 Apr 2021 15:23:15 +0300 Subject: [PATCH 1/2] Fix argument of BitsParser.load_qmgr_jobs in process_file method Instead of passing bytes-like object (file_data) bits_parser's (https://github.com/ANSSI-FR/bits_parser) method bits/bits.py->load_file(cls, fp) expects str-like object (simple file path) . It can be seen here: https://github.com/ANSSI-FR/bits_parser/blob/717337cd7a0f97561b77543fb0cebb4e7ce7c0b7/bits/bits.py#L47 Without this change BitsParser fails to parse old format qmgr state files (with .dat extension) with next exception: Processing file \qmgr0.dat Exception occurred processing file \qmgr0.dat: Traceback (most recent call last): File "BitsParser.py", line 366, in process_file jobs = self.load_qmgr_jobs(file_data) File "BitsParser.py", line 101, in load_qmgr_jobs analyzer = bits.Bits.load_file(file_data) File "\bits.py", line 46, in load_file path = Path(fp).resolve() File "\lib\pathlib.py", line 1027, in __new__ self = cls._from_parts(args, init=False) File "\lib\pathlib.py", line 674, in _from_parts drv, root, parts = self._parse_args(args) File "\lib\pathlib.py", line 666, in _parse_args % type(a)) TypeError: argument should be a str object or an os.PathLike object returning str, not --- BitsParser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/BitsParser.py b/BitsParser.py index a2a83db..e566976 100644 --- a/BitsParser.py +++ b/BitsParser.py @@ -363,7 +363,7 @@ def process_file(self, file_path): # Parse as a qmgr database (support old and Win10 formats) jobs = [] if BitsParser.is_qmgr_database(file_data): - jobs = self.load_qmgr_jobs(file_data) + jobs = self.load_qmgr_jobs(file_path) elif BitsParser.is_qmgr10_database(file_data): jobs = self.load_qmgr10_jobs(file_data) From c126ff240ebf9273da993cdc83d292f7555e6394 Mon Sep 17 00:00:00 2001 From: Anton Kargin Date: Thu, 8 Apr 2021 21:45:02 +0300 Subject: [PATCH 2/2] Add csv output support Added csv_writer.py. It's nearly exact copy of file bits/writer.py from bits_parser: https://github.com/ANSSI-FR/bits_parser/blob/master/bits/writer.py I just changed DEFAULT_VALUES dictionary keys for correct mapping with keys from FireEye's BitsParser. Also added some code in BitsParser.py. To output in csv format simply specify ".csv" extension in output filename, when using -o|--output argument. For example: python BitsParser.py -o "csv_output.csv" It will append data from all processed files into specified output file. --- BitsParser.py | 21 +++++++++++++++++++- csv_writer.py | 55 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 1 deletion(-) create mode 100644 csv_writer.py diff --git a/BitsParser.py b/BitsParser.py index e566976..c90547f 100644 --- a/BitsParser.py +++ b/BitsParser.py @@ -328,6 +328,7 @@ def output_jobs(self, file_path, jobs): """Cleans up and outputs the parsed jobs from the qmgr database files""" # If an output file is specified, open it and use it instead of stdout + if self.out_file: orig_stdout = sys.stdout sys.stdout = open(self.out_file, "w") @@ -350,6 +351,20 @@ def output_jobs(self, file_path, jobs): sys.stdout = orig_stdout + def get_unique_records(self, jobs): + records = list() + for job in jobs: + # Skip incomplete carved jobs as they do not contain useful info + if job.is_carved() and not job.is_useful_for_analysis(): + continue + + # Output unique jobs + if job.hash not in self.visited_jobs: + records.append(job.job_dict) + self.visited_jobs.add(job.hash) + return records + + def process_file(self, file_path): """ Processes the given BITS file. Attempts to find/parse jobs. """ @@ -374,7 +389,11 @@ def process_file(self, file_path): else: jobs = self.load_non_qmgr_jobs(file_data) - self.output_jobs(file_path, jobs) + if self.out_file and self.out_file.endswith(".csv"): + import csv_writer + csv_writer.write_csv(self.out_file, self.get_unique_records(jobs)) + else: + self.output_jobs(file_path, jobs) except Exception: print(f'Exception occurred processing file {file_path}: ' + traceback.format_exc(), file=sys.stderr) diff --git a/csv_writer.py b/csv_writer.py new file mode 100644 index 0000000..a566960 --- /dev/null +++ b/csv_writer.py @@ -0,0 +1,55 @@ +import csv +import os + + +DEFAULT_VALUES = ( + ('JobId', None), + ('JobName', None), + ('JobType', None), + ('JobPriority', None), + ('OwnerSID', None), + ('JobState', None), + ('CommandExecuted', None), + ('CommandArguments', None), + ('FileID', 0), + ('DestFile', None), + ('SourceURL', None), + ('TmpFile', None), + ('DownloadByteSize', -1), + ('TransferByteSize', -1), + ('VolumeGUID', None), + ('CreationTime', None), + ('ModifiedTime', None), + ('Carved', False) +) + +def flattener(job): + + def _f(index, file): + rv = {k: file.get(k, job.get(k, v)) for k, v in DEFAULT_VALUES} + rv['FileID'] = index + return rv + + files = job.get('Files', []) + + if files: + return [_f(index, f) for index, f in enumerate(files)] + + return [_f(0, {})] + + +def write_csv(filename, records): + """Write records to a CSV file.""" + if not len(records): + return + if os.path.isfile(filename): + csvfile = open(filename, "a+", newline='', encoding='utf-8') + else: + csvfile = open(filename, "w", newline='', encoding='utf-8') + + writer = csv.DictWriter(csvfile, fieldnames=[k for k, _ in DEFAULT_VALUES]) + writer.writeheader() + for r in records: + for sub_r in flattener(r): + writer.writerow(sub_r) + csvfile.close()