From 9354e812fe79da4904ffca609f48def9ef9da88e Mon Sep 17 00:00:00 2001 From: Moritz Marquardt Date: Fri, 18 Dec 2020 18:36:33 +0100 Subject: [PATCH 1/6] Add catz command to make it easier to work with Bro/Zeek logfiles --- catz | 2 ++ catz.py | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100755 catz create mode 100755 catz.py diff --git a/catz b/catz new file mode 100755 index 0000000..65547aa --- /dev/null +++ b/catz @@ -0,0 +1,2 @@ +#!/bin/sh +exec python3 "$(dirname "$(readlink -f "$0")")"/catz.py "$@" 2>&1 | less -Snc diff --git a/catz.py b/catz.py new file mode 100755 index 0000000..e8ae0e1 --- /dev/null +++ b/catz.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +import sys +from brologparse import parse_log_file + +if len(sys.argv) < 2: + print("Log parser for Bro/Zeek logfiles") + print("Usage: catz ") + sys.exit(1) + +entries = [] +for entry in parse_log_file(sys.argv[1]): + entries.append(entry) + +fields = [] +with open(sys.argv[1], "r") as f: + separator = f.readline().rstrip("\n").split(" ")[1].encode('raw_unicode_escape').decode('unicode_escape') + for i in range(5): + f.readline() + fields = [field.replace(".", "_") for field in f.readline().rstrip("\n").lstrip("#").split(separator)[1:]] + +formatted_entries = [] +field_lengths = {} +for field in fields: + field_lengths[field] = len(field) +for entry in entries: + formatted_entry = [] + for field in fields: + formatted_field = "{}".format(getattr(entry, field)) + formatted_entry.append(formatted_field) + if len(formatted_field) > field_lengths[field]: + field_lengths[field] = len(formatted_field) + formatted_entries.append(formatted_entry) + +row_format = "{:>" + "{}".format(len("{}".format(len(entries)))) + "} | " +separator_line = [] +for field in fields: + if field_lengths[field] > 50: + field_lengths[field] = 50 + row_format += "{:<" + "{}".format(field_lengths[field]) + "} " + separator_line.append("-" * field_lengths[field]) +print(row_format.format("", *fields)) +print(row_format.format("", *separator_line)) +i = 1 +for entry in formatted_entries: + print(row_format.format(i, *entry)) + i += 1 From 9e839de81af0cc85d615ab876f78edf2dfd952de Mon Sep 17 00:00:00 2001 From: Moritz Marquardt Date: Fri, 18 Dec 2020 19:03:58 +0100 Subject: [PATCH 2/6] Display empty string instead of "None" in catz.py --- catz.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/catz.py b/catz.py index e8ae0e1..dcf3c91 100755 --- a/catz.py +++ b/catz.py @@ -25,7 +25,10 @@ for entry in entries: formatted_entry = [] for field in fields: - formatted_field = "{}".format(getattr(entry, field)) + formatted_field = "" + unformatted_field = getattr(entry, field) + if unformatted_field is not None: + formatted_field = "{}".format(unformatted_field) formatted_entry.append(formatted_field) if len(formatted_field) > field_lengths[field]: field_lengths[field] = len(formatted_field) From 2eb7c0f6ece32ceb985e7b335d8832c95d9ff53b Mon Sep 17 00:00:00 2001 From: Moritz Marquardt Date: Fri, 18 Dec 2020 19:04:46 +0100 Subject: [PATCH 3/6] Add information about catz to README.md --- README.md | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 7e50bd2..190486c 100644 --- a/README.md +++ b/README.md @@ -3,16 +3,23 @@ ![GitHub Action Status](https://github.com/elnappo/bro-log-parser/workflows/Python%20package/badge.svg) [![Maintainability](https://api.codeclimate.com/v1/badges/680163011be7d7903c0f/maintainability)](https://codeclimate.com/github/elnappo/bro-log-parser/maintainability) -Simple logfile parser for [Bro IDS](https://www.bro.org/). This library parses and transforms entries +Simple logfile parser for [Bro IDS](https://www.bro.org/). This library parses and transforms entries in a logfile created by the [ASCII Writer](https://www.bro.org/sphinx/frameworks/logging.html#ascii-writer) into a dynamically generated namedtuple. Fields are converted into native Python data types. +Additionally, it provides a command-line tool named `catz` that makes it easier to work with Zeek log files manually. + ## Requirements * python3 - + ## Install python3 setup.py install +## Usage of the CLI + ./catz conn.log + +![](https://user-images.githubusercontent.com/5559994/102645090-35ca8f80-4162-11eb-9be6-ec2a2eec06c1.png) + ## Tests pytest # OR @@ -28,7 +35,7 @@ into a dynamically generated namedtuple. Fields are converted into native Python ... ConnEntry( ts=datetime.datetime(2015, 1, 23, 0, 49, 13, 396481), - uid='CjPbcf1SkE86OWWTra', + uid='CjPbcf1SkE86OWWTra', id_orig_h=IPv4Address('192.168.1.100'), id_orig_p=137, id_resp_h=IPv4Address('192.168.1.255'), From f033f992ecc2651c7fff7ce6518db8908059fed4 Mon Sep 17 00:00:00 2001 From: Moritz Marquardt Date: Fri, 18 Dec 2020 19:19:53 +0100 Subject: [PATCH 4/6] Fix width of entry number field --- catz.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catz.py b/catz.py index dcf3c91..ecdaca1 100755 --- a/catz.py +++ b/catz.py @@ -34,7 +34,7 @@ field_lengths[field] = len(formatted_field) formatted_entries.append(formatted_entry) -row_format = "{:>" + "{}".format(len("{}".format(len(entries)))) + "} | " +row_format = "{:>" + "{}".format(len("{}".format(len(formatted_entries)))) + "} | " separator_line = [] for field in fields: if field_lengths[field] > 50: From 7a3e6a13e3dbc0c2b33dc5cc3da8d18b8c65f026 Mon Sep 17 00:00:00 2001 From: Moritz Marquardt Date: Fri, 18 Dec 2020 19:20:09 +0100 Subject: [PATCH 5/6] Truncate field values to 80 characters --- catz.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/catz.py b/catz.py index ecdaca1..2839831 100755 --- a/catz.py +++ b/catz.py @@ -32,13 +32,17 @@ formatted_entry.append(formatted_field) if len(formatted_field) > field_lengths[field]: field_lengths[field] = len(formatted_field) - formatted_entries.append(formatted_entry) + formatted_entries.append(formatted_entry) +for i, field in enumerate(fields): + if field_lengths[field] > 80: + field_lengths[field] = 80 + for entry in formatted_entries: + if len(entry[i]) > field_lengths[field]: + entry[i] = entry[i][:field_lengths[field] - 1] + "…" row_format = "{:>" + "{}".format(len("{}".format(len(formatted_entries)))) + "} | " separator_line = [] for field in fields: - if field_lengths[field] > 50: - field_lengths[field] = 50 row_format += "{:<" + "{}".format(field_lengths[field]) + "} " separator_line.append("-" * field_lengths[field]) print(row_format.format("", *fields)) From 162e2c47fefb57a96c6b0e29e2d818dfc088ac6c Mon Sep 17 00:00:00 2001 From: Moritz Marquardt Date: Tue, 26 Jan 2021 11:39:46 +0100 Subject: [PATCH 6/6] Fix repeated logging message --- catz.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catz.py b/catz.py index 2839831..044e063 100755 --- a/catz.py +++ b/catz.py @@ -32,7 +32,7 @@ formatted_entry.append(formatted_field) if len(formatted_field) > field_lengths[field]: field_lengths[field] = len(formatted_field) - formatted_entries.append(formatted_entry) + formatted_entries.append(formatted_entry) for i, field in enumerate(fields): if field_lengths[field] > 80: field_lengths[field] = 80