rsyslog: Also rotate and parse stepup-logs

Author: phavekes

roles/rsyslog/templates/logrotate_stepupauth.j2

@@ -0,0 +1,16 @@
+{{ rsyslog_dir }}/log_logins/{{ item.name }}/stepup-authentication.log
+{
+    missingok
+    daily
+    rotate 180
+    sharedscripts
+    dateext
+    dateyesterday
+    compress
+    delaycompress
+    create 0640 root {{ rsyslog_read_group }}
+    postrotate
+      /usr/local/sbin/parse_stepupauth_to_mysql_{{ item.name }}.py > /dev/null
+      systemctl kill -s HUP rsyslog.service
+    endscript
+}

roles/rsyslog/templates/parse_stepupauthauth_to_mysql.py.j2

@@ -0,0 +1,81 @@
+#!/usr/bin/python3
+# This script parses the files produced by engineblock and inserts them into a mySQL table where the SURFconext stats module will analyse the data further
+# This script is intended to be used during logrotate
+# It picks up all files starting with ebauth- (all rotated files) and parses them
+
+import os
+import sys
+import shutil
+import json
+import MySQLdb
+from dateutil.parser import parse
+
+mysql_host="{{ item.db_loglogins_host }}"
+mysql_user="{{ item.db_loglogins_user }}"
+mysql_password="{{ item.db_loglogins_password }}"
+mysql_db="{{ item.db_loglogins_name }}"
+workdir="{{ rsyslog_dir }}/log_logins/{{ item.name}}/"
+
+db = MySQLdb.connect(mysql_host,mysql_user,mysql_password,mysql_db )
+cursor = db.cursor()
+
+def update_lastseen(user_id, date):
+    query = """
+    INSERT INTO last_login (userid, lastseen)
+    VALUES (%s, %s)
+    ON DUPLICATE KEY UPDATE
+    lastseen = GREATEST(lastseen, VALUES(lastseen))
+    """
+    try:
+        cursor.execute(query, (user_id, date))
+        db.commit()
+    except Exception as e:
+        db.rollback()
+        print(f"Error updating last_login for user {user_id}: {e}")
+
+def load_in_mysql(a,b,c,d,e,f,g,h):
+    sql = """insert into log_logins(idpentityid,spentityid,loginstamp,userid,keyid,sessionid,requestid,trustedproxyentityid) values(%s,%s,%s,%s,%s,%s,%s,%s)"""
+    try:
+      cursor.execute(sql, (a,b,c,d,e,f,g,h))
+      db.commit()
+    except:
+      db.rollback()
+      print(sql, (a,b,c,d,e,f,g,h))
+
+
+def parse_lines(a):
+  input_file = open((a), 'r')
+  for line in input_file:
+     try:
+        jsonline = line.split(']:',2)[1]
+        data = json.loads(jsonline)
+     except:
+        continue
+     idp = data["context"]["idp_entity_id"]
+     sp = data["context"]["sp_entity_id"]
+     timestamp = data["context"]["login_stamp"]
+     user_id = data["context"]["user_id"]
+     key_id = data["context"]["key_id"]
+     session_id = data["extra"]["session_id"]
+     request_id = data["extra"]["request_id"]
+     proxied_sp_entity_ids_list = data["context"]["proxied_sp_entity_ids"]
+     proxied_sp_entity_ids = ''.join(proxied_sp_entity_ids_list)
+     loginstamp=parse(timestamp).strftime("%Y-%m-%d %H:%M:%S")
+     last_login_date = parse(timestamp).strftime("%Y-%m-%d")
+     null = 'NULL'
+     if proxied_sp_entity_ids:
+        load_in_mysql(idp,proxied_sp_entity_ids,loginstamp,user_id,key_id,session_id,request_id,sp)
+     else:
+        load_in_mysql(idp,sp,loginstamp,user_id,key_id,session_id,request_id,null)
+     update_lastseen(user_id, last_login_date)
+
+## Loop over the files and parse them one by one
+for filename in os.listdir(workdir):
+  if os.path.isfile(workdir+filename) and filename.startswith("eb-authentication.log-") and not filename.endswith(".gz"):
+      filetoparse=(os.path.join(workdir, filename))
+      parse_lines(filetoparse)
+  else:
+      continue
+
+cursor.close()
+db.close()