From 3e6e8b75e15e7a10a67800f73338ab0e77df05e0 Mon Sep 17 00:00:00 2001 From: liberodark Date: Wed, 17 Apr 2019 16:47:12 +0200 Subject: [PATCH 01/35] Update README.md --- README.md | 72 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/README.md b/README.md index a273001..71cc82c 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,75 @@ +# Spacewalk Debian +Spacewalk Debian Installer + +## How to use : + +Copy and Past in your terminal : + +```bash +wget -Nnv https://raw.githubusercontent.com/liberodark/nrpe-installer/install.sh && chmod +x install.sh; ./install.sh +``` + +## SELinux : + + +```bash +yum install -y policycoreutils-python +grep denied /var/log/audit/audit.log | audit2allow -M nrpe +semodule -i nrpe.pp +``` + +Or use nrpe from github : + +```bash +wget -O nrpe.tar.gz https://github.com/liberodark/nrpe-installer/releases/download/1.0/nrpe.tar.gz +tar -xvf nrpe.tar.gz && sudo rm nrpe.tar.gz && semodule -i nrpe.pp +``` + +## Plugins Configuration : + +``` +command[service]=/usr/local/nagios/libexec/check_service.sh -o linux -t "systemctl list-units --state=failed" +command[memory]=/usr/local/nagios/libexec/check_mem.sh -w 70% -c 90% +command[memory_min]=/usr/local/nagios/libexec/check_mem.sh -w 70% -c 90% # For minimal informations +command[cpu]=/usr/local/nagios/libexec/check_cpu_utilization.sh -w 70 -c 90 +command[cpu_min]=/usr/local/nagios/libexec/check_cpu_utilization.sh -w 70 -c 90 # For minimal informations +command[users]=/usr/local/nagios/libexec/check_users -w 5 -c 10 +command[load]=/usr/local/nagios/libexec/check_load -w 15,10,5 -c 30,25,20 +command[check_load]=/usr/local/nagios/libexec/check_load -w 15,10,5 -c 30,25,20 +command[swap]=/usr/local/nagios/libexec/check_swap -w 20% -c 10% +command[root_disk]=/usr/local/nagios/libexec/check_disk -w 20% -c 10% -p / -m +command[usr_disk]=/usr/local/nagios/libexec/check_disk -w 20% -c 10% -p /usr -m +command[var_disk]=/usr/local/nagios/libexec/check_disk -w 20% -c 10% -p /var -m +command[zombie_procs]=/usr/local/nagios/libexec/check_procs -w 5 -c 10 -s Z +command[total_procs]=/usr/local/nagios/libexec/check_procs -w 190 -c 200 +command[proc_named]=/usr/local/nagios/libexec/check_procs -w 1: -c 1:2 -C named +command[proc_crond]=/usr/local/nagios/libexec/check_procs -w 1: -c 1:5 -C crond +command[proc_syslogd]=/usr/local/nagios/libexec/check_procs -w 1: -c 1:2 -C syslog-ng +command[proc_rsyslogd]=/usr/local/nagios/libexec/check_procs -w 1: -c 1:2 -C rsyslogd +``` + +## Debian 6.x / 7.x + +Save your source list : + +```cp -a /etc/apt/sources.list /etc/apt/sources.list.bak``` + +For Debian 6.x + +```echo "deb http://archive.debian.org/debian/ squeeze main" > /etc/apt/sources.list``` + +For Debian 7.x + +```echo "deb http://archive.debian.org/debian/ wheezy main" > /etc/apt/sources.list``` + +## Linux Compatibility : + +- Debian 7.x / 8.x / 9.x +- Ubuntu 18.04 + + +## Script informations : + - **getDebianAnnouncements.py** By https://github.com/rpasche This downloads all security announcements of debian from the current year and the year before and uses html2text to transform it to ascii text - **parseUbuntu.py** parses https://lists.ubuntu.com/archives/ubuntu-security-announce/$DATE.txt.gz into an XML which can be read by errata-import.pl / errata-import.py - **parseDebian.py** By https://github.com/rpasche the same as parseUbuntu.py, but parses all security announcements downloaded with getDebianAnnouncements.py and writes this to an XML file for later use with errata-import-debian.py From 9368116467ed86b1318f83d0fef6d0c95fd52115 Mon Sep 17 00:00:00 2001 From: liberodark Date: Wed, 17 Apr 2019 16:47:56 +0200 Subject: [PATCH 02/35] Update README.md --- README.md | 35 +---------------------------------- 1 file changed, 1 insertion(+), 34 deletions(-) diff --git a/README.md b/README.md index 71cc82c..c5cc601 100644 --- a/README.md +++ b/README.md @@ -9,43 +9,10 @@ Copy and Past in your terminal : wget -Nnv https://raw.githubusercontent.com/liberodark/nrpe-installer/install.sh && chmod +x install.sh; ./install.sh ``` -## SELinux : - - -```bash -yum install -y policycoreutils-python -grep denied /var/log/audit/audit.log | audit2allow -M nrpe -semodule -i nrpe.pp -``` - -Or use nrpe from github : - -```bash -wget -O nrpe.tar.gz https://github.com/liberodark/nrpe-installer/releases/download/1.0/nrpe.tar.gz -tar -xvf nrpe.tar.gz && sudo rm nrpe.tar.gz && semodule -i nrpe.pp -``` - ## Plugins Configuration : ``` -command[service]=/usr/local/nagios/libexec/check_service.sh -o linux -t "systemctl list-units --state=failed" -command[memory]=/usr/local/nagios/libexec/check_mem.sh -w 70% -c 90% -command[memory_min]=/usr/local/nagios/libexec/check_mem.sh -w 70% -c 90% # For minimal informations -command[cpu]=/usr/local/nagios/libexec/check_cpu_utilization.sh -w 70 -c 90 -command[cpu_min]=/usr/local/nagios/libexec/check_cpu_utilization.sh -w 70 -c 90 # For minimal informations -command[users]=/usr/local/nagios/libexec/check_users -w 5 -c 10 -command[load]=/usr/local/nagios/libexec/check_load -w 15,10,5 -c 30,25,20 -command[check_load]=/usr/local/nagios/libexec/check_load -w 15,10,5 -c 30,25,20 -command[swap]=/usr/local/nagios/libexec/check_swap -w 20% -c 10% -command[root_disk]=/usr/local/nagios/libexec/check_disk -w 20% -c 10% -p / -m -command[usr_disk]=/usr/local/nagios/libexec/check_disk -w 20% -c 10% -p /usr -m -command[var_disk]=/usr/local/nagios/libexec/check_disk -w 20% -c 10% -p /var -m -command[zombie_procs]=/usr/local/nagios/libexec/check_procs -w 5 -c 10 -s Z -command[total_procs]=/usr/local/nagios/libexec/check_procs -w 190 -c 200 -command[proc_named]=/usr/local/nagios/libexec/check_procs -w 1: -c 1:2 -C named -command[proc_crond]=/usr/local/nagios/libexec/check_procs -w 1: -c 1:5 -C crond -command[proc_syslogd]=/usr/local/nagios/libexec/check_procs -w 1: -c 1:2 -C syslog-ng -command[proc_rsyslogd]=/usr/local/nagios/libexec/check_procs -w 1: -c 1:2 -C rsyslogd + ``` ## Debian 6.x / 7.x From 083722042bbd57ed7a09d2e93652076ac3dbb236 Mon Sep 17 00:00:00 2001 From: liberodark Date: Wed, 17 Apr 2019 16:49:03 +0200 Subject: [PATCH 03/35] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c5cc601..4ffd356 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Spacewalk Debian Installer Copy and Past in your terminal : ```bash -wget -Nnv https://raw.githubusercontent.com/liberodark/nrpe-installer/install.sh && chmod +x install.sh; ./install.sh +wget -Nnv https://raw.githubusercontent.com/liberodark/spacewalk-scripts/install.sh && chmod +x install.sh; ./install.sh ``` ## Plugins Configuration : From e7c680255230fe647372f20cbb210d01f660a21b Mon Sep 17 00:00:00 2001 From: liberodark Date: Wed, 17 Apr 2019 16:52:15 +0200 Subject: [PATCH 04/35] Update README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 4ffd356..ab9432d 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,8 @@ wget -Nnv https://raw.githubusercontent.com/liberodark/spacewalk-scripts/install ``` +/usr/bin/spacewalk-repo-sync -c updates_centos_7_x86_64 + ``` ## Debian 6.x / 7.x From 36ce99a9a8d896bb90ba073ad12436fce0dfb29c Mon Sep 17 00:00:00 2001 From: liberodark Date: Wed, 17 Apr 2019 16:56:37 +0200 Subject: [PATCH 05/35] Update README.md --- README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index ab9432d..60abdb1 100644 --- a/README.md +++ b/README.md @@ -12,8 +12,11 @@ wget -Nnv https://raw.githubusercontent.com/liberodark/spacewalk-scripts/install ## Plugins Configuration : ``` - -/usr/bin/spacewalk-repo-sync -c updates_centos_7_x86_64 +python getDebianAnnouncements.py +sudo rmdir /tmp/debian_security/parsed +python parseDebian.py +python errata-import-debian.py +/usr/bin/spacewalk-repo-sync -c jessie_main ``` @@ -34,7 +37,6 @@ For Debian 7.x ## Linux Compatibility : - Debian 7.x / 8.x / 9.x -- Ubuntu 18.04 ## Script informations : From 75da36d73f8508d42dacb78d2f1579a13416cd29 Mon Sep 17 00:00:00 2001 From: liberodark Date: Wed, 17 Apr 2019 16:57:03 +0200 Subject: [PATCH 06/35] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 60abdb1..d2f9125 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,7 @@ wget -Nnv https://raw.githubusercontent.com/liberodark/spacewalk-scripts/install ## Plugins Configuration : ``` +yum install python-html2text python getDebianAnnouncements.py sudo rmdir /tmp/debian_security/parsed python parseDebian.py From 93612fc792d21635cf8dd0ab0e8e9a4aa7e46ecd Mon Sep 17 00:00:00 2001 From: liberodark Date: Wed, 17 Apr 2019 16:57:17 +0200 Subject: [PATCH 07/35] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index d2f9125..b39cc18 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,7 @@ wget -Nnv https://raw.githubusercontent.com/liberodark/spacewalk-scripts/install ## Plugins Configuration : ``` +yum install html2text yum install python-html2text python getDebianAnnouncements.py sudo rmdir /tmp/debian_security/parsed From 94aa9a522428e55927aca2d7091beec4ea76cc28 Mon Sep 17 00:00:00 2001 From: liberodark Date: Thu, 18 Apr 2019 10:15:26 +0200 Subject: [PATCH 08/35] Update README.md --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index b39cc18..9dab8e0 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,6 @@ sudo rmdir /tmp/debian_security/parsed python parseDebian.py python errata-import-debian.py /usr/bin/spacewalk-repo-sync -c jessie_main - ``` ## Debian 6.x / 7.x From ae2ae594dfb52ac0f0083d6ca28211d3854597eb Mon Sep 17 00:00:00 2001 From: liberodark Date: Thu, 18 Apr 2019 10:33:21 +0200 Subject: [PATCH 09/35] Update README.md --- README.md | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9dab8e0..c1fa19b 100644 --- a/README.md +++ b/README.md @@ -9,10 +9,22 @@ Copy and Past in your terminal : wget -Nnv https://raw.githubusercontent.com/liberodark/spacewalk-scripts/install.sh && chmod +x install.sh; ./install.sh ``` -## Plugins Configuration : +For Centos 7 + +- Download Scripts + +``` +git clone https://github.com/liberodark/spacewalk-scripts +``` + +- Install + +``` +yum install -y html2text +``` + ``` -yum install html2text yum install python-html2text python getDebianAnnouncements.py sudo rmdir /tmp/debian_security/parsed From a54e7dabaed769f940ac0272191eae82850e0638 Mon Sep 17 00:00:00 2001 From: liberodark Date: Thu, 18 Apr 2019 14:43:08 +0200 Subject: [PATCH 10/35] Update README.md --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index c1fa19b..24ac32c 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,6 @@ yum install -y html2text ``` -yum install python-html2text python getDebianAnnouncements.py sudo rmdir /tmp/debian_security/parsed python parseDebian.py From ad434709eb04ed02241a71d8249cedbcdbfca6e6 Mon Sep 17 00:00:00 2001 From: liberodark Date: Thu, 18 Apr 2019 14:47:20 +0200 Subject: [PATCH 11/35] add centos --- centos/errata-import-debian.py | 317 ++++++++++++++++++++++++ centos/getDebianAnnouncements.py | 48 ++++ centos/parseDebian.py | 398 ++++++++++++++++++++++++++++++ centos/spacewalk_sync_debian.cron | 28 +++ 4 files changed, 791 insertions(+) create mode 100644 centos/errata-import-debian.py create mode 100644 centos/getDebianAnnouncements.py create mode 100644 centos/parseDebian.py create mode 100644 centos/spacewalk_sync_debian.cron diff --git a/centos/errata-import-debian.py b/centos/errata-import-debian.py new file mode 100644 index 0000000..83c977e --- /dev/null +++ b/centos/errata-import-debian.py @@ -0,0 +1,317 @@ +#!/bin/env python +# +# This script is adopted from errata-import.py from +# Pedro Andujar || twitter: pandujar || email: @segfault.es || @digitalsec.net +# +# Robert Paschedag +# +# Changelog: +# 2016-08-08 - Print start of script with debug_level +# 2016-08-01 - Changed logging +# 2016-07-27 - Don't publish errata into excludedChannels +# 2016-05-24 - Initial working version + +from __future__ import print_function + +import sys +import xmlrpclib +import os +import errno +from datetime import datetime +import xml.etree.cElementTree as xml +import argparse + +debug_level = 1 + +parser = argparse.ArgumentParser(description='Spacewalk errata import for Debian') + +parser.add_argument("-d", "--debug", type=int, dest="debug_level", help="set debug level") +parser.add_argument("-i", "--included-channels", type=str, action="append", help="set included channels") +parser.add_argument("-e", "--excluded-channels", type=str, action="append", help="set excluded channels") + +option = parser.parse_args() + +if option.debug_level is not None and option.debug_level > 0: + debug_level = option.debug_level + +# Config Settings# +filename = '/tmp/debian_security/debian-errata.xml' +excludedChannels = ['stretch_main_main'] +includedChannels = ['wheezy_main', + 'jessie_main', + 'stretch_main'] +# Config Settings# + +url = 'http://localhost/rpc/api' +login = 'errata' +passwd = 'ObcTFUc3N2vV6R2ceFi9' +key = '' +client = '' +erratum = '' +issue_date = '' +erratainfo = {} +keywords = [] +packages = {} +bug = [] +publish = True +attempts = 0 + +if option.excluded_channels: + excludedChannels = option.excluded_channels + +if option.included_channels: + includedChannels = option.included_channels + +# xmlrpc connect to server and retrieve key +def connect(url, login, passwd): + try: + global client, key + client = xmlrpclib.Server(url, verbose=0) + if login == '': + login = os.getenv('SW_USER', '') + if passwd == '': + passwd = os.getenv('SW_PASSWD', '') + key = client.auth.login(login, passwd) + log(1, "[+] Connected to %s" % url) + except Exception, e: + log(1, "[-] Error connecting to %s: %s" % (url, e)) + sys.exit(1) + + +def logout(key): + client.auth.logout(key) + log(1, "[-] Clossing Session on %s" % url) + + +def log(level, s, f=sys.stdout): + """ + Write s to file f if l is smaller or equal to current debug level + Print _always_ if f == sys.stderr + """ + if level <= debug_level or f == sys.stderr: + print(s, file=f) + + +def createPackageList(client, key): + try: + log(1, "[+] Creating inventory from Server:") + if not includedChannels: + log(1, "[+] Retrieving the list of available channels") + tmpchannels = client.channel.listSoftwareChannels(key) + for label in tmpchannels: + if label.get('label') in excludedChannels: + log(1, "[-] Excluding channel: %s" % label.get('label')) + else: + log(1, "[+] Including channel: %s" % label.get('label')) + includedChannels.append(label.get('label')) + else: + log(1, "[+] Including channel(s): %s" % includedChannels) + + for label in includedChannels: + log(1, "[+] Retrieving Package List from Channel: %s" % label) + tmppackages = client.channel.software.listAllPackages(key, label) + for package in tmppackages: + # this will not work 100% because some packages are named like + # '...all-deb.deb', so for all architectures! We need to find the + # real name + + # sep = '-' + # packinfo = package["name"], package["version"], package["release"] + # fullpack = sep.join(packinfo) + '.amd64-deb.deb' + try: + packinfo = client.packages.getDetails(key, package['id']) + if not packinfo: + log(10, "[-] Error retrieving package details for package id %d" % package['id'], sys.stderr) + continue + except: + log(10, "[-] Error retrieving package details for package id %d" % package['id'], sys.stderr) + continue + + # this generates a hash of "package" objects (which contains "filename", etc.) + if not package['id'] in packages: + packages[package['id']] = packinfo + except Exception, e: + log(10, "[-] Error creating PackageList: %s" % e, sys.stderr) + + +def getPackagesAndChannels(filenames, packages, dist): + """Given a list of incomplete package filenames, searches + within 'packages' for a matching package object + It also verifies, that all packages + have been found within the same channel. Erratas with + packages divided into different channels might not + be deployable because a client might not subscribe + to all needed channels""" + + packages_found = [] + c = set() + + p1 = [] + # for every filename + for filename in filenames: + # first....find packages with "matching" filenames + for pkg in filter(lambda x: packages[x]['file'].startswith(filename), packages.keys()): + p1.append(packages[pkg]) + + # only include packages that have been found in channels where the label "starts" with + # the distribuion name (dist == "jessie" gets packages from "jessie_security_main", + # "jessie_security_contrib" but not "wheezy_security_main") + for p in p1: + for chan in p['providing_channels']: + if chan.startswith(dist): + packages_found.append(p) + + # each package within a "patch" must be within the same channels as the others + for idx, package in enumerate(packages_found): + if idx == 0: + # set has to be "initialized" in "first" run + # otherwise there would never be a positive + # intersection + c = set(package['providing_channels']) + else: + # build intersection. c is empty, if no channel + # is found in the other package and vice versa + c &= set(package['providing_channels']) + + # because there is still bug https://bugzilla.redhat.com/show_bug.cgi?id=1243387 present + # do NOT touch "main" channels (given in excludedChannels). If a "package" is also provided + # by one of these "main" channels, remove that channel! Otherwise, that "main" channel would + # get updates (by taskomatic) and the "Packages" file would get regenerated (in wrong way :-( ) + + # just test if a "main" channels would get updated and report that... + if excludedChannels and c & set(excludedChannels): + log(2, "Package '%s' is also provided by at least one excluded channel: (%s). This will be ignored here. Please check." % ( + package['file'], + ', '.join(excludedChannels) + )) + + # remove the possible "main" channels + c -= set(excludedChannels) + + if not c: + # once the set is empty, no need to continue + break + + if not c: + # return empty list of packages and empty list of channels + # if all packages are not within the same channel + return [], [] + + # we only need the ids here + return [p['id'] for p in packages_found], list(c) + + +# Check if the errata already exist or check if its applicable to any channel +def getDetailsErratum(client, key, erratum, issue_date, erratainfo, keywords, packageids, cves, publish, channels): + try: + eDetails = client.errata.getDetails(key, erratum) + log(1, "[-] %s already exist: check if update needed..." % erratum) + # generate sets of the packages currently included within this errata and + # from this *new* packageids. Packageids NOT within existing erratum is calculated with + # + # missing_ids = list(set(new packageids) - set(list of current packages in errata)) + missing_ids = list(set(packageids) - set([x['id'] for x in client.errata.listPackages(key, erratum)])) + if missing_ids: + log(1, "[+] adding %d missing packages to erratum %s" % (len(missing_ids), erratum)) + try: + ret = client.errata.addPackages(key, erratum, missing_ids) + if ret != len(missing_ids): + log(1, "[-] only %d of %d packages have been added to errata %s" % (ret, len(missing_ids), erratum)) + except: + log(1, "[-] update of errata %s failed!" % erratum) + else: + log(1, "[+] %s is up-to-date" % erratum) + return + except xmlrpclib.Fault: + log(0, "[+] %s doesn't exist: creating" % erratum) + createErratum(client, key, erratum, issue_date, erratainfo, keywords, packageids, cves, publish, channels) + + +# Create and publish errata +def createErratum(client, key, erratum, issue_date, erratainfo, keywords, packageids, cves, publish, channels): + try: + log(1, "[+] Creating errata %s:" % erratum) + log(2, client.errata.create(key, erratainfo, bug, keywords, packageids, publish, channels)) + # Include aditional info using setDetails method (not supported by create method) + # http://www.spacewalkproject.org/documentation/api/2.3/handlers/ErrataHandler.html#setDetails + client.errata.setDetails(key, erratum, {'update_date': issue_date}) + client.errata.setDetails(key, erratum, {'issue_date': issue_date}) + if publish: + client.errata.setDetails(key, erratum, {'cves': cves}) + except Exception, e: + log(10, "[-] Error creating errata: %s" % e, sys.stderr) + + +def parseXML(client, key, filename): + try: + log(1, "[+] Retrieving data from %s" % filename) + global erratum, erratainfo, newpackages, cves, issue_date, packageids, channels + adv = xml.parse(filename).getroot() + for element in adv: + # Resetting values + newpackages = [] + cves = [] + packageids = [] + channels = [] + keywords = [] + references = '' + erratum = element.tag + description = element.get('description')[:4000] + issue_date = datetime.strptime(element.get('issue_date')[:-6], '%a, %d %b %Y %H:%M:%S') + errataFrom = element.get('errataFrom') + synopsis = element.get('synopsis') + release = int(element.get('release')) + product = element.get('product') + topic = element.get('topic') + solution = element.get('solution') + notes = element.get('notes') + if element.get('keywords'): + keywords = [element.get('keywords')] + advisory_type = element.get('type') + if element.get('references'): + references = element.get('references') + + for cve in element.findall('cve'): + cves.append(cve.text) + # go through all "dists" and create an errata for every distribution + # the name of the distribution will be prepended to build the advisory_name + # (e.g. given "DSA-1234-1", will create "jessie-DSA-1234-1") + for dist in element.findall('dist'): + # change the erratum (advisory_name). Prepend dist + distname = dist.get('name') + dist_erratum = distname + '-' + erratum + newpackages, channels = getPackagesAndChannels([p.text for p in dist.findall('package')], packages, distname) + if not newpackages: + log(2, "[-] Error finding packages for advisory %s for distribution %s." % (erratum, distname)) + # continue with next distribution + continue + + # Prepare struct for method + # create:http://www.spacewalkproject.org/documentation/api/2.3/handlers/ErrataHandler.html#create + erratainfo = {'synopsis': synopsis, 'advisory_name': dist_erratum, 'advisory_release': release, + 'advisory_type': advisory_type, 'product': product, 'errataFrom': errataFrom, + 'topic': topic, 'description': description, 'references': references, 'notes': notes, + 'solution': solution} + getDetailsErratum(client, key, dist_erratum, issue_date, erratainfo, keywords, newpackages, cves, + publish, channels) + except Exception, e: + log(10, "[-] Error parsing %s: %s" % (filename, e), sys.stderr) + + +def main(): + print("Started errata import..... Debug level: %d" % debug_level) + try: + os.stat(filename) + except OSError as e: + if e.errno == errno.ENOENT: + raise + + connect(url, login, passwd) + createPackageList(client, key) + parseXML(client, key, filename) + logout(key) + print("Finished errata import") + + +if __name__ == "__main__": + main() diff --git a/centos/getDebianAnnouncements.py b/centos/getDebianAnnouncements.py new file mode 100644 index 0000000..a945e3e --- /dev/null +++ b/centos/getDebianAnnouncements.py @@ -0,0 +1,48 @@ +#!/usr/bin/python -tt +# +# Author: robert.paschedag@netlution.de +# +# This script downloads the Debian security announcements +# and saves them to a temporary directory for later parsing +# +# Changelog: +# +# 2016-05-10 - Do not create subdirectories for the announcements +# 2016-05-02 - Initial working version + +import re +import sys +import os +from datetime import date +import tempfile +import subprocess +from urllib import urlopen + +# we download all security announcements from this year and the year before +years = [str(date.today().year - 1), str(date.today().year)] +base_url = 'https://lists.debian.org/debian-security-announce/' +tmp_dir = '/tmp/debian_security/' +msg_href_regex = re.compile('(?Pmsg\d{5}\.html)') + +# create temporary directory if not present +try: + os.stat(tmp_dir) +except OSError: + os.mkdir(tmp_dir) + +for year in years: + # show all announcements of the year + year_url = urlopen(base_url + 'debian-security-announce-' + year + '/threads.html') + for line in year_url: + m = msg_href_regex.search(line) + if m: + # open msg.... + msg_url = urlopen(base_url + year + '/' + m.group('msg_ref')) + # ... and write it to our temp directory + with open(tmp_dir + year + '-' + m.group('msg_ref'), "w+") as tmp_msg: + # tmp_msg.write(msg_url.read()) + # parse through html2text + child = subprocess.Popen(['html2text', '-'], stdin=subprocess.PIPE, stdout=tmp_msg, stderr=None) + ret = child.communicate(msg_url.read().replace('\r\n', '\n')) + +sys.exit(0) diff --git a/centos/parseDebian.py b/centos/parseDebian.py new file mode 100644 index 0000000..833b096 --- /dev/null +++ b/centos/parseDebian.py @@ -0,0 +1,398 @@ +#!/usr/bin/python +# +# Author: robert.paschedag@netlution.de +# +# This script is adopted from the very good parseUbuntu.py of +# Philipp Schuler (philipp.schuler@holidaycheck.com) +# +# Changelog: +# +# 2016-05-10 Initial version +# 2018-03-12 Only parse new patches + +import re +import urllib2 +import os +import errno +import traceback +import sys +import xml.etree.cElementTree as XML + + +class MessageAnnounce: + + def __init__(self, + errata_type=None, + errata_name=None, + errata_id=None, + errata_release=None, + errata_year=None, + errata_severity=None, + errata_synopsis=None, + errata_date=None, + errata_from=None, + errata_desc=None, + errata_reboot='', + errata_dist=None, + msg_subject=None, + references=''): + + self.packages = {} + self.cves = list() + + self.errataType = 'Security Advisory' + self.errataName = errata_name + self.errataID = errata_id + self.errataRelease = errata_release + self.errataYear = errata_year + self.errataSynopsis = errata_synopsis + self.errataDate = errata_date + self.errataFrom = errata_from + self.errataDesc = errata_desc + self.errataReboot = errata_reboot + self.errataDist = errata_dist + self.messageSubject = msg_subject + self.errataReferences = references + + def getAdvisoryName(self): + advisory_name = "DSA-%s-%s" % (self.errataID, self.errataRelease) + return advisory_name + + +class MessagePackageInfo: + + def __init__(self, pkg_release, pkg_file, pkg_version): + self.release = pkg_release + self.filename = pkg_file + self.version = pkg_version + + +class MessageParser(object): + # release should be the release of the channel we are working on + # + # global debian binary search url prefix + SOURCE_URL = 'https://packages.debian.org/source' + BIN_PACKAGELIST_REGEX = '^.*(?P).*$' + BIN_PACKAGE_REGEX = '
(?P.*?)
' + DIST = "distribution \((?P.*?)\)" + VERSION = "version (?P\S+?)(?=\.?( |$))" + PKGINFO = "Package\s+:\s(?P.*)" + MAILING_LIST = "^Mailing list:" + ERRATA_INFO = "_Subject_: \[SECURITY\] \[DSA (?P\d+)-(?P\d+)\] (?P\S+)( (?P.*))?$" + DATE = "_Date_:\s+(?P.*?)(?=(\s\(.*\))?$)" + FROM = "_From_:\s+(?P.*)$" + CVE = "(?PCVE-\d{4}-\d{4})" + EOH = '-----BEGIN PGP SIGNED MESSAGE-----' + + bin_packagelist_re = re.compile(BIN_PACKAGELIST_REGEX) + bin_package_re = re.compile(BIN_PACKAGE_REGEX) + pkginfo_re = re.compile(PKGINFO) + mailing_list_re = re.compile(MAILING_LIST) + erratum_info_re = re.compile(ERRATA_INFO) + date_re = re.compile(DATE) + from_re = re.compile(FROM) + cve_re = re.compile(CVE) + version_re = re.compile(VERSION) + header_re = re.compile(r'^.*\s+:\s+.*$') + eoh_re = re.compile(EOH) + + def _get_bin_packages(self, dist, sourcepackage): + """ + Call url of source package + """ + + packages = [] + + try: + resp = urllib2.urlopen("%s/%s/%s" % (self.SOURCE_URL, dist, sourcepackage)) + url_data = re.sub("\n", "", resp.read()) + resp.close() + except urllib2.HTTPError as e: + print "Failed to fetch information for package '%s' in distribution '%s'" % (sourcepackage, dist) + print e.reason + return packages + + # search for package list match + packages_match = self.bin_packagelist_re.search(url_data) + + if packages_match: + packages = self.bin_package_re.findall(packages_match.group('bin_packages')) + + return packages + + # parse reboot + def processMessageReboot(self, message_body): + reboot = '' + + if message_body.find('to reboot') != -1: + reboot = 'reboot_suggested' + + return reboot + + # parse the summary and details + def processMessageSummary(self, message_body): + summary = '' + summary_found = False + package_found = False + header_found = False + + for line in message_body.split('\n'): + # this parsing sucks... + # sorry...Debian....but parsing Ubuntu security is easier ;-) + # we start finding the package re, and continue + # to ignore all following "header" lines. The first + # non-header line marks our "summary" + package_match = MessageParser.pkginfo_re.match(line) + header_match = MessageParser.header_re.match(line) + mailing_match = MessageParser.mailing_list_re.match(line) + + # set our flag if found... + if header_match: + header_found = True + + # set if not found + if not header_match: + header_found = False + + if package_match: + package_found = True + + # end of summary ? Found line "Mailing list:"? + if mailing_match: + break + + # continue, until we find a Package line + if not package_found: + continue + + # continue, if we are still in header section + if package_found and header_found: + continue + + # because package_found stays True if found once, here, header_found + # must be False, which indicates our summary beginning + summary_found = True + + if summary_found: + summary += line + '\n' + + if summary == '': + summary = 'Parsing description failed' + + return summary + + def processMessageCVEs(self, message_body): + cves = list() + + for line in message_body.split('\n'): + cve_match = re.findall(MessageParser.CVE, line) + if cve_match: + cves.extend(cve_match) + + # generate list with unique cves + cves = list(set(cves)) + return cves + + # parse the message summary for version strings and build "filenames" + # incomplete filenames. REAL filenames have to be searched later. + # "all-deb" packages also possible in amd64-deb channel!!! + def processPackageList(self, err_name, message_body): + current_release = None + dist_packages = {} + + # for each "paragraph" + for p in message_body.split('\n\n'): + p = re.sub("\n", "", p) + + dists = re.findall(MessageParser.DIST, p) + v_match = MessageParser.version_re.search(p) + mailing_match = MessageParser.mailing_list_re.match(p) + + # if we ran over the packages, then stop + if mailing_match: + break + + if dists and v_match: + for dist in dists: + # get all binary packages this "source" package creates + bin_packages = self._get_bin_packages(dist, err_name) + if not bin_packages: + continue + dist_packages[dist] = [pkg + '-' + v_match.group('version') for pkg in bin_packages] + + return dist_packages + + # Construct the basic details about the errata from the message subject + def processMessageSubject(self, message_subject): + subject_found = False + + parsed_msg = MessageAnnounce() + + for line in message_subject.split("\n"): + erratum_info_match = MessageParser.erratum_info_re.search(line) + eoh_match = MessageParser.eoh_re.search(line) + date_match = MessageParser.date_re.search(line) + from_match = MessageParser.from_re.search(line) + + if eoh_match: + break + + if erratum_info_match: + subject_found = True + # in very few security announcements, the subject line is not consistent + # and "errata_other" might not be found + if erratum_info_match.group('errata_other') is None: + parsed_msg.messageSubject = parsed_msg.errataSynopsis = erratum_info_match.group('errata_name') + else: + parsed_msg.messageSubject = parsed_msg.errataSynopsis = erratum_info_match.group('errata_name') + ' ' + erratum_info_match.group('errata_other') + parsed_msg.errataName = erratum_info_match.group('errata_name') + parsed_msg.errataID = erratum_info_match.group('errata_id') + parsed_msg.errataRelease = erratum_info_match.group('errata_release') + # parsed_msg.errataSynopsis = erratum_info_match.group('errata_name') + ' ' + erratum_info_match.group('errata_other') + continue + + if date_match: + parsed_msg.errataDate = date_match.group('errata_date') + continue + + if from_match: + parsed_msg.errataFrom = from_match.group('errata_from') + continue + + if not subject_found: + print "Message not parseable" + return None + + return parsed_msg + + # Processes an individual mailing list message and returns a messageAnnounce object or none if parsing failed + # Really bad parsing errors lead to an exception + def processMessage(self, message_text): + try: + parsed_msg = self.processMessageSubject(message_text) + + if parsed_msg is None: + return None + + parsed_msg.packages = self.processPackageList(parsed_msg.errataName, message_text) + parsed_msg.errataDesc = self.processMessageSummary(message_text) + parsed_msg.errataReboot = self.processMessageReboot(message_text) + # parsed_msg.errataReferences = self.processMessageReferences(message_text) + parsed_msg.cves = self.processMessageCVEs(message_text) + + return parsed_msg + except Exception, e: + print "Failed to process message. Reason:" + print e + traceback.print_exc(file=sys.stdout) + + return None + + # Performs parsing on the specified errata source. What this + # actually means will vary between the different parsers + # Will return list of MessageAnnounce objects, or throw an exception + def parse(self): + raise NotImplementedError("This method is implemented in subclasses, you should not call it from MessageParser") + + +class MessageFile(MessageParser): + + def __init__(self, input_file): + self.inputFile = input_file + + def parse(self): + inputData = open(self.inputFile).read() + + return self.processMessage(inputData) + + +def main(): + + security_msg = "/tmp/debian_security/" + errata_file = 'debian-errata.xml' + parsed_dir = security_msg + 'parsed/' + + try: + os.stat(parsed_dir) + except OSError: + os.mkdir(parsed_dir) + + # remove errata_file from previous run + try: + os.remove(security_msg + errata_file) + except OSError as e: + if e.errno != errno.ENOENT: + raise + + try: + files = filter(lambda f: re.match('^\d{4}-msg\d{5}\.html', f), os.listdir(security_msg)) + parsed_files = filter(lambda f: re.match('^\d{4}-msg\d{5}\.html', f), os.listdir(parsed_dir)) + + files_to_parse = list(set(files) - set(parsed_files)) + announcements = list() + + if not files_to_parse: + print "No security announcements to parse today. Bye." + sys.exit(0) + + if len(files_to_parse) != len(files): + print "Ignoring %d old security announcements already parsed." % (len(files) - len(files_to_parse)) + + for idx, f in enumerate(files_to_parse, start=1): + print "Processing patch %d/%d (file: %s)" % (idx, len(files_to_parse), f) + message_parser = MessageFile(os.path.join(security_msg, f)) + errata = message_parser.parse() + if errata: + announcements.append(errata) + # move the file to our parsed files dir + os.rename(security_msg + f, parsed_dir + f) + + # if there are no advisories, just quit + if not announcements: + print "No security announcements available or parseable. Bye." + sys.exit(0) + + # write the advisory XML + opt = XML.Element('patches') + for advisory in announcements: + adv = XML.SubElement(opt, advisory.getAdvisoryName()) + adv.set('description', advisory.errataDesc.strip()) + adv.set('issue_date', advisory.errataDate) + adv.set('errataFrom', advisory.errataFrom) + # prepend advisory name to synopsis. This makes it easier to search for the errata in Spacewalk WebUI + adv.set('synopsis', advisory.getAdvisoryName() + ' ' + advisory.errataSynopsis) + adv.set('release', advisory.errataRelease) + adv.set('product', 'Debian Linux') + adv.set('topic', 'N/A') + adv.set('solution', 'N/A') + adv.set('notes', 'N/A') + if advisory.errataReboot != "": + adv.set('keywords', advisory.errataReboot) + adv.set('type', advisory.errataType) + # adv.set('references', advisory.errataReferences.strip()) + + # for every distribution (jessie, stretch, ...) + # add the packages + for dist in advisory.packages: + d = XML.SubElement(adv, 'dist') + d.set('name', dist) + for package in advisory.packages[dist]: + pkg = XML.SubElement(d, 'package') + pkg.text = package + + # add CVEs + for cve in advisory.cves: + c = XML.SubElement(adv, 'cve') + c.text = cve + + xml = XML.ElementTree(opt) + xml.write(security_msg + errata_file) + + except Exception, e: + print "Failed to parse messages due to exception %s" % e + traceback.print_exc(file=sys.stdout) + sys.exit(2) + + +if __name__ == "__main__": + main() diff --git a/centos/spacewalk_sync_debian.cron b/centos/spacewalk_sync_debian.cron new file mode 100644 index 0000000..bc55769 --- /dev/null +++ b/centos/spacewalk_sync_debian.cron @@ -0,0 +1,28 @@ +MAILTO=root + +# try to create the lock and check the outcome +LOCKFILE=/var/run/spacewalk_sync_debian.lock +#lockfile -r 0 ${LOCKFILE} 1>/dev/null 2>&1 +#status=$? +#if [ ${status} -ne 0 ] ;then +if [ -e "$LOCKFILE" ]; then + echo "Another instance already running. Aborting." + exit 1 +else + touch "$LOCKFILE" +fi +trap "rm ${LOCKFILE}" EXIT + +#sync channels and publish updates +/usr/bin/spacewalk-repo-sync -c wheezy_main +/usr/bin/spacewalk-repo-sync -c jessie_main +/usr/bin/spacewalk-repo-sync -c stretch_main + + +#get errata file and checksums +cd /home/errata/spacewalk-scripts-master/ + +mkdir -p /tmp/debian_security/parsed +python getDebianAnnouncements.py +python parseDebian.py +python errata-import-debian.py \ No newline at end of file From 3cbea88954f414f0ac3060324e0d17a76781a2cf Mon Sep 17 00:00:00 2001 From: liberodark Date: Thu, 18 Apr 2019 14:52:23 +0200 Subject: [PATCH 12/35] up --- centos/errata-import-debian.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/centos/errata-import-debian.py b/centos/errata-import-debian.py index 83c977e..9802ac3 100644 --- a/centos/errata-import-debian.py +++ b/centos/errata-import-debian.py @@ -43,8 +43,8 @@ # Config Settings# url = 'http://localhost/rpc/api' -login = 'errata' -passwd = 'ObcTFUc3N2vV6R2ceFi9' +login = 'MYLOGIN' +passwd = 'MYPASSWORD' key = '' client = '' erratum = '' From 372296d2ddd04e128a4db17ab53f2bb8271634c2 Mon Sep 17 00:00:00 2001 From: liberodark Date: Thu, 18 Apr 2019 15:00:09 +0200 Subject: [PATCH 13/35] up --- centos/spacewalk_sync_debian.cron | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/centos/spacewalk_sync_debian.cron b/centos/spacewalk_sync_debian.cron index bc55769..8c5165c 100644 --- a/centos/spacewalk_sync_debian.cron +++ b/centos/spacewalk_sync_debian.cron @@ -20,7 +20,7 @@ trap "rm ${LOCKFILE}" EXIT #get errata file and checksums -cd /home/errata/spacewalk-scripts-master/ +cd /home/errata/spacewalk-scripts/ mkdir -p /tmp/debian_security/parsed python getDebianAnnouncements.py From 69115015131d252b583d510a71564f889038ef41 Mon Sep 17 00:00:00 2001 From: liberodark Date: Thu, 18 Apr 2019 15:00:23 +0200 Subject: [PATCH 14/35] Update README.md --- README.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/README.md b/README.md index 24ac32c..fa02d1a 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,24 @@ git clone https://github.com/liberodark/spacewalk-scripts yum install -y html2text ``` +``` +cd spacewalk-scripts/centos +cp -a spacewalk_sync_debian.cron /etc/cron.daily/spacewalk_sync_debian.cron +``` + +- Edit errata-import-debian.py + +L46 = MYLOGIN and L47 = MYPASSWORD + +- Install Files : + +``` +mkdir -p /home/errata/spacewalk-scripts/ +cp -a * /home/errata/spacewalk-scripts/ +``` + + +## Manual Use ``` python getDebianAnnouncements.py From e3199ec72eba05c0ee6369e32ed34917cc4cc76d Mon Sep 17 00:00:00 2001 From: liberodark Date: Thu, 18 Apr 2019 15:01:25 +0200 Subject: [PATCH 15/35] Update README.md --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index fa02d1a..358071c 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,10 @@ cp -a spacewalk_sync_debian.cron /etc/cron.daily/spacewalk_sync_debian.cron - Edit errata-import-debian.py -L46 = MYLOGIN and L47 = MYPASSWORD +``` +login = 'MYLOGIN' # Line 46 +passwd = 'MYPASSWORD' # Line 47 +``` - Install Files : From 74c829838e9f430afd8acced8adc341cb10caee1 Mon Sep 17 00:00:00 2001 From: liberodark Date: Thu, 18 Apr 2019 15:02:50 +0200 Subject: [PATCH 16/35] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 358071c..2ce2cde 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ passwd = 'MYPASSWORD' # Line 47 ``` mkdir -p /home/errata/spacewalk-scripts/ -cp -a * /home/errata/spacewalk-scripts/ +mv *.py /home/errata/spacewalk-scripts/ ``` From f62ee0a0bfac4ce164036f9bb7c1c2ac48394737 Mon Sep 17 00:00:00 2001 From: liberodark Date: Fri, 19 Apr 2019 10:01:39 +0200 Subject: [PATCH 17/35] add installer --- install.sh | 71 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 install.sh diff --git a/install.sh b/install.sh new file mode 100644 index 0000000..92eeb7a --- /dev/null +++ b/install.sh @@ -0,0 +1,71 @@ +#!/bin/bash +# +# About: Install Script of Spacewalk automatically +# Author: liberodark +# License: GNU GPLv3 + +#================================================= +# CHECK UPDATE +#================================================= + + update_source="https://raw.githubusercontent.com/liberodark/spacewalk-scripts/master/install.sh" + version="0.0.1" + + echo "Welcome on Spacewalk Script Install $version" + + # make update if asked + if [ "$1" = "noupdate" ]; then + update_status="false" + else + update_status="true" + fi ; + + # update updater + if [ "$update_status" = "true" ]; then + wget -O $0 $update_source + $0 noupdate + exit 0 +fi ; + +#================================================= +# CHECK ROOT +#================================================= + +if [[ $(id -u) -ne 0 ]] ; then echo "Please run as root" ; exit 1 ; fi + +#================================================= +# IP +#================================================= + +echo "What is your user of spacewalk ?" +read user + +echo "What is your password of spacewalk ?" +read password + +#================================================= +# RETRIEVE ARGUMENTS FROM THE MANIFEST AND VAR +#================================================= + +distribution=$(cat /etc/*release | head -n +1 | awk '{print $1}') + +#============================================== +# SPACEWALK +#============================================== +echo "Install Script for Spacewalk" + +# Check OS & spacewalk + + if [ $? != 1 ]; then + + if [[ "$distribution" =~ .CentOS || "$distribution" = CentOS ]]; then + yum install -y html2text + cd centos/ + cp -a spacewalk_sync_debian.cron /etc/cron.daily/spacewalk_sync_debian.cron + sed -i "s@MYLOGIN@${user}@@g" errata-import-debian.py + sed -i "s@MYPASSWORD@${password}@@g" errata-import-debian.py + mkdir -p /home/errata/spacewalk-scripts/ + mv *.py /home/errata/spacewalk-scripts/ + + fi +fi \ No newline at end of file From a25ad1a9c39eb4af3b2c09f916f49dbc6e98fa99 Mon Sep 17 00:00:00 2001 From: liberodark Date: Fri, 19 Apr 2019 10:04:37 +0200 Subject: [PATCH 18/35] update --- install.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/install.sh b/install.sh index 92eeb7a..baa4f9b 100644 --- a/install.sh +++ b/install.sh @@ -59,7 +59,9 @@ echo "Install Script for Spacewalk" if [ $? != 1 ]; then if [[ "$distribution" =~ .CentOS || "$distribution" = CentOS ]]; then - yum install -y html2text + yum install -y html2text wget + wget https://github.com/liberodark/spacewalk-scripts/releases/download/1.0/centos.tar.gz + tar -xvf centos.tar.gz & rm centos.tar.gz cd centos/ cp -a spacewalk_sync_debian.cron /etc/cron.daily/spacewalk_sync_debian.cron sed -i "s@MYLOGIN@${user}@@g" errata-import-debian.py From 6f9195f56fa70445094f69202159bbc77b6bf89a Mon Sep 17 00:00:00 2001 From: liberodark Date: Fri, 19 Apr 2019 10:06:30 +0200 Subject: [PATCH 19/35] Update README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 2ce2cde..5b4f6c0 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,8 @@ Copy and Past in your terminal : wget -Nnv https://raw.githubusercontent.com/liberodark/spacewalk-scripts/install.sh && chmod +x install.sh; ./install.sh ``` +## How to use Manualy : + For Centos 7 - Download Scripts From ca3a05773629d3e7e85a03e3abb16f90787b1339 Mon Sep 17 00:00:00 2001 From: liberodark Date: Fri, 19 Apr 2019 10:23:41 +0200 Subject: [PATCH 20/35] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5b4f6c0..a3e7a32 100644 --- a/README.md +++ b/README.md @@ -48,8 +48,8 @@ mv *.py /home/errata/spacewalk-scripts/ ## Manual Use ``` +mkdir -p /tmp/debian_security/parsed python getDebianAnnouncements.py -sudo rmdir /tmp/debian_security/parsed python parseDebian.py python errata-import-debian.py /usr/bin/spacewalk-repo-sync -c jessie_main From 44b3a43d30e87f82b56cd94e512be3e4d29c4540 Mon Sep 17 00:00:00 2001 From: liberodark Date: Fri, 19 Apr 2019 11:16:05 +0200 Subject: [PATCH 21/35] clean --- centos/errata-import-debian.py | 317 -------------- centos/getDebianAnnouncements.py | 48 --- centos/parseDebian.py | 398 ------------------ install.sh | 10 +- ..._debian.cron => spacewalk_sync_debian.cron | 5 +- 5 files changed, 7 insertions(+), 771 deletions(-) delete mode 100644 centos/errata-import-debian.py delete mode 100644 centos/getDebianAnnouncements.py delete mode 100644 centos/parseDebian.py rename centos/spacewalk_sync_debian.cron => spacewalk_sync_debian.cron (88%) diff --git a/centos/errata-import-debian.py b/centos/errata-import-debian.py deleted file mode 100644 index 9802ac3..0000000 --- a/centos/errata-import-debian.py +++ /dev/null @@ -1,317 +0,0 @@ -#!/bin/env python -# -# This script is adopted from errata-import.py from -# Pedro Andujar || twitter: pandujar || email: @segfault.es || @digitalsec.net -# -# Robert Paschedag -# -# Changelog: -# 2016-08-08 - Print start of script with debug_level -# 2016-08-01 - Changed logging -# 2016-07-27 - Don't publish errata into excludedChannels -# 2016-05-24 - Initial working version - -from __future__ import print_function - -import sys -import xmlrpclib -import os -import errno -from datetime import datetime -import xml.etree.cElementTree as xml -import argparse - -debug_level = 1 - -parser = argparse.ArgumentParser(description='Spacewalk errata import for Debian') - -parser.add_argument("-d", "--debug", type=int, dest="debug_level", help="set debug level") -parser.add_argument("-i", "--included-channels", type=str, action="append", help="set included channels") -parser.add_argument("-e", "--excluded-channels", type=str, action="append", help="set excluded channels") - -option = parser.parse_args() - -if option.debug_level is not None and option.debug_level > 0: - debug_level = option.debug_level - -# Config Settings# -filename = '/tmp/debian_security/debian-errata.xml' -excludedChannels = ['stretch_main_main'] -includedChannels = ['wheezy_main', - 'jessie_main', - 'stretch_main'] -# Config Settings# - -url = 'http://localhost/rpc/api' -login = 'MYLOGIN' -passwd = 'MYPASSWORD' -key = '' -client = '' -erratum = '' -issue_date = '' -erratainfo = {} -keywords = [] -packages = {} -bug = [] -publish = True -attempts = 0 - -if option.excluded_channels: - excludedChannels = option.excluded_channels - -if option.included_channels: - includedChannels = option.included_channels - -# xmlrpc connect to server and retrieve key -def connect(url, login, passwd): - try: - global client, key - client = xmlrpclib.Server(url, verbose=0) - if login == '': - login = os.getenv('SW_USER', '') - if passwd == '': - passwd = os.getenv('SW_PASSWD', '') - key = client.auth.login(login, passwd) - log(1, "[+] Connected to %s" % url) - except Exception, e: - log(1, "[-] Error connecting to %s: %s" % (url, e)) - sys.exit(1) - - -def logout(key): - client.auth.logout(key) - log(1, "[-] Clossing Session on %s" % url) - - -def log(level, s, f=sys.stdout): - """ - Write s to file f if l is smaller or equal to current debug level - Print _always_ if f == sys.stderr - """ - if level <= debug_level or f == sys.stderr: - print(s, file=f) - - -def createPackageList(client, key): - try: - log(1, "[+] Creating inventory from Server:") - if not includedChannels: - log(1, "[+] Retrieving the list of available channels") - tmpchannels = client.channel.listSoftwareChannels(key) - for label in tmpchannels: - if label.get('label') in excludedChannels: - log(1, "[-] Excluding channel: %s" % label.get('label')) - else: - log(1, "[+] Including channel: %s" % label.get('label')) - includedChannels.append(label.get('label')) - else: - log(1, "[+] Including channel(s): %s" % includedChannels) - - for label in includedChannels: - log(1, "[+] Retrieving Package List from Channel: %s" % label) - tmppackages = client.channel.software.listAllPackages(key, label) - for package in tmppackages: - # this will not work 100% because some packages are named like - # '...all-deb.deb', so for all architectures! We need to find the - # real name - - # sep = '-' - # packinfo = package["name"], package["version"], package["release"] - # fullpack = sep.join(packinfo) + '.amd64-deb.deb' - try: - packinfo = client.packages.getDetails(key, package['id']) - if not packinfo: - log(10, "[-] Error retrieving package details for package id %d" % package['id'], sys.stderr) - continue - except: - log(10, "[-] Error retrieving package details for package id %d" % package['id'], sys.stderr) - continue - - # this generates a hash of "package" objects (which contains "filename", etc.) - if not package['id'] in packages: - packages[package['id']] = packinfo - except Exception, e: - log(10, "[-] Error creating PackageList: %s" % e, sys.stderr) - - -def getPackagesAndChannels(filenames, packages, dist): - """Given a list of incomplete package filenames, searches - within 'packages' for a matching package object - It also verifies, that all packages - have been found within the same channel. Erratas with - packages divided into different channels might not - be deployable because a client might not subscribe - to all needed channels""" - - packages_found = [] - c = set() - - p1 = [] - # for every filename - for filename in filenames: - # first....find packages with "matching" filenames - for pkg in filter(lambda x: packages[x]['file'].startswith(filename), packages.keys()): - p1.append(packages[pkg]) - - # only include packages that have been found in channels where the label "starts" with - # the distribuion name (dist == "jessie" gets packages from "jessie_security_main", - # "jessie_security_contrib" but not "wheezy_security_main") - for p in p1: - for chan in p['providing_channels']: - if chan.startswith(dist): - packages_found.append(p) - - # each package within a "patch" must be within the same channels as the others - for idx, package in enumerate(packages_found): - if idx == 0: - # set has to be "initialized" in "first" run - # otherwise there would never be a positive - # intersection - c = set(package['providing_channels']) - else: - # build intersection. c is empty, if no channel - # is found in the other package and vice versa - c &= set(package['providing_channels']) - - # because there is still bug https://bugzilla.redhat.com/show_bug.cgi?id=1243387 present - # do NOT touch "main" channels (given in excludedChannels). If a "package" is also provided - # by one of these "main" channels, remove that channel! Otherwise, that "main" channel would - # get updates (by taskomatic) and the "Packages" file would get regenerated (in wrong way :-( ) - - # just test if a "main" channels would get updated and report that... - if excludedChannels and c & set(excludedChannels): - log(2, "Package '%s' is also provided by at least one excluded channel: (%s). This will be ignored here. Please check." % ( - package['file'], - ', '.join(excludedChannels) - )) - - # remove the possible "main" channels - c -= set(excludedChannels) - - if not c: - # once the set is empty, no need to continue - break - - if not c: - # return empty list of packages and empty list of channels - # if all packages are not within the same channel - return [], [] - - # we only need the ids here - return [p['id'] for p in packages_found], list(c) - - -# Check if the errata already exist or check if its applicable to any channel -def getDetailsErratum(client, key, erratum, issue_date, erratainfo, keywords, packageids, cves, publish, channels): - try: - eDetails = client.errata.getDetails(key, erratum) - log(1, "[-] %s already exist: check if update needed..." % erratum) - # generate sets of the packages currently included within this errata and - # from this *new* packageids. Packageids NOT within existing erratum is calculated with - # - # missing_ids = list(set(new packageids) - set(list of current packages in errata)) - missing_ids = list(set(packageids) - set([x['id'] for x in client.errata.listPackages(key, erratum)])) - if missing_ids: - log(1, "[+] adding %d missing packages to erratum %s" % (len(missing_ids), erratum)) - try: - ret = client.errata.addPackages(key, erratum, missing_ids) - if ret != len(missing_ids): - log(1, "[-] only %d of %d packages have been added to errata %s" % (ret, len(missing_ids), erratum)) - except: - log(1, "[-] update of errata %s failed!" % erratum) - else: - log(1, "[+] %s is up-to-date" % erratum) - return - except xmlrpclib.Fault: - log(0, "[+] %s doesn't exist: creating" % erratum) - createErratum(client, key, erratum, issue_date, erratainfo, keywords, packageids, cves, publish, channels) - - -# Create and publish errata -def createErratum(client, key, erratum, issue_date, erratainfo, keywords, packageids, cves, publish, channels): - try: - log(1, "[+] Creating errata %s:" % erratum) - log(2, client.errata.create(key, erratainfo, bug, keywords, packageids, publish, channels)) - # Include aditional info using setDetails method (not supported by create method) - # http://www.spacewalkproject.org/documentation/api/2.3/handlers/ErrataHandler.html#setDetails - client.errata.setDetails(key, erratum, {'update_date': issue_date}) - client.errata.setDetails(key, erratum, {'issue_date': issue_date}) - if publish: - client.errata.setDetails(key, erratum, {'cves': cves}) - except Exception, e: - log(10, "[-] Error creating errata: %s" % e, sys.stderr) - - -def parseXML(client, key, filename): - try: - log(1, "[+] Retrieving data from %s" % filename) - global erratum, erratainfo, newpackages, cves, issue_date, packageids, channels - adv = xml.parse(filename).getroot() - for element in adv: - # Resetting values - newpackages = [] - cves = [] - packageids = [] - channels = [] - keywords = [] - references = '' - erratum = element.tag - description = element.get('description')[:4000] - issue_date = datetime.strptime(element.get('issue_date')[:-6], '%a, %d %b %Y %H:%M:%S') - errataFrom = element.get('errataFrom') - synopsis = element.get('synopsis') - release = int(element.get('release')) - product = element.get('product') - topic = element.get('topic') - solution = element.get('solution') - notes = element.get('notes') - if element.get('keywords'): - keywords = [element.get('keywords')] - advisory_type = element.get('type') - if element.get('references'): - references = element.get('references') - - for cve in element.findall('cve'): - cves.append(cve.text) - # go through all "dists" and create an errata for every distribution - # the name of the distribution will be prepended to build the advisory_name - # (e.g. given "DSA-1234-1", will create "jessie-DSA-1234-1") - for dist in element.findall('dist'): - # change the erratum (advisory_name). Prepend dist - distname = dist.get('name') - dist_erratum = distname + '-' + erratum - newpackages, channels = getPackagesAndChannels([p.text for p in dist.findall('package')], packages, distname) - if not newpackages: - log(2, "[-] Error finding packages for advisory %s for distribution %s." % (erratum, distname)) - # continue with next distribution - continue - - # Prepare struct for method - # create:http://www.spacewalkproject.org/documentation/api/2.3/handlers/ErrataHandler.html#create - erratainfo = {'synopsis': synopsis, 'advisory_name': dist_erratum, 'advisory_release': release, - 'advisory_type': advisory_type, 'product': product, 'errataFrom': errataFrom, - 'topic': topic, 'description': description, 'references': references, 'notes': notes, - 'solution': solution} - getDetailsErratum(client, key, dist_erratum, issue_date, erratainfo, keywords, newpackages, cves, - publish, channels) - except Exception, e: - log(10, "[-] Error parsing %s: %s" % (filename, e), sys.stderr) - - -def main(): - print("Started errata import..... Debug level: %d" % debug_level) - try: - os.stat(filename) - except OSError as e: - if e.errno == errno.ENOENT: - raise - - connect(url, login, passwd) - createPackageList(client, key) - parseXML(client, key, filename) - logout(key) - print("Finished errata import") - - -if __name__ == "__main__": - main() diff --git a/centos/getDebianAnnouncements.py b/centos/getDebianAnnouncements.py deleted file mode 100644 index a945e3e..0000000 --- a/centos/getDebianAnnouncements.py +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/python -tt -# -# Author: robert.paschedag@netlution.de -# -# This script downloads the Debian security announcements -# and saves them to a temporary directory for later parsing -# -# Changelog: -# -# 2016-05-10 - Do not create subdirectories for the announcements -# 2016-05-02 - Initial working version - -import re -import sys -import os -from datetime import date -import tempfile -import subprocess -from urllib import urlopen - -# we download all security announcements from this year and the year before -years = [str(date.today().year - 1), str(date.today().year)] -base_url = 'https://lists.debian.org/debian-security-announce/' -tmp_dir = '/tmp/debian_security/' -msg_href_regex = re.compile('(?Pmsg\d{5}\.html)') - -# create temporary directory if not present -try: - os.stat(tmp_dir) -except OSError: - os.mkdir(tmp_dir) - -for year in years: - # show all announcements of the year - year_url = urlopen(base_url + 'debian-security-announce-' + year + '/threads.html') - for line in year_url: - m = msg_href_regex.search(line) - if m: - # open msg.... - msg_url = urlopen(base_url + year + '/' + m.group('msg_ref')) - # ... and write it to our temp directory - with open(tmp_dir + year + '-' + m.group('msg_ref'), "w+") as tmp_msg: - # tmp_msg.write(msg_url.read()) - # parse through html2text - child = subprocess.Popen(['html2text', '-'], stdin=subprocess.PIPE, stdout=tmp_msg, stderr=None) - ret = child.communicate(msg_url.read().replace('\r\n', '\n')) - -sys.exit(0) diff --git a/centos/parseDebian.py b/centos/parseDebian.py deleted file mode 100644 index 833b096..0000000 --- a/centos/parseDebian.py +++ /dev/null @@ -1,398 +0,0 @@ -#!/usr/bin/python -# -# Author: robert.paschedag@netlution.de -# -# This script is adopted from the very good parseUbuntu.py of -# Philipp Schuler (philipp.schuler@holidaycheck.com) -# -# Changelog: -# -# 2016-05-10 Initial version -# 2018-03-12 Only parse new patches - -import re -import urllib2 -import os -import errno -import traceback -import sys -import xml.etree.cElementTree as XML - - -class MessageAnnounce: - - def __init__(self, - errata_type=None, - errata_name=None, - errata_id=None, - errata_release=None, - errata_year=None, - errata_severity=None, - errata_synopsis=None, - errata_date=None, - errata_from=None, - errata_desc=None, - errata_reboot='', - errata_dist=None, - msg_subject=None, - references=''): - - self.packages = {} - self.cves = list() - - self.errataType = 'Security Advisory' - self.errataName = errata_name - self.errataID = errata_id - self.errataRelease = errata_release - self.errataYear = errata_year - self.errataSynopsis = errata_synopsis - self.errataDate = errata_date - self.errataFrom = errata_from - self.errataDesc = errata_desc - self.errataReboot = errata_reboot - self.errataDist = errata_dist - self.messageSubject = msg_subject - self.errataReferences = references - - def getAdvisoryName(self): - advisory_name = "DSA-%s-%s" % (self.errataID, self.errataRelease) - return advisory_name - - -class MessagePackageInfo: - - def __init__(self, pkg_release, pkg_file, pkg_version): - self.release = pkg_release - self.filename = pkg_file - self.version = pkg_version - - -class MessageParser(object): - # release should be the release of the channel we are working on - # - # global debian binary search url prefix - SOURCE_URL = 'https://packages.debian.org/source' - BIN_PACKAGELIST_REGEX = '^.*(?P).*$' - BIN_PACKAGE_REGEX = '
(?P.*?)
' - DIST = "distribution \((?P.*?)\)" - VERSION = "version (?P\S+?)(?=\.?( |$))" - PKGINFO = "Package\s+:\s(?P.*)" - MAILING_LIST = "^Mailing list:" - ERRATA_INFO = "_Subject_: \[SECURITY\] \[DSA (?P\d+)-(?P\d+)\] (?P\S+)( (?P.*))?$" - DATE = "_Date_:\s+(?P.*?)(?=(\s\(.*\))?$)" - FROM = "_From_:\s+(?P.*)$" - CVE = "(?PCVE-\d{4}-\d{4})" - EOH = '-----BEGIN PGP SIGNED MESSAGE-----' - - bin_packagelist_re = re.compile(BIN_PACKAGELIST_REGEX) - bin_package_re = re.compile(BIN_PACKAGE_REGEX) - pkginfo_re = re.compile(PKGINFO) - mailing_list_re = re.compile(MAILING_LIST) - erratum_info_re = re.compile(ERRATA_INFO) - date_re = re.compile(DATE) - from_re = re.compile(FROM) - cve_re = re.compile(CVE) - version_re = re.compile(VERSION) - header_re = re.compile(r'^.*\s+:\s+.*$') - eoh_re = re.compile(EOH) - - def _get_bin_packages(self, dist, sourcepackage): - """ - Call url of source package - """ - - packages = [] - - try: - resp = urllib2.urlopen("%s/%s/%s" % (self.SOURCE_URL, dist, sourcepackage)) - url_data = re.sub("\n", "", resp.read()) - resp.close() - except urllib2.HTTPError as e: - print "Failed to fetch information for package '%s' in distribution '%s'" % (sourcepackage, dist) - print e.reason - return packages - - # search for package list match - packages_match = self.bin_packagelist_re.search(url_data) - - if packages_match: - packages = self.bin_package_re.findall(packages_match.group('bin_packages')) - - return packages - - # parse reboot - def processMessageReboot(self, message_body): - reboot = '' - - if message_body.find('to reboot') != -1: - reboot = 'reboot_suggested' - - return reboot - - # parse the summary and details - def processMessageSummary(self, message_body): - summary = '' - summary_found = False - package_found = False - header_found = False - - for line in message_body.split('\n'): - # this parsing sucks... - # sorry...Debian....but parsing Ubuntu security is easier ;-) - # we start finding the package re, and continue - # to ignore all following "header" lines. The first - # non-header line marks our "summary" - package_match = MessageParser.pkginfo_re.match(line) - header_match = MessageParser.header_re.match(line) - mailing_match = MessageParser.mailing_list_re.match(line) - - # set our flag if found... - if header_match: - header_found = True - - # set if not found - if not header_match: - header_found = False - - if package_match: - package_found = True - - # end of summary ? Found line "Mailing list:"? - if mailing_match: - break - - # continue, until we find a Package line - if not package_found: - continue - - # continue, if we are still in header section - if package_found and header_found: - continue - - # because package_found stays True if found once, here, header_found - # must be False, which indicates our summary beginning - summary_found = True - - if summary_found: - summary += line + '\n' - - if summary == '': - summary = 'Parsing description failed' - - return summary - - def processMessageCVEs(self, message_body): - cves = list() - - for line in message_body.split('\n'): - cve_match = re.findall(MessageParser.CVE, line) - if cve_match: - cves.extend(cve_match) - - # generate list with unique cves - cves = list(set(cves)) - return cves - - # parse the message summary for version strings and build "filenames" - # incomplete filenames. REAL filenames have to be searched later. - # "all-deb" packages also possible in amd64-deb channel!!! - def processPackageList(self, err_name, message_body): - current_release = None - dist_packages = {} - - # for each "paragraph" - for p in message_body.split('\n\n'): - p = re.sub("\n", "", p) - - dists = re.findall(MessageParser.DIST, p) - v_match = MessageParser.version_re.search(p) - mailing_match = MessageParser.mailing_list_re.match(p) - - # if we ran over the packages, then stop - if mailing_match: - break - - if dists and v_match: - for dist in dists: - # get all binary packages this "source" package creates - bin_packages = self._get_bin_packages(dist, err_name) - if not bin_packages: - continue - dist_packages[dist] = [pkg + '-' + v_match.group('version') for pkg in bin_packages] - - return dist_packages - - # Construct the basic details about the errata from the message subject - def processMessageSubject(self, message_subject): - subject_found = False - - parsed_msg = MessageAnnounce() - - for line in message_subject.split("\n"): - erratum_info_match = MessageParser.erratum_info_re.search(line) - eoh_match = MessageParser.eoh_re.search(line) - date_match = MessageParser.date_re.search(line) - from_match = MessageParser.from_re.search(line) - - if eoh_match: - break - - if erratum_info_match: - subject_found = True - # in very few security announcements, the subject line is not consistent - # and "errata_other" might not be found - if erratum_info_match.group('errata_other') is None: - parsed_msg.messageSubject = parsed_msg.errataSynopsis = erratum_info_match.group('errata_name') - else: - parsed_msg.messageSubject = parsed_msg.errataSynopsis = erratum_info_match.group('errata_name') + ' ' + erratum_info_match.group('errata_other') - parsed_msg.errataName = erratum_info_match.group('errata_name') - parsed_msg.errataID = erratum_info_match.group('errata_id') - parsed_msg.errataRelease = erratum_info_match.group('errata_release') - # parsed_msg.errataSynopsis = erratum_info_match.group('errata_name') + ' ' + erratum_info_match.group('errata_other') - continue - - if date_match: - parsed_msg.errataDate = date_match.group('errata_date') - continue - - if from_match: - parsed_msg.errataFrom = from_match.group('errata_from') - continue - - if not subject_found: - print "Message not parseable" - return None - - return parsed_msg - - # Processes an individual mailing list message and returns a messageAnnounce object or none if parsing failed - # Really bad parsing errors lead to an exception - def processMessage(self, message_text): - try: - parsed_msg = self.processMessageSubject(message_text) - - if parsed_msg is None: - return None - - parsed_msg.packages = self.processPackageList(parsed_msg.errataName, message_text) - parsed_msg.errataDesc = self.processMessageSummary(message_text) - parsed_msg.errataReboot = self.processMessageReboot(message_text) - # parsed_msg.errataReferences = self.processMessageReferences(message_text) - parsed_msg.cves = self.processMessageCVEs(message_text) - - return parsed_msg - except Exception, e: - print "Failed to process message. Reason:" - print e - traceback.print_exc(file=sys.stdout) - - return None - - # Performs parsing on the specified errata source. What this - # actually means will vary between the different parsers - # Will return list of MessageAnnounce objects, or throw an exception - def parse(self): - raise NotImplementedError("This method is implemented in subclasses, you should not call it from MessageParser") - - -class MessageFile(MessageParser): - - def __init__(self, input_file): - self.inputFile = input_file - - def parse(self): - inputData = open(self.inputFile).read() - - return self.processMessage(inputData) - - -def main(): - - security_msg = "/tmp/debian_security/" - errata_file = 'debian-errata.xml' - parsed_dir = security_msg + 'parsed/' - - try: - os.stat(parsed_dir) - except OSError: - os.mkdir(parsed_dir) - - # remove errata_file from previous run - try: - os.remove(security_msg + errata_file) - except OSError as e: - if e.errno != errno.ENOENT: - raise - - try: - files = filter(lambda f: re.match('^\d{4}-msg\d{5}\.html', f), os.listdir(security_msg)) - parsed_files = filter(lambda f: re.match('^\d{4}-msg\d{5}\.html', f), os.listdir(parsed_dir)) - - files_to_parse = list(set(files) - set(parsed_files)) - announcements = list() - - if not files_to_parse: - print "No security announcements to parse today. Bye." - sys.exit(0) - - if len(files_to_parse) != len(files): - print "Ignoring %d old security announcements already parsed." % (len(files) - len(files_to_parse)) - - for idx, f in enumerate(files_to_parse, start=1): - print "Processing patch %d/%d (file: %s)" % (idx, len(files_to_parse), f) - message_parser = MessageFile(os.path.join(security_msg, f)) - errata = message_parser.parse() - if errata: - announcements.append(errata) - # move the file to our parsed files dir - os.rename(security_msg + f, parsed_dir + f) - - # if there are no advisories, just quit - if not announcements: - print "No security announcements available or parseable. Bye." - sys.exit(0) - - # write the advisory XML - opt = XML.Element('patches') - for advisory in announcements: - adv = XML.SubElement(opt, advisory.getAdvisoryName()) - adv.set('description', advisory.errataDesc.strip()) - adv.set('issue_date', advisory.errataDate) - adv.set('errataFrom', advisory.errataFrom) - # prepend advisory name to synopsis. This makes it easier to search for the errata in Spacewalk WebUI - adv.set('synopsis', advisory.getAdvisoryName() + ' ' + advisory.errataSynopsis) - adv.set('release', advisory.errataRelease) - adv.set('product', 'Debian Linux') - adv.set('topic', 'N/A') - adv.set('solution', 'N/A') - adv.set('notes', 'N/A') - if advisory.errataReboot != "": - adv.set('keywords', advisory.errataReboot) - adv.set('type', advisory.errataType) - # adv.set('references', advisory.errataReferences.strip()) - - # for every distribution (jessie, stretch, ...) - # add the packages - for dist in advisory.packages: - d = XML.SubElement(adv, 'dist') - d.set('name', dist) - for package in advisory.packages[dist]: - pkg = XML.SubElement(d, 'package') - pkg.text = package - - # add CVEs - for cve in advisory.cves: - c = XML.SubElement(adv, 'cve') - c.text = cve - - xml = XML.ElementTree(opt) - xml.write(security_msg + errata_file) - - except Exception, e: - print "Failed to parse messages due to exception %s" % e - traceback.print_exc(file=sys.stdout) - sys.exit(2) - - -if __name__ == "__main__": - main() diff --git a/install.sh b/install.sh index baa4f9b..9ce51ec 100644 --- a/install.sh +++ b/install.sh @@ -59,14 +59,14 @@ echo "Install Script for Spacewalk" if [ $? != 1 ]; then if [[ "$distribution" =~ .CentOS || "$distribution" = CentOS ]]; then - yum install -y html2text wget - wget https://github.com/liberodark/spacewalk-scripts/releases/download/1.0/centos.tar.gz - tar -xvf centos.tar.gz & rm centos.tar.gz - cd centos/ + yum install -y html2text git + mkdir -p /home/errata/spacewalk-scripts/ + git clone https://github.com/liberodark/spacewalk-scripts/ + mv spacewalk-scripts /home/errata/spacewalk-scripts cp -a spacewalk_sync_debian.cron /etc/cron.daily/spacewalk_sync_debian.cron sed -i "s@MYLOGIN@${user}@@g" errata-import-debian.py sed -i "s@MYPASSWORD@${password}@@g" errata-import-debian.py - mkdir -p /home/errata/spacewalk-scripts/ + #chmod 777 mv *.py /home/errata/spacewalk-scripts/ fi diff --git a/centos/spacewalk_sync_debian.cron b/spacewalk_sync_debian.cron similarity index 88% rename from centos/spacewalk_sync_debian.cron rename to spacewalk_sync_debian.cron index 8c5165c..5f158ba 100644 --- a/centos/spacewalk_sync_debian.cron +++ b/spacewalk_sync_debian.cron @@ -23,6 +23,5 @@ trap "rm ${LOCKFILE}" EXIT cd /home/errata/spacewalk-scripts/ mkdir -p /tmp/debian_security/parsed -python getDebianAnnouncements.py -python parseDebian.py -python errata-import-debian.py \ No newline at end of file +python errata-import-debian.py +sudo rm -r /tmp/debian_security \ No newline at end of file From e477e9255b060ce584b4a7e609d7359c1b4ccd64 Mon Sep 17 00:00:00 2001 From: liberodark Date: Fri, 19 Apr 2019 11:16:48 +0200 Subject: [PATCH 22/35] up --- install.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/install.sh b/install.sh index 9ce51ec..0e11f5e 100644 --- a/install.sh +++ b/install.sh @@ -67,7 +67,6 @@ echo "Install Script for Spacewalk" sed -i "s@MYLOGIN@${user}@@g" errata-import-debian.py sed -i "s@MYPASSWORD@${password}@@g" errata-import-debian.py #chmod 777 - mv *.py /home/errata/spacewalk-scripts/ fi fi \ No newline at end of file From 8ddd3e7ab7f93f87009c8c6b258e5f2c2f4e893e Mon Sep 17 00:00:00 2001 From: liberodark Date: Fri, 19 Apr 2019 11:24:51 +0200 Subject: [PATCH 23/35] update --- install.sh | 9 ++++++++- spacewalk_errata_debian.cron | 23 +++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 spacewalk_errata_debian.cron diff --git a/install.sh b/install.sh index 0e11f5e..32c1687 100644 --- a/install.sh +++ b/install.sh @@ -66,7 +66,14 @@ echo "Install Script for Spacewalk" cp -a spacewalk_sync_debian.cron /etc/cron.daily/spacewalk_sync_debian.cron sed -i "s@MYLOGIN@${user}@@g" errata-import-debian.py sed -i "s@MYPASSWORD@${password}@@g" errata-import-debian.py - #chmod 777 + + elif [[ "$distribution" =~ .Debian || "$distribution" = Debian || "$distribution" =~ .Ubuntu || "$distribution" = Ubuntu ]]; then + apt-get update + apt-get install -y html2text git cron + mkdir -p /home/errata/spacewalk-scripts/ + git clone https://github.com/liberodark/spacewalk-scripts/ + mv spacewalk-scripts /home/errata/spacewalk-scripts + cp -a spacewalk_errata_debian.cron /etc/cron.daily/spacewalk_errata_debian.cron fi fi \ No newline at end of file diff --git a/spacewalk_errata_debian.cron b/spacewalk_errata_debian.cron new file mode 100644 index 0000000..c5bb051 --- /dev/null +++ b/spacewalk_errata_debian.cron @@ -0,0 +1,23 @@ +MAILTO=root + +# try to create the lock and check the outcome +LOCKFILE=/var/run/spacewalk_errata_debian.lock +#lockfile -r 0 ${LOCKFILE} 1>/dev/null 2>&1 +#status=$? +#if [ ${status} -ne 0 ] ;then +if [ -e "$LOCKFILE" ]; then + echo "Another instance already running. Aborting." + exit 1 +else + touch "$LOCKFILE" +fi +trap "rm ${LOCKFILE}" EXIT + + +#get errata file and checksums +cd /home/errata/spacewalk-scripts/ + + +python getDebianAnnouncements.py +python parseDebian.py +scp -r -p /tmp/debian_security ssh_user@ssh_ip:/tmp/debian_security \ No newline at end of file From 3a5006d8450e798a3368884142765c72c53f6913 Mon Sep 17 00:00:00 2001 From: liberodark Date: Fri, 19 Apr 2019 11:28:36 +0200 Subject: [PATCH 24/35] better echo --- install.sh | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/install.sh b/install.sh index 32c1687..a442676 100644 --- a/install.sh +++ b/install.sh @@ -33,16 +33,6 @@ fi ; if [[ $(id -u) -ne 0 ]] ; then echo "Please run as root" ; exit 1 ; fi -#================================================= -# IP -#================================================= - -echo "What is your user of spacewalk ?" -read user - -echo "What is your password of spacewalk ?" -read password - #================================================= # RETRIEVE ARGUMENTS FROM THE MANIFEST AND VAR #================================================= @@ -59,21 +49,34 @@ echo "Install Script for Spacewalk" if [ $? != 1 ]; then if [[ "$distribution" =~ .CentOS || "$distribution" = CentOS ]]; then + echo "What is your user of spacewalk ?" + read user + + echo "What is your password of spacewalk ?" + read password + yum install -y html2text git mkdir -p /home/errata/spacewalk-scripts/ git clone https://github.com/liberodark/spacewalk-scripts/ mv spacewalk-scripts /home/errata/spacewalk-scripts cp -a spacewalk_sync_debian.cron /etc/cron.daily/spacewalk_sync_debian.cron - sed -i "s@MYLOGIN@${user}@@g" errata-import-debian.py - sed -i "s@MYPASSWORD@${password}@@g" errata-import-debian.py + sed -i "s@MYLOGIN@${user}@@g" /home/errata/spacewalk-scripts/errata-import-debian.py + sed -i "s@MYPASSWORD@${password}@@g" /home/errata/spacewalk-scripts/errata-import-debian.py elif [[ "$distribution" =~ .Debian || "$distribution" = Debian || "$distribution" =~ .Ubuntu || "$distribution" = Ubuntu ]]; then + echo "What is your user of spacewalk ?" + read ssh_user + + echo "What is your password of spacewalk ?" + read ssh_ip apt-get update apt-get install -y html2text git cron mkdir -p /home/errata/spacewalk-scripts/ git clone https://github.com/liberodark/spacewalk-scripts/ mv spacewalk-scripts /home/errata/spacewalk-scripts cp -a spacewalk_errata_debian.cron /etc/cron.daily/spacewalk_errata_debian.cron + sed -i "s@ssh_user@${ssh_user}@@g" /etc/cron.daily/spacewalk_errata_debian.cron + sed -i "s@ssh_ip@${ssh_ip}@@g" /etc/cron.daily/spacewalk_errata_debian.cron fi fi \ No newline at end of file From eb68c4254ee01b01598aa32826761fc7c78768e7 Mon Sep 17 00:00:00 2001 From: liberodark Date: Fri, 19 Apr 2019 11:29:14 +0200 Subject: [PATCH 25/35] new version --- install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install.sh b/install.sh index a442676..6410ed4 100644 --- a/install.sh +++ b/install.sh @@ -9,7 +9,7 @@ #================================================= update_source="https://raw.githubusercontent.com/liberodark/spacewalk-scripts/master/install.sh" - version="0.0.1" + version="0.0.4" echo "Welcome on Spacewalk Script Install $version" From 5a90bb90d2c6f1e04df3fcb921e90c93f9acaf26 Mon Sep 17 00:00:00 2001 From: liberodark Date: Fri, 19 Apr 2019 11:31:32 +0200 Subject: [PATCH 26/35] add password --- install.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/install.sh b/install.sh index 6410ed4..fddfd66 100644 --- a/install.sh +++ b/install.sh @@ -64,11 +64,15 @@ echo "Install Script for Spacewalk" sed -i "s@MYPASSWORD@${password}@@g" /home/errata/spacewalk-scripts/errata-import-debian.py elif [[ "$distribution" =~ .Debian || "$distribution" = Debian || "$distribution" =~ .Ubuntu || "$distribution" = Ubuntu ]]; then - echo "What is your user of spacewalk ?" + echo "What is your ssh user of spacewalk ?" read ssh_user - echo "What is your password of spacewalk ?" + echo "What is your ssh password of spacewalk ?" + read ssh_password + + echo "What is your ip of spacewalk ?" read ssh_ip + apt-get update apt-get install -y html2text git cron mkdir -p /home/errata/spacewalk-scripts/ From 370014f86f98a9f08d0a9c1755ff0eb4ba60fa39 Mon Sep 17 00:00:00 2001 From: liberodark Date: Fri, 19 Apr 2019 11:53:28 +0200 Subject: [PATCH 27/35] add password support --- install.sh | 5 +++-- spacewalk_errata_debian.cron | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/install.sh b/install.sh index fddfd66..b2f0278 100644 --- a/install.sh +++ b/install.sh @@ -72,14 +72,15 @@ echo "Install Script for Spacewalk" echo "What is your ip of spacewalk ?" read ssh_ip - + apt-get update - apt-get install -y html2text git cron + apt-get install -y html2text git sshpass mkdir -p /home/errata/spacewalk-scripts/ git clone https://github.com/liberodark/spacewalk-scripts/ mv spacewalk-scripts /home/errata/spacewalk-scripts cp -a spacewalk_errata_debian.cron /etc/cron.daily/spacewalk_errata_debian.cron sed -i "s@ssh_user@${ssh_user}@@g" /etc/cron.daily/spacewalk_errata_debian.cron + sed -i "s@ssh_password@${ssh_password}@@g" /etc/cron.daily/spacewalk_errata_debian.cron sed -i "s@ssh_ip@${ssh_ip}@@g" /etc/cron.daily/spacewalk_errata_debian.cron fi diff --git a/spacewalk_errata_debian.cron b/spacewalk_errata_debian.cron index c5bb051..1c99315 100644 --- a/spacewalk_errata_debian.cron +++ b/spacewalk_errata_debian.cron @@ -20,4 +20,4 @@ cd /home/errata/spacewalk-scripts/ python getDebianAnnouncements.py python parseDebian.py -scp -r -p /tmp/debian_security ssh_user@ssh_ip:/tmp/debian_security \ No newline at end of file +sshpass -p "ssh_password" scp -r -p /tmp/debian_security ssh_user@ssh_ip:/tmp/debian_security \ No newline at end of file From cd0c693e982320d6ac1b6b7bb8bac0dcc5bb1e5f Mon Sep 17 00:00:00 2001 From: liberodark Date: Fri, 19 Apr 2019 12:14:20 +0200 Subject: [PATCH 28/35] Update README.md --- README.md | 58 +++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 48 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index a3e7a32..73ee615 100644 --- a/README.md +++ b/README.md @@ -9,9 +9,26 @@ Copy and Past in your terminal : wget -Nnv https://raw.githubusercontent.com/liberodark/spacewalk-scripts/install.sh && chmod +x install.sh; ./install.sh ``` -## How to use Manualy : +## How is work : -For Centos 7 +For Centos 7 is a client + +Just install the script ! + +For Debian 9 is a server + +Just install the script ! +need to make a first ssh connexion on your spacewalk from this debian + +## How is use manually : + +On Debian + +- Install + +``` +apt install -y html2text git +``` - Download Scripts @@ -19,20 +36,44 @@ For Centos 7 git clone https://github.com/liberodark/spacewalk-scripts ``` -- Install +- Need to edit spacewalk_errata_debian.cron with ip / user / password ``` -yum install -y html2text +nano spacewalk_errata_debian.cron +cp -a spacewalk_errata_debian.cron /etc/cron.daily/spacewalk_errata_debian.cron +``` + +- Install Files : + ``` +mkdir -p /home/errata/spacewalk-scripts/ +mv spacewalk-scripts /home/errata/spacewalk-scripts +``` + +On Centos + + +``` +yum install -y git +``` + +- Download Scripts + +``` +git clone https://github.com/liberodark/spacewalk-scripts +``` + +- Need to install file ``` -cd spacewalk-scripts/centos -cp -a spacewalk_sync_debian.cron /etc/cron.daily/spacewalk_sync_debian.cron +nano spacewalk_sync_debian.cron +cp -a nano spacewalk_sync_debian.cron /etc/cron.daily/nano spacewalk_sync_debian.cron ``` - Edit errata-import-debian.py ``` +nano errata-import-debian.py login = 'MYLOGIN' # Line 46 passwd = 'MYPASSWORD' # Line 47 ``` @@ -41,16 +82,13 @@ passwd = 'MYPASSWORD' # Line 47 ``` mkdir -p /home/errata/spacewalk-scripts/ -mv *.py /home/errata/spacewalk-scripts/ +mv spacewalk-scripts /home/errata/spacewalk-scripts ``` ## Manual Use ``` -mkdir -p /tmp/debian_security/parsed -python getDebianAnnouncements.py -python parseDebian.py python errata-import-debian.py /usr/bin/spacewalk-repo-sync -c jessie_main ``` From 456c2287752fbcfef232318ad49b35e1b2f217ed Mon Sep 17 00:00:00 2001 From: liberodark Date: Fri, 19 Apr 2019 12:14:52 +0200 Subject: [PATCH 29/35] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 73ee615..4ff5b92 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ need to make a first ssh connexion on your spacewalk from this debian ## How is use manually : -On Debian +### On Debian - Install @@ -50,7 +50,7 @@ mkdir -p /home/errata/spacewalk-scripts/ mv spacewalk-scripts /home/errata/spacewalk-scripts ``` -On Centos +### On Centos ``` From cee087e0ee3f949e7b125eef1ad227937a96a242 Mon Sep 17 00:00:00 2001 From: liberodark Date: Fri, 19 Apr 2019 12:15:26 +0200 Subject: [PATCH 30/35] Update README.md --- README.md | 8 -------- 1 file changed, 8 deletions(-) diff --git a/README.md b/README.md index 4ff5b92..57db49d 100644 --- a/README.md +++ b/README.md @@ -85,14 +85,6 @@ mkdir -p /home/errata/spacewalk-scripts/ mv spacewalk-scripts /home/errata/spacewalk-scripts ``` - -## Manual Use - -``` -python errata-import-debian.py -/usr/bin/spacewalk-repo-sync -c jessie_main -``` - ## Debian 6.x / 7.x Save your source list : From 54e249f7eff695ea01a66aacdebcb0730c704bb9 Mon Sep 17 00:00:00 2001 From: liberodark Date: Fri, 19 Apr 2019 12:17:18 +0200 Subject: [PATCH 31/35] Update README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 57db49d..2bc06d5 100644 --- a/README.md +++ b/README.md @@ -101,7 +101,8 @@ For Debian 7.x ## Linux Compatibility : -- Debian 7.x / 8.x / 9.x +- Debian 8.x / 9.x +- Centos 7.x ## Script informations : From 22c6a867007fc1aabacd15a51381b7b552788cc8 Mon Sep 17 00:00:00 2001 From: liberodark Date: Fri, 19 Apr 2019 12:20:12 +0200 Subject: [PATCH 32/35] Update README.md --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 2bc06d5..0ba81bb 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,9 @@ # Spacewalk Debian -Spacewalk Debian Installer +Spacewalk Script Installer + +This script is for client / server installation. +Centos = client and execute daily cron task for import errata and sync chanels. +Debian = server and execute daily cron task for download errata and send on spacewalk. ## How to use : From d5cc335184d668b9c01072e99409930778ec6bea Mon Sep 17 00:00:00 2001 From: liberodark Date: Fri, 19 Apr 2019 12:20:41 +0200 Subject: [PATCH 33/35] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0ba81bb..4753f82 100644 --- a/README.md +++ b/README.md @@ -2,8 +2,8 @@ Spacewalk Script Installer This script is for client / server installation. -Centos = client and execute daily cron task for import errata and sync chanels. -Debian = server and execute daily cron task for download errata and send on spacewalk. +- Centos = client and execute daily cron task for import errata and sync chanels. +- Debian = server and execute daily cron task for download errata and send on spacewalk. ## How to use : From c8b0f0e36288bf38bfd77fae78a20f3963196669 Mon Sep 17 00:00:00 2001 From: liberodark Date: Fri, 19 Apr 2019 12:22:40 +0200 Subject: [PATCH 34/35] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4753f82..228eca9 100644 --- a/README.md +++ b/README.md @@ -79,7 +79,7 @@ cp -a nano spacewalk_sync_debian.cron /etc/cron.daily/nano spacewalk_sync_debian ``` nano errata-import-debian.py login = 'MYLOGIN' # Line 46 -passwd = 'MYPASSWORD' # Line 47 +password = 'MYPASSWORD' # Line 47 ``` - Install Files : From 5bcbafe78c9595b3bf676fd8fcb84d15ea367b89 Mon Sep 17 00:00:00 2001 From: liberodark Date: Wed, 24 Apr 2019 09:41:18 +0200 Subject: [PATCH 35/35] Update install.sh --- install.sh | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/install.sh b/install.sh index b2f0278..c45d501 100644 --- a/install.sh +++ b/install.sh @@ -49,11 +49,8 @@ echo "Install Script for Spacewalk" if [ $? != 1 ]; then if [[ "$distribution" =~ .CentOS || "$distribution" = CentOS ]]; then - echo "What is your user of spacewalk ?" - read user - - echo "What is your password of spacewalk ?" - read password + read -p "What is your user of spacewalk ?" user + read -p "What is your password of spacewalk ?" password yum install -y html2text git mkdir -p /home/errata/spacewalk-scripts/ @@ -64,14 +61,9 @@ echo "Install Script for Spacewalk" sed -i "s@MYPASSWORD@${password}@@g" /home/errata/spacewalk-scripts/errata-import-debian.py elif [[ "$distribution" =~ .Debian || "$distribution" = Debian || "$distribution" =~ .Ubuntu || "$distribution" = Ubuntu ]]; then - echo "What is your ssh user of spacewalk ?" - read ssh_user - - echo "What is your ssh password of spacewalk ?" - read ssh_password - - echo "What is your ip of spacewalk ?" - read ssh_ip + read -p "What is your ssh user of spacewalk ?" ssh_user + read -p "What is your ssh password of spacewalk ?" ssh_password + read -p "What is your ip of spacewalk ?" ssh_ip apt-get update apt-get install -y html2text git sshpass @@ -84,4 +76,4 @@ echo "Install Script for Spacewalk" sed -i "s@ssh_ip@${ssh_ip}@@g" /etc/cron.daily/spacewalk_errata_debian.cron fi -fi \ No newline at end of file +fi