diff --git a/multivault/base/__init__.py b/multivault/base/__init__.py index ad561e7..efde4db 100644 --- a/multivault/base/__init__.py +++ b/multivault/base/__init__.py @@ -1,4 +1,4 @@ #!/usr/bin/env python3 ''' - The base module of ansible_multivault + The base module of multivault ''' diff --git a/multivault/integrities/__init__.py b/multivault/integrities/__init__.py new file mode 100644 index 0000000..82cd94e --- /dev/null +++ b/multivault/integrities/__init__.py @@ -0,0 +1,4 @@ +#!/usr/bin/env python3 +''' + The integrities module of multivault +''' diff --git a/multivault/integrities/check.py b/multivault/integrities/check.py new file mode 100644 index 0000000..40b73d4 --- /dev/null +++ b/multivault/integrities/check.py @@ -0,0 +1,233 @@ +import configparser +import yaml +import os +import copy +import json +from multivault.utilities import util_check +from multivault.utilities import util_crypt +from multivault.utilities import util_ldap +from multivault.base.config import config +from multivault.base import crypter +from pprint import pprint +try: + from ansible.parsing.dataloader import DataLoader + from ansible.inventory.manager import InventoryManager + import ansible.playbook.play as play +except ImportError: + print("The integrity module relies on ansible!") + print("\tpip3 install ansible") + exit(1) + +INVENTORY = None + + +class bcolors: + HEADER = '\033[95m' + OKBLUE = '\033[94m' + OKGREEN = '\033[92m' + WARNING = '\033[93m' + FAIL = '\033[91m' + ENDC = '\033[0m' + BOLD = '\033[1m' + UNDERLINE = '\033[4m' + + +def get_all_users_and_subkeys(): + results = {} + users = util_ldap.get('users', data='all') + if users: + authorized = crypter._map_sudoers_to_fingerprints(users) + else: + authorized = {} + for user, key in authorized: + if not isinstance(key, str): + results[user] = {} + results[user]['key'] = key + results[user]['subkeys'] = [] + for subkey in key.subkeys: + results[user]['subkeys'].append(subkey.keyid) + return results + + +def init(workdir='/home/cellebyte/git/selfnet/playbooks'): + workdir = os.path.join(workdir) + ansible_config = configparser.ConfigParser() + + ansible_cfg = os.path.join(workdir, 'ansible.cfg') + DEFAULT_INVENTORY = None + ROLES_PATH = None + USER_SUBKEYS = get_all_users_and_subkeys() + # USER_SUBKEYS = None + ansible_config.read(ansible_cfg) + for section in ansible_config: + for key_pair in ansible_config[section]: + if key_pair == 'inventory': + DEFAULT_INVENTORY = ansible_config[section][key_pair] + elif key_pair == 'roles_path': + ROLES_PATH = ansible_config[section][key_pair] + else: + pass + if not DEFAULT_INVENTORY: + DEFAULT_INVENTORY = 'inventory.ini' + if not ROLES_PATH: + ROLES_PATH = './roles' + + inventory_file = os.path.join(workdir, DEFAULT_INVENTORY) + roles_path = os.path.join(workdir, ROLES_PATH) + INVENTORY = InventoryManager(loader=DataLoader(), + sources=[inventory_file]) + return workdir, INVENTORY, roles_path, USER_SUBKEYS + + +def checkout_information(MAPPING, DEPENDENCY_TREE, workdir=None, roles_path=None): + results = [] + for role, hosts in MAPPING.items(): + gpg_path = os.path.join(workdir, roles_path, role, 'gpg') + if os.path.exists(gpg_path): + results.append(check_with_structure(role, hosts, gpg_path)) + roles_with_gpg = [result['role'] for result in results] + for result in results: + role = result['role'] + hosts = result['hosts'] + dependency_roles = DEPENDENCY_TREE[role] + for dependency_role in dependency_roles: + if dependency_role in roles_with_gpg: + gpg_role = [ + result for result in results if result['role'] == dependency_role][0] + pprint(gpg_role) + for _, information in gpg_role['files'].items(): + for host in hosts: + if host not in information['hosts'] and hosts == information['hosts']: + information['hosts'].append(host) + pprint(role) + print('---------------------------') + pprint(gpg_role) + return results + + +def check_with_structure(role, hosts, gpg_path): + results = {} + results['role'] = role + results['hosts'] = hosts + results['files'] = {} + for path, _, files in os.walk(gpg_path): + for file in files: + file_path = os.path.join(path, file) + if file_path.endswith('.gpg'): + minified_path = util_check.remove_string( + gpg_path+os.sep, file_path) + splitted_path = minified_path.split(os.sep) + to_be_encrypted_for = [] + results['files'][file_path] = {} + if len(splitted_path) > 1: + for host in splitted_path: + # host is an encrypted file and not a host + if host.endswith('.gpg') and os.path.isfile(file_path): + results['files'][file_path]['hosts'] = to_be_encrypted_for + elif util_check.is_valid_hostname(host) and host in util_check.match(['all'], INVENTORY): + to_be_encrypted_for.append(host) + else: + results['files'][file_path]['hosts'] = hosts + break + else: + results['files'][file_path]['hosts'] = hosts + return results + + +def get_encrypters_from_file(informations): + for information in informations: + for path, _ in information['files'].items(): + print("Analyzing File: {}".format(path)) + information['files'][path]['encrypters'] = list( + util_check.read_message(path)) + return informations + + +def get_encrypters_from_ldap(informations, USER_SUBKEYS): + for information in informations: + for path in information['files'].keys(): + hosts = [host.split('.')[0] + for host in information['files'][path]['hosts']] + authorized = util_ldap.get_authorized(hosts) + if not authorized: + authorized = {} + information['files'][path]['sudoers'] = [] + for user, _ in authorized: + for uid, keys in USER_SUBKEYS.items(): + if uid == user: + information['files'][path]['sudoers'].append( + {uid: keys}) + return informations + + +def crossover(informations, USER_SUBKEYS): + for information in informations: + print(bcolors.OKBLUE + + 'INFO\t:::\tCheck Encryption for role >> {} <<'.format(information['role'])) + for path, file_information in information['files'].items(): + keys = list(copy.deepcopy(file_information['encrypters'])) + print(bcolors.OKBLUE + + '\tINFO\t:::\tCheck for hosts >> {} <<'.format(file_information['hosts'])) + filename = '.../' + str(path.split(os.sep)[-1]) + for sudoer in file_information['sudoers']: + for user, key_information in sudoer.items(): + matched = False + for sub_key in key_information['subkeys']: + if sub_key in keys: + matched = True + if not matched: + print( + bcolors.WARNING + '\tWARNING\t:::\t{} not encrypted for user >> {} <<'.format(filename, user)) + else: + print( + bcolors.OKGREEN + '\tOK\t:::\t{} correctly encrypted for user >> {} <<'.format(filename, user)) + for sub_key in key_information['subkeys']: + try: + keys.remove(sub_key) + except ValueError: + pass + if len(keys) > 0: + for key in keys: + user = get_user(key, USER_SUBKEYS) + if not user: + print( + bcolors.WARNING + '\tWARNING\t:::\t{} encrypted for unknown key >> {} <<'.format(filename, key)) + else: + print( + bcolors.FAIL + '\tERROR\t:::\t{} wrong encrypted for user >> {} <<(please fix!)'.format(filename, user)) + + +def get_user(key, USER_SUBKEYS): + for user, keys in USER_SUBKEYS.items(): + if key in keys: + return user + return None + + +def check(workdir=None, playbook='all.yml'): + global INVENTORY + if not workdir: + workdir, INVENTORY, roles_path, USER_SUBKEYS = init() + else: + return "Not Implemented" + # pprint(USER_SUBKEYS) + playbook_file = os.path.join(workdir, playbook) + mapping = util_check.parse_play(playbook_file, INVENTORY=INVENTORY) + MAPPING = util_check.look_for_dependencies(mapping, workdir, roles_path) + DEPENDENCY_TREE = util_check.build_dependency_tree(workdir, util_check.get_roles( + workdir, roles_path=roles_path), roles_path=roles_path) + # pprint(DEPENDENCY_TREE) + FILE_HOSTS = checkout_information( + MAPPING, DEPENDENCY_TREE, workdir=workdir, roles_path=roles_path) + IS_ENCRYPTED_FOR = get_encrypters_from_file(FILE_HOSTS) + SHOULD_BE_ENCRYPTED_FOR = get_encrypters_from_ldap( + IS_ENCRYPTED_FOR, USER_SUBKEYS) + # pprint(SHOULD_BE_ENCRYPTED_FOR) + crossover(SHOULD_BE_ENCRYPTED_FOR, USER_SUBKEYS) + + +get_all_users_and_subkeys + +if __name__ == '__main__': + + check() diff --git a/multivault/utilities/util_check.py b/multivault/utilities/util_check.py new file mode 100644 index 0000000..15b00ad --- /dev/null +++ b/multivault/utilities/util_check.py @@ -0,0 +1,145 @@ +import re +import yaml +import os +import sys +from subprocess import check_output, CalledProcessError +from copy import deepcopy +from multivault.utilities import util_crypt +from multivault.base.config import config + +IGNORED = ['ubuntu', 'debian', 'ubuntu_host', 'debian_host'] +encrypter = re.compile(r"^:pubkey.*keyid (?P.*?)$", re.MULTILINE) +allowed = re.compile(r"(?!-)[A-Z\d-]{1,63}(? 255: + return False + if hostname[-1] == ".": + # strip exactly one dot from the right, if present + hostname = hostname[:-1] + + return all(allowed.match(x) for x in hostname.split(".")) + + +def parse_play(play_file, INVENTORY=None): + MAPPING = {} + with open(play_file, mode="r") as playbook: + playbook = yaml.load(playbook) + for task in playbook: + if 'roles' in task.keys(): + if isinstance(task['hosts'], list): + hosts = task['hosts'] + elif isinstance(task['hosts'], str): + hosts = task['hosts'].lower().split(',') + else: + print('Unhandleable Type!') + + if isinstance(task['roles'], list): + roles = task['roles'] + elif isinstance(task['roles'], str): + roles = task['roles'].lower().split(',') + else: + print('Unhandleable Type!') + hosts = [host.strip() for host in hosts] + roles = [role.strip() for role in roles] + div = set(hosts)-set(IGNORED) + div = list(div) + hosts = match(div, INVENTORY=INVENTORY) + MAPPING = merge_hosts_to_roles(roles, MAPPING, hosts) + return {k: v for k, v in MAPPING.items() if v} + + +def match(groups, INVENTORY=None): + hosts = [] + for group in groups: + if group.startswith('!'): + pass + else: + try: + hosts.append(INVENTORY.get_groups_dict()[group]) + except Exception: + if len(group.split('.')) > 1: + hosts.append([group]) + return util_crypt.flatten(hosts) + + +def merge_hosts_to_roles(roles, MAPPING, hosts): + for role in roles: + if role in MAPPING.keys(): + for host in hosts: + if host not in MAPPING[role]: + MAPPING[role].append(host) + else: + MAPPING[role] = [] + for host in hosts: + if host not in MAPPING[role]: + MAPPING[role].append(host) + return MAPPING + + +def read_message(file_to_read): + try: + with open(os.devnull, 'w') as devnull: + output = check_output(['/usr/bin/env', 'gpg', '--homedir', + config.gpg['key_home'], '--list-packets', file_to_read], stderr=devnull) + output = output.decode(sys.stdout.encoding) + except CalledProcessError as e: + if e.returncode != 2: + print(e) + output = e.output.decode(sys.stdout.encoding) + encrypters = encrypter.findall(output) + return encrypters + + +def look_for_dependencies(mapping, workdir, roles_path="./roles"): + MAPPING = deepcopy(mapping) + for role, hosts in mapping.items(): + meta_path = os.path.join(workdir, roles_path, role, 'meta', 'main.yml') + roles = get_meta_info(meta_path) + MAPPING = merge_hosts_to_roles(roles, MAPPING, hosts) + if mapping == MAPPING: + return MAPPING + else: + return look_for_dependencies(MAPPING, workdir, roles_path=roles_path) + + +def get_roles(workdir, roles_path="./roles"): + return [role for role in os.listdir(os.path.join(workdir, roles_path)) if role] + + +def remove_string(gpg_path, path): + temp = 0 + for i in range(0, len(gpg_path)): + if gpg_path[i] == path[i]: + temp = i + + return path[temp+1:] + + +def get_meta_info(meta_path): + if not os.path.exists(meta_path): + return [] + roles = [] + with open(meta_path, 'r') as meta: + meta_info = yaml.load(meta) + if not meta_info: + return [] + if 'dependencies' in meta_info.keys(): + if not meta_info['dependencies']: + return [] + + for dependency in meta_info['dependencies']: + if isinstance(dependency, dict): + roles.append(dependency.get('role', [])) + elif isinstance(dependency, str): + roles.append(dependency) + return list(set(roles)) + + +def build_dependency_tree(workdir, roles, roles_path="./roles"): + DEPENDENCY_TREE = {} + for role in roles: + meta_path = os.path.join(workdir, roles_path, role, 'meta', 'main.yml') + DEPENDENCY_TREE[role] = get_meta_info(meta_path) + return DEPENDENCY_TREE diff --git a/multivault/utilities/util_crypt.py b/multivault/utilities/util_crypt.py index fe5b379..b19dff5 100644 --- a/multivault/utilities/util_crypt.py +++ b/multivault/utilities/util_crypt.py @@ -16,7 +16,6 @@ allow-secret-key-import trust-model tofu+pgp tofu-default-policy unknown -enable-large-rsa enable-dsa2 cert-digest-algo SHA512 default-preference-list {0} {1} {2} {3} {4} {5} @@ -27,6 +26,7 @@ agentconf = """# gpg-agent.conf settings for key generation: default-cache-ttl 300 +max-cache-ttl 500 """ import os @@ -67,6 +67,7 @@ def password_generator(size=20, chars=string.ascii_letters + string.digits): # Use secrets instead of random, cause random is very predictable return ''.join(secrets.choice(chars) for _ in range(size)) + def create_gnupghome(path): ''' creates a gnupg home with the configurations above @@ -82,4 +83,4 @@ def create_gnupghome(path): os.chmod("{0}/{1}".format(path, "gpg.conf"), 0o600) with open("{0}/{1}".format(path, "gpg-agent.conf"), "w") as conf: conf.write(agentconf) - os.chmod("{0}/{1}".format(path, "gpg-agent.conf"), 0o600) \ No newline at end of file + os.chmod("{0}/{1}".format(path, "gpg-agent.conf"), 0o600) diff --git a/multivault/utilities/util_filter.py b/multivault/utilities/util_filter.py index a257ff8..29f2f9d 100644 --- a/multivault/utilities/util_filter.py +++ b/multivault/utilities/util_filter.py @@ -41,6 +41,11 @@ def create_filter_users(key, values): ''' Creates LDAP readable filter for all uids specified on cli ''' + if isinstance(values, str): + if values == 'all': + return "({}=*)".format(key) + else: + return "({}={})".format(key, values) filter = "(|" for value in values: filter = filter + "({}={})".format(key, value) diff --git a/multivault/utilities/util_ldap.py b/multivault/utilities/util_ldap.py index e0d251b..7a28bae 100644 --- a/multivault/utilities/util_ldap.py +++ b/multivault/utilities/util_ldap.py @@ -130,11 +130,13 @@ def get_authorized(hostnames): ''' sudoers = get('hostnames', data=hostnames) masters = get('none') - if not sudoers or not masters: + if not masters and not sudoers: print("Sudoers:", sudoers) print("Masters:", masters) print("An error ocurred by getting the required ldap information!") - return None + return {} + if not sudoers: + return masters in_masters_but_not_in_sudoers = set(masters) - set(sudoers) authorized_list = list(sudoers) + list(in_masters_but_not_in_sudoers) return authorized_list diff --git a/multivault/utilities/util_ssh.py b/multivault/utilities/util_ssh.py index fcb1375..13afd48 100644 --- a/multivault/utilities/util_ssh.py +++ b/multivault/utilities/util_ssh.py @@ -39,6 +39,7 @@ import paramiko from multivault.base.config import config + class ForwardServer (SocketServer.ThreadingTCPServer): daemon_threads = True allow_reuse_address = True @@ -126,8 +127,10 @@ class SubHander (Handler): chain_host = remote chain_port = remote_port ssh_transport = client.get_transport() - forwarder = ForwardServer(('127.0.0.1', config.ldap['connection']['forward_port']), SubHander) + forwarder = ForwardServer( + ('127.0.0.1', config.ldap['connection']['forward_port']), SubHander) threading.Thread(target=forwarder.serve_forever).start() yield + forwarder.server_close() forwarder.shutdown() client.close()