Add salt connection

stacklight_tests/mcp_config.py

@@ -0,0 +1,199 @@
+import subprocess
+import socket
+
+import yaml
+from pprint import pprint
+
+import settings
+import utils
+from io import StringIO
+
+class LOG(object):
+    @staticmethod
+    def info(msg):
+        pprint(msg)
+
+
+class NoApplication(Exception):
+    pass
+
+
+class MKConfig(object):
+    def __init__(self, cluster_name=None):
+
+        if cluster_name is None:
+            cluster_name = socket.getfqdn().split('.', 1)[-1]
+            LOG.info("No domain/cluster_name passed, use generated: {}"
+                     .format(cluster_name))
+        salt = utils.init_salt_client()
+        inv = salt.cmd('salt:master', 'cmd.run', ['reclass --inventory'], expr_form='pillar').values()
+        file_like_io = StringIO(''.join(inv).decode("utf-8"))
+        inventory = yaml.load(file_like_io)
+        LOG.info("Try to load nodes for domain {}".format(cluster_name))
+        self.nodes = {k: v for k, v in inventory["nodes"].items()
+                      if cluster_name in k}
+        LOG.info("Load nodes: {}".format(self.nodes.keys()))
+
+    def get_application_node(self, application):
+        for fqdn, node in self.nodes.items():
+            # LOG.info("Check application {} for node {}".
+            #          format(application, fqdn))
+            if application in node["applications"]:
+                LOG.info("Found application {} for node {}".
+                         format(application, fqdn))
+                return node
+        raise NoApplication()
+
+    def generate_nodes_config(self):
+        nodes_config = []
+
+        def parse_roles_from_classes(node):
+            roles_mapping = {
+                "openstack.control": "controller",
+                "openstack.compute": "compute",
+                "stacklight.server": "monitoring",
+                "galera.master": "galera.master",
+                "galera.slave": "galera.slave",
+                "kubernetes.control": "k8s_controller",
+                "kubernetes.compute": "k8s_compute",
+                "grafana.client": "grafana_client",
+                "kibana.server": "elasticsearch_server",
+                "prometheus.server": "prometheus_server",
+            }
+            cls_based_roles = [
+                role for role_name, role in roles_mapping.items()
+                if any(role_name in c for c in node["classes"])
+            ]
+            # Avoid simultaneous existence of k8s_controller
+            # and k8s_compute roles
+            if ("k8s_compute" in cls_based_roles and
+                    "k8s_controller" in cls_based_roles):
+                cls_based_roles.remove("k8s_compute")
+            return cls_based_roles
+
+        for current_node in self.nodes.values():
+            node_params = current_node["parameters"]
+            roles = current_node["applications"]
+            roles.extend(parse_roles_from_classes(current_node))
+            roles.extend(current_node["classes"])
+            roles.sort()
+            nodes_config.append({
+                "address": node_params['_param']['single_address'],
+                "hostname": node_params['linux']['network']['fqdn'],
+                "roles": roles,
+            })
+
+        return nodes_config
+
+    def generate_influxdb_config(self):
+        _param = self.get_application_node("influxdb")['parameters']['_param']
+        return {
+            "influxdb_vip":
+                _param.get('grafana_influxdb_host') or
+                _param['stacklight_monitor_address'],
+            "influxdb_port":
+                _param['influxdb_port'],
+            "influxdb_username":
+                _param.get('influxdb_user') or "root",
+            "influxdb_password":
+                _param.get('influxdb_password') or
+                _param["influxdb_admin_password"],
+            "influxdb_db_name":
+                _param.get('influxdb_database') or "lma",
+        }
+
+    def generate_elasticsearch_config(self):
+        _param = (
+            self.get_application_node("elasticsearch_server")['parameters'])
+        _kibana_param = _param['kibana']['server']
+        return {
+            "elasticsearch_vip": _param['_param']['kibana_elasticsearch_host'],
+            "elasticsearch_port": _kibana_param['database']['port'],
+            "kibana_port": _kibana_param['bind']['port'],
+        }
+
+    def generate_grafana_config(self):
+        _param = self.get_application_node("grafana_client")['parameters']
+        _client_param = _param['grafana']['client']
+        return {
+            "grafana_vip": _client_param['server']['host'],
+            "grafana_port": _client_param['server']['port'],
+            "grafana_username": _client_param['server']['user'],
+            "grafana_password": _client_param['server']['password'],
+            "grafana_default_datasource": _client_param['datasource'].keys()[0]
+        }
+
+    def generate_nagios_config(self):
+        _param = self.get_application_node("nagios")['parameters']['_param']
+        return {
+            "nagios_vip": _param['nagios_host'],
+            "nagios_port": 80,
+            "nagios_tls": False,
+            "nagios_username": _param['nagios_username'],
+            "nagios_password": _param['nagios_password'],
+        }
+
+    def generate_keystone_config(self):
+        _param = (
+            self.get_application_node("keystone")['parameters']['keystone'])
+        return {
+            "admin_name": _param['server']['admin_name'],
+            "admin_password": _param['server']['admin_password'],
+            "admin_tenant": _param['server']['admin_tenant'],
+            "private_address": _param['server']['bind']['private_address'],
+            "public_address": _param['server']['bind']['public_address'],
+        }
+
+    def generate_mysql_config(self):
+        _param = self.get_application_node("galera")['parameters']['_param']
+        return {
+            "mysql_user": _param['mysql_admin_user'],
+            "mysql_password": _param['mysql_admin_password']
+        }
+
+    def generate_prometheus_config(self):
+        def get_port(input_line):
+            return input_line["ports"][0].split(":")[0]
+        _param = self.get_application_node("rundeck")['parameters']
+        expose_params = (
+            _param["docker"]["client"]["stack"]["monitoring"]["service"])
+
+        return {
+            "use_prometheus_query_alert": True,
+            "prometheus_vip": _param["_param"]["prometheus_control_address"],
+            "prometheus_server_port":
+                get_port(expose_params["server"]),
+            "prometheus_alertmanager":
+                get_port(expose_params["alertmanager"]),
+            "prometheus_pushgateway":
+                get_port(expose_params["pushgateway"]),
+        }
+
+    def main(self):
+        config = {
+            "env": {"type": "mk"},
+        }
+        for application in settings.CONFIGURE_APPS:
+            try:
+                method = getattr(self, "generate_{}_config".
+                                 format(application))
+                config.update({
+                    application: method()
+                })
+                LOG.info("INFO: {} configured".format(application))
+            except NoApplication:
+                LOG.info("INFO: No {} installed, skip".format(application))
+
+        config_filename = utils.get_fixture("config.yaml",
+                                            check_existence=False)
+        LOG.info("INFO: Saving config to {}".format(config_filename))
+        with open(config_filename, "w") as f:
+            yaml.safe_dump(config, f, default_flow_style=False)
+
+
+def main():
+    MKConfig(cluster_name=settings.ENV_CLUSTER_NAME).main()
+
+
+if __name__ == '__main__':
+    main()

stacklight_tests/objects.py

@@ -1,4 +1,6 @@
 import random
+import re
+import utils
 
 import stacklight_tests.custom_exceptions as exceptions
 from stacklight_tests.clients.system import general_client
@@ -74,22 +76,18 @@ def get_random_compute(self):
 
 class Host(object):
     def __init__(self, address, roles=None, *args, **kwargs):
-        self.os = general_client.GeneralActionsClient(
-            address=address,
-            username=kwargs.get("username", "root"),
-            password=kwargs.get("password"),
-            private_key=kwargs.get("private_key"))
-
         self.address = address
         self.roles = roles or []
-        self.exec_command = self.os.exec_command
-        self.check_call = self.os.check_call
-        self.fqdn = kwargs.get("hostname") or self.long_hostname
+        self.fqdn = kwargs.get("hostname")
 
     @property
     def hostname(self):
-        return self.os.short_hostname
+        return self.fqdn.split('.')[0]
 
     @property
     def long_hostname(self):
-        return self.os.long_hostname
+        return self.fqdn
+
+    def execute_salt(self, cmd):
+        salt = utils.init_salt_client()
+        return salt.cmd(self.fqdn, 'cmd.run', [cmd], expr_form='pcre').values()

stacklight_tests/tests/prometheus/test_metrics.py

@@ -180,11 +180,11 @@ def test_mysql_metrics(self, cluster):
             expected_metrics.append("mysql_handler_{}".format(handler))
 
         for host in mysql_hosts:
-            got_metrics = host.os.exec_command(
+            got_metrics = host.execute_salt(
                 "curl -s localhost:9126/metrics | awk '/^mysql/{print $1}'")
             hostname = host.hostname
             for metric in expected_metrics:
                 metric = metric + '{host="' + hostname + '"}'
                 err_msg = ("Metric {} not found in received list of mysql "
                            "metrics on {} node".format(metric, hostname))
-                assert metric in got_metrics, err_msg
+                assert metric in got_metrics[0].split("\n"), err_msg

stacklight_tests/utils.py

@@ -8,7 +8,35 @@
 from requests.packages.urllib3 import poolmanager
 import yaml
 
-from stacklight_tests import custom_exceptions as exceptions
+import custom_exceptions as exceptions
+
+
+class salt_remote:
+    def cmd(self, tgt, fun, param=None, expr_form=None, tgt_type=None):
+        headers = {'Accept': 'application/json'}
+        login_payload = {'username': os.environ['SALT_USERNAME'],
+                         'password': os.environ['SALT_PASSWORD'], 'eauth': 'pam'}
+        accept_key_payload = {'fun': fun, 'tgt': tgt, 'client': 'local',
+                              'expr_form': expr_form, 'tgt_type': tgt_type,
+                              'timeout': 1}
+        if param:
+            accept_key_payload['arg'] = param
+
+        login_request = requests.post(os.path.join(os.environ['SALT_URL'],
+                                                   'login'),
+                                      headers=headers, data=login_payload)
+        if login_request.ok:
+            request = requests.post(os.environ['SALT_URL'], headers=headers,
+                                    data=accept_key_payload,
+                                    cookies=login_request.cookies)
+            return request.json()['return'][0]
+        else:
+            raise EnvironmentError("401 Not authorized.")
+
+
+def init_salt_client():
+    local = salt_remote()
+    return local
 
 
 class TestHTTPAdapter(requests.adapters.HTTPAdapter):