Add salt client

requirements.txt

@@ -15,6 +15,7 @@ python-novaclient>=6.0.0,!=7.0.0  # Apache-2.0
 python-saharaclient>=1.1.0  # Apache-2.0
 PyYAML>=3.12  # MIT
 requests>=2.10.0,!=2.12.2 # Apache-2.0
+salt
 six>=1.10.0  # MIT
 tox>=2.5.0  # MIT
 urllib3>=1.15.1

stacklight_tests/clients/fixtures.py

@@ -5,6 +5,7 @@
 from stacklight_tests.clients.openstack import client_manager
 from stacklight_tests.clients import influxdb_api
 from stacklight_tests.clients import nagios_api
+from stacklight_tests.clients import salt_api
 from stacklight_tests.clients.prometheus import alertmanager_client
 from stacklight_tests.clients.prometheus import prometheus_client
 
@@ -112,3 +113,8 @@ def os_clients(keystone_config):
 @pytest.fixture(scope="session")
 def os_actions(os_clients):
     return client_manager.OSCliActions(os_clients)
+
+
+@pytest.fixture(scope="session")
+def salt_actions():
+    return salt_api.SaltApi()

stacklight_tests/clients/salt_api.py

@@ -0,0 +1,29 @@
+import salt.client
+
+
+class SaltApi(object):
+    def __init__(self):
+        self.salt_api = salt.client.LocalClient()
+
+    def run_cmd(self, tgt, command, tgt_type='compound'):
+        return self.salt_api.cmd(
+            tgt, "cmd.run", [command], tgt_type=tgt_type).values()
+
+    def ping(self, tgt='*', tgt_type='compound'):
+        nodes = self.salt_api.cmd(tgt, "test.ping", tgt_type=tgt_type).keys()
+        return nodes
+
+    def get_pillar(self, tgt, pillar, tgt_type='compound'):
+        result = self.salt_api.cmd(
+            tgt, 'pillar.get', [pillar], tgt_type=tgt_type)
+        return result
+
+    def get_pillar_item(self, tgt, pillar_item, tgt_type='compound'):
+        result = self.salt_api.cmd(
+            tgt, 'pillar.get', [pillar_item], tgt_type=tgt_type).values()
+        return [i for i in result if i]
+
+    def get_grains(self, tgt, grains, tgt_type='compound'):
+        result = self.salt_api.cmd(
+            tgt, 'grains.get', [grains], tgt_type=tgt_type)
+        return result

stacklight_tests/config/localclient_config.py

@@ -0,0 +1,204 @@
+import salt.client as client
+import subprocess
+import socket
+
+import yaml
+from pprint import pprint
+
+import settings
+import utils
+from io import StringIO
+
+class LOG(object):
+    @staticmethod
+    def info(msg):
+        pprint(msg)
+
+
+class NoApplication(Exception):
+    pass
+
+
+class MKConfig(object):
+    def __init__(self, cluster_name=None):
+
+        if cluster_name is None:
+            cluster_name = socket.getfqdn().split('.', 1)[-1]
+            LOG.info("No domain/cluster_name passed, use generated: {}"
+                     .format(cluster_name))
+        salt = client.LocalClient()
+        inv = salt.cmd('salt:master', 'cmd.run', ['reclass --inventory'],
+                       expr_form='pillar').values()
+        file_like_io = StringIO(''.join(inv).decode("utf-8"))
+        inventory = yaml.load(file_like_io)
+        LOG.info("Try to load nodes for domain {}".format(cluster_name))
+        self.nodes = {k: v for k, v in inventory["nodes"].items()
+                      if cluster_name in k}
+        LOG.info("Load nodes: {}".format(self.nodes.keys()))
+
+    def get_application_node(self, applications):
+        if isinstance(applications, basestring):
+            applications = [applications]
+        for fqdn, node in self.nodes.items():
+            # LOG.info("Check application {} for node {}".
+            #          format(application, fqdn))
+            if all(app in node["applications"] for app in applications):
+                LOG.info("Found applications {} for node {}".
+                         format(applications, fqdn))
+                return node
+        raise NoApplication()
+
+    def generate_nodes_config(self):
+        nodes_config = []
+
+        def parse_roles_from_classes(node):
+            roles_mapping = {
+                "openstack.control": "controller",
+                "openstack.compute": "compute",
+                "stacklight.server": "monitoring",
+                "galera.master": "galera.master",
+                "galera.slave": "galera.slave",
+                "kubernetes.control": "k8s_controller",
+                "kubernetes.compute": "k8s_compute",
+                "grafana.client": "grafana_client",
+                "kibana.server": "elasticsearch_server",
+                "prometheus.server": "prometheus_server",
+            }
+            cls_based_roles = [
+                role for role_name, role in roles_mapping.items()
+                if any(role_name in c for c in node["classes"])
+            ]
+            # Avoid simultaneous existence of k8s_controller
+            # and k8s_compute roles
+            if ("k8s_compute" in cls_based_roles and
+                    "k8s_controller" in cls_based_roles):
+                cls_based_roles.remove("k8s_compute")
+            return cls_based_roles
+
+        for current_node in self.nodes.values():
+            node_params = current_node["parameters"]
+            roles = current_node["applications"]
+            roles.extend(parse_roles_from_classes(current_node))
+            roles.extend(current_node["classes"])
+            roles.sort()
+            nodes_config.append({
+                "address": node_params['_param']['single_address'],
+                "hostname": node_params['linux']['network']['fqdn'],
+                "roles": roles,
+            })
+
+        return nodes_config
+
+    def generate_influxdb_config(self):
+        _param = self.get_application_node("influxdb")['parameters']['_param']
+        return {
+            "influxdb_vip":
+                _param.get('grafana_influxdb_host') or
+                _param['stacklight_monitor_address'],
+            "influxdb_port":
+                _param['influxdb_port'],
+            "influxdb_username":
+                _param.get('influxdb_user') or "root",
+            "influxdb_password":
+                _param.get('influxdb_password') or
+                _param["influxdb_admin_password"],
+            "influxdb_db_name":
+                _param.get('influxdb_database') or "lma",
+        }
+
+    def generate_elasticsearch_config(self):
+        _param = (
+            self.get_application_node("elasticsearch_server")['parameters'])
+        _kibana_param = _param['kibana']['server']
+        return {
+            "elasticsearch_vip": _param['_param']['kibana_elasticsearch_host'],
+            "elasticsearch_port": _kibana_param['database']['port'],
+            "kibana_port": _kibana_param['bind']['port'],
+        }
+
+    def generate_grafana_config(self):
+        _param = self.get_application_node("grafana_client")['parameters']
+        _client_param = _param['grafana']['client']
+        return {
+            "grafana_vip": _client_param['server']['host'],
+            "grafana_port": _client_param['server']['port'],
+            "grafana_username": _client_param['server']['user'],
+            "grafana_password": _client_param['server']['password'],
+            "grafana_default_datasource": _client_param['datasource'].keys()[0]
+        }
+
+    def generate_nagios_config(self):
+        _param = self.get_application_node("nagios")['parameters']['_param']
+        return {
+            "nagios_vip": _param['nagios_host'],
+            "nagios_port": 80,
+            "nagios_tls": False,
+            "nagios_username": _param['nagios_username'],
+            "nagios_password": _param['nagios_password'],
+        }
+
+    def generate_keystone_config(self):
+        _param = (
+            self.get_application_node("keystone")['parameters']['keystone'])
+        return {
+            "admin_name": _param['server']['admin_name'],
+            "admin_password": _param['server']['admin_password'],
+            "admin_tenant": _param['server']['admin_tenant'],
+            "private_address": _param['server']['bind']['private_address'],
+            "public_address": _param['server']['bind']['public_address'],
+        }
+
+    def generate_mysql_config(self):
+        _param = self.get_application_node("galera")['parameters']['_param']
+        return {
+            "mysql_user": _param['mysql_admin_user'],
+            "mysql_password": _param['mysql_admin_password']
+        }
+
+    def generate_prometheus_config(self):
+        def get_port(input_line):
+            return input_line["ports"][0].split(":")[0]
+        _param = self.get_application_node(
+            ["prometheus_server", "service.docker.client"])['parameters']
+        expose_params = (
+            _param["docker"]["client"]["stack"]["monitoring"]["service"])
+
+        return {
+            "use_prometheus_query_alert": True,
+            "prometheus_vip": _param["_param"]["prometheus_control_address"],
+            "prometheus_server_port":
+                get_port(expose_params["server"]),
+            "prometheus_alertmanager":
+                get_port(expose_params["alertmanager"]),
+            "prometheus_pushgateway":
+                get_port(expose_params["pushgateway"]),
+        }
+
+    def main(self):
+        config = {
+            "env": {"type": "mk"},
+        }
+        for application in settings.CONFIGURE_APPS:
+            try:
+                method = getattr(self, "generate_{}_config".
+                                 format(application))
+                config.update({
+                    application: method()
+                })
+                LOG.info("INFO: {} configured".format(application))
+            except NoApplication:
+                LOG.info("INFO: No {} installed, skip".format(application))
+
+        config_filename = utils.get_fixture("config.yaml",
+                                            check_existence=False)
+        LOG.info("INFO: Saving config to {}".format(config_filename))
+        with open(config_filename, "w") as f:
+            yaml.safe_dump(config, f, default_flow_style=False)
+
+
+def main():
+    MKConfig(cluster_name=settings.ENV_CLUSTER_NAME).main()
+
+
+if __name__ == '__main__':
+    main()

stacklight_tests/objects.py

@@ -73,23 +73,16 @@ def get_random_compute(self):
 
 
 class Host(object):
-    def __init__(self, address, roles=None, *args, **kwargs):
-        self.os = general_client.GeneralActionsClient(
-            address=address,
-            username=kwargs.get("username", "root"),
-            password=kwargs.get("password"),
-            private_key=kwargs.get("private_key"))
-
-        self.address = address
+    def __init__(self, roles=None, *args, **kwargs):
+        self.hostname = kwargs.get("hostname")
+        self.address = kwargs.get("address")
         self.roles = roles or []
-        self.exec_command = self.os.exec_command
-        self.check_call = self.os.check_call
         self.fqdn = kwargs.get("hostname") or self.long_hostname
 
     @property
-    def hostname(self):
-        return self.os.short_hostname
+    def short_hostname(self):
+        return self.hostname.split('.')[0]
 
     @property
     def long_hostname(self):
-        return self.os.long_hostname
+        return self.hostname

stacklight_tests/tests/prometheus/test_smoke.py

@@ -1,3 +1,4 @@
+import pytest
 import socket
 
 
@@ -17,6 +18,20 @@ def test_prometheus_container_up(node):
     def test_prometheus_datasource(self, prometheus_api):
         assert prometheus_api.get_all_measurements()
 
+    def test_prometheus_relay(self, salt_actions):
+        hosts = salt_actions.ping("I@prometheus:relay")
+        if not hosts:
+            pytest.skip("Prometheus relay is not installed in the cluster")
+        url = salt_actions.get_pillar_item(
+            '*', "_param:grafana_prometheus_address")[0]
+        port = salt_actions.get_pillar_item(
+            hosts[0], "prometheus:relay:bind:port")[0]
+        output = salt_actions.run_cmd(
+            hosts[0],
+            "curl -s {}:{}/metrics | awk '/^prometheus/{{print $1}}'".format(
+                url, port))
+        assert output
+
 
 class TestAlertmanagerSmoke(object):
     def test_alertmanager_endpoint_availability(self, prometheus_config):