diff --git a/changelog/63114.added.md b/changelog/63114.added.md new file mode 100644 index 000000000000..7b4210badc0a --- /dev/null +++ b/changelog/63114.added.md @@ -0,0 +1 @@ +Control the collection of lvm grains via disabled_grains. diff --git a/changelog/63114.deprecated.md b/changelog/63114.deprecated.md new file mode 100644 index 000000000000..e26508ee535b --- /dev/null +++ b/changelog/63114.deprecated.md @@ -0,0 +1 @@ +The per-grain configuration options ``enable_fqdns_grains``, ``enable_gpu_grains``, and similar ``enable_*_grains`` / ``*_grains`` options are deprecated in favor of the unified ``disabled_grains`` list. They will be removed in Calcium. diff --git a/conf/master b/conf/master index ebe5503cdc16..5924fff39a8d 100644 --- a/conf/master +++ b/conf/master @@ -137,10 +137,13 @@ # Set the directory used to hold unix sockets: #sock_dir: /var/run/salt/master + +# Some grains can be slow to compute. Add grains to this list to disable their computation. +#disabled_grains: +# - gpu # The master can take a while to start up when lspci and/or dmidecode is used -# to populate the grains for the master. Enable if you want to see GPU hardware -# data for your master. -# enable_gpu_grains: False +# to populate the grains for the master. Remove "gpu" from disabled_grains if +# you want to see GPU hardware data for your master. # The master maintains a job cache. While this is a great addition, it can be # a burden on the master for larger deployments (over 5000 minions). diff --git a/conf/minion b/conf/minion index 323299a04feb..8057cb3b901f 100644 --- a/conf/minion +++ b/conf/minion @@ -168,21 +168,23 @@ # Set the directory used to hold unix sockets. #sock_dir: /var/run/salt/minion +# Some grains can be slow to compute. Add grains to this list to disable their computation. +# Important: it only works if the grains module / function is aware of this configuration. +#disabled_grains: +# - lvm +# # In order to calculate the fqdns grain, all the IP addresses from the minion # are processed with underlying calls to `socket.gethostbyaddr` which can take # 5 seconds to be released (after reaching `socket.timeout`) when there is no # fqdn for that IP. These calls to `socket.gethostbyaddr` are processed # asynchronously, however, it still adds 5 seconds every time grains are -# generated if an IP does not resolve. In Windows grains are regenerated each -# time a new process is spawned. Therefore, the default for Windows is `False`. -# On macOS, FQDN resolution can be very slow, therefore the default for macOS is -# `False` as well. All other OSes default to `True` -# enable_fqdns_grains: True - +# generated if an IP does not resolve. The fqdns grain is disabled by default. +# Remove "fqdns" from disabled_grains to enable it. +# # The minion can take a while to start up when lspci and/or dmidecode is used -# to populate the grains for the minion. Set this to False if you do not need -# GPU hardware grains for your minion. -# enable_gpu_grains: True +# to populate the grains for the minion. +# Add "gpu" to `disabled_grains` to disable their computation. + # Set the default outputter used by the salt-call command. The default is # "nested". diff --git a/salt/config/__init__.py b/salt/config/__init__.py index f7d551749ba4..6cbbcf168421 100644 --- a/salt/config/__init__.py +++ b/salt/config/__init__.py @@ -51,6 +51,7 @@ log = logging.getLogger(__name__) _DFLT_REFSPECS = ["+refs/heads/*:refs/remotes/origin/*", "+refs/tags/*:refs/tags/*"] +_DFLT_DISABLED_GRAINS = ["fqdns"] DEFAULT_INTERVAL = 60 DEFAULT_HASH_TYPE = "sha256" @@ -60,27 +61,14 @@ # support in ZeroMQ, we want the default to be something that has a # chance of working. _DFLT_IPC_MODE = "tcp" - _DFLT_FQDNS_GRAINS = False _MASTER_TRIES = -1 # This needs to be SYSTEM in order for salt-master to run as a Service # Otherwise, it will not respond to CLI calls _MASTER_USER = "SYSTEM" -elif salt.utils.platform.is_proxy(): - _DFLT_IPC_MODE = "ipc" - _DFLT_FQDNS_GRAINS = False - _MASTER_TRIES = 1 +else: _MASTER_USER = salt.utils.user.get_user() -elif salt.utils.platform.is_darwin(): - _DFLT_IPC_MODE = "ipc" - # fqdn resolution can be very slow on macOS, see issue #62168 - _DFLT_FQDNS_GRAINS = False _MASTER_TRIES = 1 - _MASTER_USER = salt.utils.user.get_user() -else: _DFLT_IPC_MODE = "ipc" - _DFLT_FQDNS_GRAINS = False - _MASTER_TRIES = 1 - _MASTER_USER = salt.utils.user.get_user() def _gather_buffer_space(): @@ -424,9 +412,9 @@ def _gather_buffer_space(): "test": bool, # Tell the loader to attempt to import *.pyx cython files if cython is available "cython_enable": bool, - # Whether or not to load grains for FQDNs + # Whether or not to load grains for FQDNs (deprecated) "enable_fqdns_grains": bool, - # Whether or not to load grains for the GPU + # Whether or not to load grains for the GPU (deprecated) "enable_gpu_grains": bool, # Tell the loader to attempt to import *.zip archives "enable_zip_modules": bool, @@ -834,13 +822,15 @@ def _gather_buffer_space(): "winrepo_refspecs": list, # Set a hard limit for the amount of memory modules can consume on a minion. "modules_max_memory": int, - # Blacklist specific core grains to be filtered + # Blacklist specific core grains to be filtered (after computation) "grains_blacklist": list, # The number of minutes between the minion refreshing its cache of grains "grains_refresh_every": int, # Enable grains refresh prior to any operation "grains_refresh_pre_exec": bool, - # Use lspci to gather system data for grains on a minion + # Disable grain computation (replaces grain-specific options) + "disabled_grains": list, + # Use lspci to gather system data for grains on a minion (deprecated) "enable_lspci": bool, # The number of seconds for the salt client to wait for additional syndics to # check in with their lists of expected minions before giving up @@ -1148,6 +1138,7 @@ def _gather_buffer_space(): "grains_cache": False, "grains_cache_expiration": 300, "grains_deep_merge": False, + "disabled_grains": _DFLT_DISABLED_GRAINS, "conf_file": os.path.join(salt.syspaths.CONFIG_DIR, "minion"), "sock_dir": os.path.join(salt.syspaths.SOCK_DIR, "minion"), "sock_pool_size": 1, @@ -1311,8 +1302,6 @@ def _gather_buffer_space(): "test": False, "ext_job_cache": "", "cython_enable": False, - "enable_fqdns_grains": _DFLT_FQDNS_GRAINS, - "enable_gpu_grains": True, "enable_zip_modules": False, "state_verbose": True, "state_output": "full", @@ -1758,7 +1747,7 @@ def _gather_buffer_space(): "ssh_list_nodegroups": {}, "ssh_use_home_key": False, "cython_enable": False, - "enable_gpu_grains": False, + "disabled_grains": ["gpu"], # XXX: Remove 'key_logfile' support in 2014.1.0 "key_logfile": os.path.join(salt.syspaths.LOGS_DIR, "key"), "verify_env": True, @@ -4237,9 +4226,44 @@ def apply_minion_config( if "__cachedir" not in opts: opts["__cachedir"] = opts["cachedir"] + # Update disabled_grains from deprecated, grain-specific options + _update_disabled_grains(opts) + return opts +def _update_disabled_grains(opts): + """ + Update disabled grains based on (deprecated) grain-specific options. + + This function provides backwards compatibility for grain-specific options + such as ``enable_fqdns_grains: False`` + """ + grain_specific_option = re.compile(r"(?:enable_)?(?P[\w_]+)grains$") + if "disabled_grains" not in opts: + opts["disabled_grains"] = [] + + for opt_name, opt_val in opts.items(): + m = re.match(grain_specific_option, opt_name) + if m: + grain = m["grain"].rstrip("_") + if opt_val is True and grain in opts["disabled_grains"]: + salt.utils.versions.warn_until( + "Calcium", + f"The '{opt_name}' option is deprecated. Use 'disabled_grains' instead.", + ) + opts["disabled_grains"].remove(grain) + elif opt_val is False: + salt.utils.versions.warn_until( + "Calcium", + f"The '{opt_name}' option is deprecated. Use 'disabled_grains' instead.", + ) + opts["disabled_grains"].append(grain) + else: + # regex hit a non-bool option such as start_event_grains + pass + + def _update_discovery_config(opts): """ Update discovery config for all instances. @@ -4554,6 +4578,10 @@ def apply_master_config(overrides=None, defaults=None): ) salt.features.setup_features(opts) + + # Update disabled_grains from deprecated, individual configs + _update_disabled_grains(opts) + return opts diff --git a/salt/grains/core.py b/salt/grains/core.py index 5a4ae1ae671e..b7df5676efea 100644 --- a/salt/grains/core.py +++ b/salt/grains/core.py @@ -2954,24 +2954,13 @@ def fqdns(): """ Return all known FQDNs for the system by enumerating all interfaces and then trying to reverse resolve them (excluding 'lo' interface). - To disable the fqdns grain, set enable_fqdns_grains: False in the minion configuration file. + To disable the fqdns grain, add "fqdns" to "disabled_grains" in the minion + configuration file. """ # Provides: # fqdns opt = {"fqdns": []} - if __opts__.get( - "enable_fqdns_grains", - ( - False - if salt.utils.platform.is_windows() - or salt.utils.platform.is_proxy() - or salt.utils.platform.is_sunos() - or salt.utils.platform.is_aix() - or salt.utils.platform.is_junos() - or salt.utils.platform.is_darwin() - else True - ), - ): + if "fqdns" not in __opts__.get("disabled_grains", []): opt = __salt__["network.fqdns"]() return opt diff --git a/salt/grains/lvm.py b/salt/grains/lvm.py index bb08de96e8b8..03be66b493c3 100644 --- a/salt/grains/lvm.py +++ b/salt/grains/lvm.py @@ -1,5 +1,14 @@ """ Detect LVM Volumes + +.. versionchanged:: 3009.0 + +To disable these grains add "lvm" to "disabled_grains" in the minion config. + +.. code-block:: yaml + + disabled_grains: + - lvm """ import logging @@ -16,6 +25,14 @@ log = logging.getLogger(__name__) +__virtualname__ = "lvm" + + +def __virtual__(): + if __virtualname__ in __opts__.get("disabled_grains", []): + return False + return __virtualname__ + def lvm(): """ diff --git a/tests/pytests/unit/config/test_config.py b/tests/pytests/unit/config/test_config.py new file mode 100644 index 000000000000..48d1f93a9010 --- /dev/null +++ b/tests/pytests/unit/config/test_config.py @@ -0,0 +1,74 @@ +import importlib + +import pytest + +import salt.config +import salt.utils +from tests.support.mock import patch + + +def os_config(os): + """Fixture helper that manipulates all is_ functions. + The is_ function for the passed ``os`` returns True, all others + return ``False``. + """ + is_os_functions = [f for f in dir(salt.utils.platform) if f.startswith("is_")] + + def gen_return(is_os_func, os): + if is_os_func.endswith(os): + return lambda: True + return lambda: False + + patch_args = {func: gen_return(func, os) for func in is_os_functions} + + with patch.multiple("salt.utils.platform", **patch_args): + yield importlib.reload(salt.config) + importlib.reload(salt.config) + + +@pytest.fixture +def linux_config(): + yield from os_config("linux") + + +@pytest.fixture +def proxy_config(): + yield from os_config("proxy") + + +@pytest.fixture +def windows_config(): + yield from os_config("windows") + + +@pytest.fixture +def macos_config(): + yield from os_config("darwin") + + +def test_disabled_grains_defaults_linux(linux_config): + with patch("salt.config", linux_config): + result = salt.config.DEFAULT_MINION_OPTS["disabled_grains"] + expected = ["fqdns"] + assert sorted(result) == sorted(expected) + + +def test_disabled_grains_defaults_proxy(proxy_config): + with patch("salt.config", proxy_config): + result = salt.config.DEFAULT_MINION_OPTS["disabled_grains"] + expected = ["fqdns"] + assert sorted(result) == sorted(expected) + + +def test_disabled_grains_defaults_windows(windows_config): + with patch("salt.config", windows_config): + result = salt.config.DEFAULT_MINION_OPTS["disabled_grains"] + expected = ["fqdns"] + assert sorted(result) == sorted(expected) + + +def test_disabled_grains_defaults_macos(macos_config): + with patch("salt.config", macos_config): + result = salt.config.DEFAULT_MINION_OPTS["disabled_grains"] + expected = ["fqdns"] + assert sorted(result) == sorted(expected) diff --git a/tests/pytests/unit/grains/test_core.py b/tests/pytests/unit/grains/test_core.py index 1a1bb7632c27..aabc0c9ff5a9 100644 --- a/tests/pytests/unit/grains/test_core.py +++ b/tests/pytests/unit/grains/test_core.py @@ -2527,102 +2527,15 @@ def test_dns_return(ipv4_tuple, ipv6_tuple): assert core.dns() == {} -def test_enable_fqdns_false(): - """ - tests enable_fqdns_grains is set to False - """ - with patch.dict("salt.grains.core.__opts__", {"enable_fqdns_grains": False}): - assert core.fqdns() == {"fqdns": []} - - -def test_enable_fqdns_true(): - """ - testing that grains uses network.fqdns module - """ - with patch.dict( - "salt.grains.core.__salt__", - {"network.fqdns": MagicMock(return_value="my.fake.domain")}, - ), patch.dict("salt.grains.core.__opts__", {"enable_fqdns_grains": True}): - assert core.fqdns() == "my.fake.domain" - - -def test_enable_fqdns_none(): - """ - testing default fqdns grains is returned when enable_fqdns_grains is None - """ - with patch.dict("salt.grains.core.__opts__", {"enable_fqdns_grains": None}): - assert core.fqdns() == {"fqdns": []} - - -def test_enable_fqdns_without_patching(): - """ - testing fqdns grains is enabled by default - """ +@pytest.mark.parametrize( + "disabled_grains,expected", (([], ["my.fake.domain"]), (["fqdns"], [])) +) +def test_fqdns_disabling(disabled_grains, expected): with patch.dict( "salt.grains.core.__salt__", - {"network.fqdns": MagicMock(return_value="my.fake.domain")}, - ): - # fqdns is disabled by default on Windows and macOS - if salt.utils.platform.is_windows() or salt.utils.platform.is_darwin(): - assert core.fqdns() == {"fqdns": []} - else: - assert core.fqdns() == "my.fake.domain" - - -def test_enable_fqdns_false_is_proxy(): - """ - testing fqdns grains is disabled by default for proxy minions - """ - with patch( - "salt.utils.platform.is_proxy", return_value=True, autospec=True - ), patch.dict( - "salt.grains.core.__salt__", - {"network.fqdns": MagicMock(return_value="my.fake.domain")}, - ): - # fqdns is disabled by default on proxy minions - assert core.fqdns() == {"fqdns": []} - - -def test_enable_fqdns_false_is_aix(): - """ - testing fqdns grains is disabled by default for minions on AIX - """ - with patch( - "salt.utils.platform.is_aix", return_value=True, autospec=True - ), patch.dict( - "salt.grains.core.__salt__", - {"network.fqdns": MagicMock(return_value="my.fake.domain")}, - ): - # fqdns is disabled by default on minions on AIX - assert core.fqdns() == {"fqdns": []} - - -def test_enable_fqdns_false_is_sunos(): - """ - testing fqdns grains is disabled by default for minions on Solaris platforms - """ - with patch( - "salt.utils.platform.is_sunos", return_value=True, autospec=True - ), patch.dict( - "salt.grains.core.__salt__", - {"network.fqdns": MagicMock(return_value="my.fake.domain")}, - ): - # fqdns is disabled by default on minions on Solaris platforms - assert core.fqdns() == {"fqdns": []} - - -def test_enable_fqdns_false_is_junos(): - """ - testing fqdns grains is disabled by default for minions on Junos - """ - with patch( - "salt.utils.platform.is_junos", return_value=True, autospec=True - ), patch.dict( - "salt.grains.core.__salt__", - {"network.fqdns": MagicMock(return_value="my.fake.domain")}, - ): - # fqdns is disabled by default on minions on Junos (Juniper) - assert core.fqdns() == {"fqdns": []} + {"network.fqdns": MagicMock(return_value={"fqdns": ["my.fake.domain"]})}, + ), patch.dict("salt.grains.core.__opts__", {"disabled_grains": disabled_grains}): + assert core.fqdns() == {"fqdns": expected} @pytest.mark.skip_unless_on_linux diff --git a/tests/pytests/unit/grains/test_lvm.py b/tests/pytests/unit/grains/test_lvm.py index 5bad50b66a02..2b8c12d3d8ec 100644 --- a/tests/pytests/unit/grains/test_lvm.py +++ b/tests/pytests/unit/grains/test_lvm.py @@ -11,7 +11,7 @@ @pytest.fixture def configure_loader_modules(): return { - lvm: {"__salt__": {}}, + lvm: {"__salt__": {}, "__opts__": {}}, } @@ -239,3 +239,27 @@ def test__aix_lvm(): "othervg": ["loglv01", "datalv"], } }, ret + + +def test_virtual_enabled(): + """ + Test lvm.__virtual__ when lvm is not in disabled_grains + """ + with patch.dict(lvm.__opts__, {"disabled_grains": []}): + assert lvm.__virtual__() == "lvm" + + +def test_virtual_disabled(): + """ + Test lvm.__virtual__ when lvm is in disabled_grains + """ + with patch.dict(lvm.__opts__, {"disabled_grains": ["lvm"]}): + assert lvm.__virtual__() is False + + +def test_virtual_disabled_grains_not_set(): + """ + Test lvm.__virtual__ when disabled_grains is not in opts (lvm enabled by default) + """ + with patch.dict(lvm.__opts__, {}): + assert lvm.__virtual__() == "lvm"