From 18718b28146bc4fb391ce907df5a19aef91684d3 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 20 Nov 2024 16:02:15 -0500 Subject: [PATCH 1/4] BF: restrict value of get_fdmax MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In my case I kept finding celery running at 100% and doing nothing. py-spy pointed to the close_open_fds and then ulimit inside the container showed gory detail of ❯ docker run -it --rm --entrypoint bash dandiarchive/dandiarchive-api -c "ulimit -n" 1073741816 situation is not unique to me. See more at https://github.com/dandi/dandi-cli/issues/1488 --- billiard/compat.py | 40 ++++++++++++++++++++++++++++++---------- 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/billiard/compat.py b/billiard/compat.py index bea9746..81f6066 100644 --- a/billiard/compat.py +++ b/billiard/compat.py @@ -100,16 +100,36 @@ def get_fdmax(default=None): descriptor limit. """ - try: - return os.sysconf('SC_OPEN_MAX') - except: - pass - if resource is None: # Windows - return default - fdmax = resource.getrlimit(resource.RLIMIT_NOFILE)[1] - if fdmax == resource.RLIM_INFINITY: - return default - return fdmax + # Some system might be "mis"configured and allow for ulimit -n + # of e.g. 1073741816 which would be infeasible to sweep through. + def _get_fdmax(): + try: + return os.sysconf('SC_OPEN_MAX') + except: + pass + if resource is None: # Windows + return default + fdmax = resource.getrlimit(resource.RLIMIT_NOFILE)[1] + if fdmax == resource.RLIM_INFINITY: + return default + return fdmax + + value = _get_fdmax() + if value != default and value >= 1e5 : + # limiting value is ad-hoc and already more than sensible + import warnings + if default: + new_value = default + msg = "default" + else: + new_value = 10000 + msg = "new smaller" + warnings.warn(UserWarning( + f"System set max number of open files is way too high and set to {value}." + f"Will use {msg} value of {new_value}")) + return new_value + else: + return value def uniq(it): From 35c3ae1c5376cbd81ec28f13c1ab6d8afaa294d5 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 21 Nov 2024 10:58:19 -0500 Subject: [PATCH 2/4] RF limiting within get_fdmax into something more sensible --- billiard/compat.py | 39 +++++++++++++++++++-------------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/billiard/compat.py b/billiard/compat.py index 81f6066..6f318e6 100644 --- a/billiard/compat.py +++ b/billiard/compat.py @@ -3,6 +3,7 @@ import os import subprocess import sys +import warnings from itertools import zip_longest @@ -100,36 +101,34 @@ def get_fdmax(default=None): descriptor limit. """ - # Some system might be "mis"configured and allow for ulimit -n - # of e.g. 1073741816 which would be infeasible to sweep through. - def _get_fdmax(): - try: - return os.sysconf('SC_OPEN_MAX') - except: - pass + + try: + fdmax = os.sysconf('SC_OPEN_MAX') + except: + fdmax = None + + if not fdmax: if resource is None: # Windows return default fdmax = resource.getrlimit(resource.RLIMIT_NOFILE)[1] if fdmax == resource.RLIM_INFINITY: return default - return fdmax - - value = _get_fdmax() - if value != default and value >= 1e5 : + # Some system might be "mis"configured and allow for ulimit -n + # of e.g. 1073741816 which would be infeasible to sweep through. + if fdmax >= 1e5: # limiting value is ad-hoc and already more than sensible - import warnings if default: - new_value = default - msg = "default" + fdmax_limited = default + msg = "the default" else: - new_value = 10000 - msg = "new smaller" + fdmax_limited = 10000 + msg = "a new smaller" warnings.warn(UserWarning( - f"System set max number of open files is way too high and set to {value}." - f"Will use {msg} value of {new_value}")) - return new_value + f"System set max number of open files ({fdmax}) is way too high. " + f"Will use {msg} value of {fdmax_limited}")) + return fdmax_limited else: - return value + return fdmax def uniq(it): From 3a8f7b6ff7c1a6e04a86d89555c5f9a4c4eb4818 Mon Sep 17 00:00:00 2001 From: Asif Saif Uddin Date: Wed, 21 May 2025 05:59:49 +0000 Subject: [PATCH 3/4] Update billiard/compat.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- billiard/compat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/billiard/compat.py b/billiard/compat.py index 6f318e6..d9db08e 100644 --- a/billiard/compat.py +++ b/billiard/compat.py @@ -107,7 +107,7 @@ def get_fdmax(default=None): except: fdmax = None - if not fdmax: + if fdmax is None: if resource is None: # Windows return default fdmax = resource.getrlimit(resource.RLIMIT_NOFILE)[1] From e2af20efa2e78cd64c0993968b10c9c628ce5cd9 Mon Sep 17 00:00:00 2001 From: Asif Saif Uddin Date: Wed, 21 May 2025 06:03:42 +0000 Subject: [PATCH 4/4] Update billiard/compat.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- billiard/compat.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/billiard/compat.py b/billiard/compat.py index d9db08e..eedc7a9 100644 --- a/billiard/compat.py +++ b/billiard/compat.py @@ -123,9 +123,10 @@ def get_fdmax(default=None): else: fdmax_limited = 10000 msg = "a new smaller" - warnings.warn(UserWarning( + warnings.warn( f"System set max number of open files ({fdmax}) is way too high. " - f"Will use {msg} value of {fdmax_limited}")) + f"Will use {msg} value of {fdmax_limited}", + UserWarning) return fdmax_limited else: return fdmax