Merge branch 'main' into develop

Author: mhucka

CHANGES.md

@@ -1,5 +1,28 @@
 # Change log for CommonPy
 
+## Version 1.12.4
+
+Changes in this release:
+* Revise the retry algorithm in `timed_request()` again.
+* Make `timed_request()` print the response text in case of failures
+* Remove `extra_requires` from `setup.py` because it caused installation problems due to the syntax of the requirements file and I don't have patience for figuring out what `setuptools` needs this time.
+
+
+## Version 1.12.3
+
+This release updates the version of the `dateparser` package in `requirements.txt`, to avoid a deprecation warning when using CommonPy's `parsed_datetime(...)` function.
+
+
+## Version 1.12.2
+
+This release updates the versions of dependencies in `requirements.txt`.
+
+
+## Version 1.12.1
+
+This version prevents certain exceptions from being buried and ignored, and improves the network failure retry algorithm.
+
+
 ## Version 1.12.0
 
 Additions in this release:

CITATION.cff

@@ -10,8 +10,8 @@ cff-version: "1.1.0"
 message: "If you use this software, please cite it using these metadata."
 repository-code: "https://github.com/caltechlibrary/commonpy"
 title: "CommonPy: assortment of Python helper functions and utility classes"
-date-released: 2023-03-24
-version: "1.12.0"
+date-released: 2023-04-21
+version: "1.12.4"
 doi: 10.22002/20217
 keywords: 
   - Python

codemeta.json

@@ -6,7 +6,7 @@
   "codeRepository": "https://github.com/caltechlibrary/commonpy",
   "issueTracker": "https://github.com/caltechlibrary/commonpy/issues",
   "license": "https://github.com/caltechlibrary/commonpy/blob/master/LICENSE",
-  "version": "1.12.0",
+  "version": "1.12.4",
   "author": [
     {
       "@type": "Person",

commonpy/__init__.py

@@ -21,7 +21,7 @@
 #  |    by the Makefile. Manual changes to these values will be lost.      |
 #  ╰────────────────────── Notice ── Notice ── Notice ─────────────────────╯
 
-__version__     = '1.12.0'
+__version__     = '1.12.4'
 __description__ = 'Assortment of Python helper functions and utility classes'
 __url__         = 'https://github.com/caltechlibrary/commonpy'
 __author__      = 'Michael Hucka'

commonpy/network_utils.py

@@ -37,9 +37,6 @@
 encountering a network error before they stop and give up.'''
 
 _MAX_CONSECUTIVE_FAILS = 3
-'''Maximum number of consecutive failures before pause and try another round.'''
-
-_MAX_RETRIES = 5
 '''Maximum number of times we back off and try again.  This also affects the
 maximum wait time that will be reached after repeated retries.'''
 
@@ -148,24 +145,26 @@ def addurl(text):
     elif client == 'stream':
         client = httpx.stream
 
+    response = None
     failures = 0
-    retries = 0
     error = None
-    while failures < _MAX_CONSECUTIVE_FAILS and not interrupted():
+    while failures <= _MAX_CONSECUTIVE_FAILS and not interrupted():
         try:
             if __debug__: log(addurl(f'doing http {method}'))
             func = getattr(client, method)
             response = func(url, **kwargs)
             # For some statuses, retry once, in case it's a transient problem.
             code = response.status_code
             if __debug__: log(addurl(f'got response with code {code}'))
-            if code not in [400, 409, 502, 503, 504] or failures > 0:
+            if code not in [400, 409, 502, 503, 504] or failures > _MAX_CONSECUTIVE_FAILS:
                 return response
-            else:
-                failures += 1
         except KeyboardInterrupt as ex:
             if __debug__: log(addurl(f'network {method} interrupted by {antiformat(ex)}'))
             raise
+        except ImportError as ex:
+            # This can happen if the installation environment has inconsistecies.
+            log('import error: ' + str(ex))
+            raise
         except (TypeError, httpx.UnsupportedProtocol) as ex:
             # Bad arguments to the call, like passing data to a 'get'.
             if __debug__: log(addurl(f'exception {antiformat(ex)}'))
@@ -178,7 +177,6 @@ def addurl(text):
             if __debug__: log(addurl(f'exception {antiformat(ex)}'))
             if failures > 0:
                 raise
-            failures += 1
             if __debug__: log(addurl('will retry one more time after brief pause'))
         except Exception as ex:         # noqa PIE786
             # Problem might be transient.  Don't quit right away.
@@ -189,20 +187,27 @@ def addurl(text):
             # about being unable to reconnect and not the original problem.
             if not error:
                 error = ex
-        if failures >= _MAX_CONSECUTIVE_FAILS:
+        # If there's response text, it may contain diagnostic info.
+        if __debug__ and response:
+            text = response.text[:500] + (' ...' if len(response.text) > 500 else '')
+            log('response text: ' + text)
+        if failures == 0:
+            # Pause briefly b/c it's rarely a good idea to retry immediately.
+            if __debug__: log(addurl('pausing briefly before retrying'))
+            wait(0.5)
+            failures += 1
+        elif failures < _MAX_CONSECUTIVE_FAILS:
             # Pause with exponential back-off, reset failure count & try again.
-            if retries < _MAX_RETRIES:
-                retries += 1
-                failures = 0
-                pause = 10 * retries * retries
-                if __debug__: log(addurl('pausing due to consecutive failures'))
-                wait(pause)
-            else:
-                if __debug__: log(addurl('exceeded max failures and max retries'))
+            pause = 5 * failures * failures
+            if __debug__: log(addurl('pausing due to consecutive failures'))
+            wait(pause)
+            failures += 1
+        else:
+            if __debug__: log(addurl('exceeded max failures'))
+            if error:
                 raise error
-        # Pause briefly b/c it's rarely a good idea to retry immediately.
-        if __debug__: log(addurl('pausing briefly before retrying'))
-        wait(0.5)
+            else:
+                return response
     if interrupted():
         if __debug__: log(addurl('interrupted'))
         raise Interrupted(addurl('Network request has been interrupted'))

requirements.txt

@@ -8,10 +8,10 @@
 
 PyYAML               >= 5.3.1
 boltons              == 21.0.0
-dateparser           == 1.0.0
+dateparser           == 1.1.8
 deprecation          == 2.1.0
-h2                   == 3.2.0
-httpx                == 0.23.0
+h2                   == 4.0.0
+httpx                == 0.23.1
 humanize             >= 3.9.0
 python_dateutil      == 2.8.2
 pywin32              >= 301; sys_platform == 'win32'
@@ -22,6 +22,7 @@ pywin32              >= 301; sys_platform == 'win32'
 # to Stack Overflow at https://stackoverflow.com/a/71504213/743730.
 regex                <= 2022.3.2
 
+setuptools           >= 67.5.1
 sidetrack            >= 2.0.0
 tldextract           == 3.0.2
 validator-collection == 1.5.0

setup.cfg

@@ -8,7 +8,7 @@
 
 [metadata]
 name          = commonpy
-version       = 1.12.0
+version       = 1.12.4
 description   = Assortment of Python helper functions and utility classes
 author        = Michael Hucka
 author_email  = mhucka@caltech.edu

setup.py

@@ -13,6 +13,7 @@
 
 from setuptools import setup
 
+
 def requirements(file):
     from os import path
     required = []
@@ -33,8 +34,8 @@ def requirements(file):
                 pass
     return required
 
+
 setup(
     setup_requires=['wheel'],
     install_requires=requirements('requirements.txt'),
-    extras_require={'dev': requirements('requirements-dev.txt')},
 )