From 489839d88d7ec85a91b5e6dcc0c218994779c8f6 Mon Sep 17 00:00:00 2001 From: moowiz Date: Mon, 26 Sep 2022 13:24:00 -0700 Subject: [PATCH 1/6] Remove overly verbose logging --- poe_client/client.py | 2 -- poe_client/rate_limiter.py | 16 ++++------------ 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/poe_client/client.py b/poe_client/client.py index 474c32f..5252df0 100644 --- a/poe_client/client.py +++ b/poe_client/client.py @@ -170,10 +170,8 @@ async def _get_json( # We ignore typing in the dict assignment. kwargs only has dicts as values, # but we're assigning booleans here. We can't set the typing inline without # flake8 complaining about overly complex annotation. - logging.info("NOT BLOCKING") kwargs["raise_for_status"] = True # type: ignore else: - logging.info("BLOCKING") kwargs["raise_for_status"] = False # type: ignore # The types are ignored because for some reason it can't understand diff --git a/poe_client/rate_limiter.py b/poe_client/rate_limiter.py index 3345b00..97d5b19 100644 --- a/poe_client/rate_limiter.py +++ b/poe_client/rate_limiter.py @@ -46,22 +46,16 @@ def __init__(self, name: str, max_hits: int, period: int, restriction: int): async def update_state(self, current_hits: int, restriction: int): """Update the state of the policy.""" async with self.mutex: - logging.info( - "Updating state[{0}] to {1} hits, {2} restriction".format( - self.name, - current_hits, - restriction, - ) - ) self.state = PolicyState(current_hits, restriction) async def get_semaphore(self) -> bool: """Check state to see if request is allowed.""" - logging.info("{0} = {1}".format(self.name, self.state.__dict__)) # If last request was restricted, wait and allow if self.state.restriction: logging.info( - "restricted. Sleeping for {0} seconds".format(self.state.restriction) + "restricted. Sleeping for {0} seconds".format( + self.state.restriction + 1 + ) ) await asyncio.sleep(self.state.restriction + 1) return True @@ -73,7 +67,7 @@ async def get_semaphore(self) -> bool: if self.state.current_hits >= self.max_hits: logging.info( - "max hits reached. Sleeping for {0} seconds".format(self.period) + "max hits reached. Sleeping for {0} seconds".format(self.period + 1) ) await asyncio.sleep(self.period + 1) return True @@ -133,13 +127,11 @@ async def parse_headers(self, headers) -> str: async def get_semaphore(self, policy_name: str) -> bool: """Get a semaphore to make a request.""" if not self.policies: - logging.info("No policies, do a blocking request") return False semaphores = [] for name, policy in self.policies.items(): if name.startswith(policy_name): - logging.info("getting semaphore {0}".format(name)) for limit in policy.values(): semaphores.append(limit.get_semaphore()) From 4e5d635d9c0cdf6fab3db3948fc52dc280633afe Mon Sep 17 00:00:00 2001 From: Stephen Martinis Date: Mon, 26 Sep 2022 13:29:25 -0700 Subject: [PATCH 2/6] Update rate_limiter.py --- poe_client/rate_limiter.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/poe_client/rate_limiter.py b/poe_client/rate_limiter.py index d460e92..617ae0e 100644 --- a/poe_client/rate_limiter.py +++ b/poe_client/rate_limiter.py @@ -47,14 +47,15 @@ def __init__(self, name: str, max_hits: int, period: int, restriction: int): async def update_state(self, current_hits: int, restriction: int): """Update the state of the policy.""" async with self.mutex: - self.state = PolicyState(current_hits, restriction) + self.state.current_hits = current_hits + self.state.restriction = restriction async def get_semaphore(self) -> bool: """Check state to see if request is allowed.""" # If last request was restricted, wait and allow if self.state.restriction: logging.info( - "restricted. Sleeping for {0} seconds".format( + "Rate limiter restricted. Sleeping for {0} seconds".format( self.state.restriction + 1 ) @@ -64,7 +65,7 @@ async def get_semaphore(self) -> bool: if self.state.current_hits >= self.max_hits: logging.info( - "max hits reached. Sleeping for {0} seconds".format(self.period + 1) + "Rate limiter max hits reached. Sleeping for {0} seconds".format(self.period + 1) ) await asyncio.sleep(self.period + 1) return True From ad47b874a2a79c03dae3b97fb325500c6ee5f092 Mon Sep 17 00:00:00 2001 From: moowiz Date: Tue, 9 May 2023 09:00:52 -0700 Subject: [PATCH 3/6] Remove extraneous logging Also formats the file --- poe_client/rate_limiter.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poe_client/rate_limiter.py b/poe_client/rate_limiter.py index 617ae0e..735685f 100644 --- a/poe_client/rate_limiter.py +++ b/poe_client/rate_limiter.py @@ -57,7 +57,6 @@ async def get_semaphore(self) -> bool: logging.info( "Rate limiter restricted. Sleeping for {0} seconds".format( self.state.restriction + 1 - ) ) await asyncio.sleep(self.state.restriction + 1) @@ -65,7 +64,9 @@ async def get_semaphore(self) -> bool: if self.state.current_hits >= self.max_hits: logging.info( - "Rate limiter max hits reached. Sleeping for {0} seconds".format(self.period + 1) + "Rate limiter max hits reached. Sleeping for {0} seconds".format( + self.period + 1 + ) ) await asyncio.sleep(self.period + 1) return True @@ -136,7 +137,6 @@ async def get_semaphore(self, policy_name: str) -> bool: """Get a semaphore to make a request.""" async with self.mutex: if not self.policies: - logging.debug("No policies, do a blocking request") return False semaphores = [] From 4169e03a44f11119ba1197f77599971a4235c5f1 Mon Sep 17 00:00:00 2001 From: moowiz Date: Tue, 9 May 2023 09:04:54 -0700 Subject: [PATCH 4/6] Remove functional changes --- poe_client/rate_limiter.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/poe_client/rate_limiter.py b/poe_client/rate_limiter.py index 735685f..38c89d1 100644 --- a/poe_client/rate_limiter.py +++ b/poe_client/rate_limiter.py @@ -56,19 +56,19 @@ async def get_semaphore(self) -> bool: if self.state.restriction: logging.info( "Rate limiter restricted. Sleeping for {0} seconds".format( - self.state.restriction + 1 + self.state.restriction ) ) - await asyncio.sleep(self.state.restriction + 1) + await asyncio.sleep(self.state.restriction) return True if self.state.current_hits >= self.max_hits: logging.info( "Rate limiter max hits reached. Sleeping for {0} seconds".format( - self.period + 1 + self.period ) ) - await asyncio.sleep(self.period + 1) + await asyncio.sleep(self.period) return True # If we haven't reached the quota, increase and allow From 7f8041d776d24249e15af5744a145ba68e401281 Mon Sep 17 00:00:00 2001 From: moowiz Date: Fri, 12 May 2023 19:34:16 -0700 Subject: [PATCH 5/6] Debug log for fetch error --- poe_client/client.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/poe_client/client.py b/poe_client/client.py index 52c209e..b7eec40 100644 --- a/poe_client/client.py +++ b/poe_client/client.py @@ -187,6 +187,9 @@ async def _get_json( ] = await self._limiter.parse_headers(resp.headers) if resp.status != 200: + logging.debug( + "Got status code %s with text %s", resp.status, resp.text() + ) raise ValueError( "Invalid request: status code {0}, expected 200".format( resp.status, From d90c760ee9e6014abac12775bc2eeb369d0afd32 Mon Sep 17 00:00:00 2001 From: moowiz Date: Fri, 12 May 2023 22:03:14 -0700 Subject: [PATCH 6/6] Better logging --- poe_client/client.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/poe_client/client.py b/poe_client/client.py index b7eec40..cc4112f 100644 --- a/poe_client/client.py +++ b/poe_client/client.py @@ -188,7 +188,10 @@ async def _get_json( if resp.status != 200: logging.debug( - "Got status code %s with text %s", resp.status, resp.text() + "Got status code %s with text %s and headers %s", + resp.status, + await resp.text(), + resp.headers, ) raise ValueError( "Invalid request: status code {0}, expected 200".format(