From d0c40dc4a7fce60af0a9a17e79017e401fcc7d5b Mon Sep 17 00:00:00 2001 From: Andrew Malyshev Date: Wed, 29 Jan 2025 00:06:20 +0300 Subject: [PATCH 1/5] add proxy and verify cert arguments --- waybackpack/cli.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/waybackpack/cli.py b/waybackpack/cli.py index 5e7f202..5366aab 100644 --- a/waybackpack/cli.py +++ b/waybackpack/cli.py @@ -120,6 +120,19 @@ def parse_args(): help="Sleep X seconds between each post-error retry.", ) + parser.add_argument( + "--proxy", + default=None, + help="Use a proxy to connect to the archive.org" + ) + + parser.add_argument( + "--no-verify-certificate", + action='store_false', + default=True, + help="Trust the certificate of archive.org" + ) + args = parser.parse_args() return args @@ -137,6 +150,8 @@ def main(): follow_redirects=args.follow_redirects, max_retries=args.max_retries, delay_retry=args.delay_retry, + proxy=args.proxy, + verify=args.no_verify_certificate ) snapshots = search( From ab82ce20d66f82000640397b1e97c4be0500f57b Mon Sep 17 00:00:00 2001 From: Andrew Malyshev Date: Wed, 29 Jan 2025 00:06:47 +0300 Subject: [PATCH 2/5] Add handling of proxy and no-verify-certificate args --- waybackpack/session.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/waybackpack/session.py b/waybackpack/session.py index 10f441b..a2e697d 100644 --- a/waybackpack/session.py +++ b/waybackpack/session.py @@ -15,22 +15,29 @@ def __init__( user_agent=DEFAULT_USER_AGENT, max_retries=3, delay_retry=5, + proxy=None, + verify=True ): self.follow_redirects = follow_redirects self.user_agent = user_agent self.max_retries = max_retries self.delay_retry = delay_retry + self.proxy = proxy + self.verify = verify def try_get(self, url, **kwargs): headers = { "User-Agent": self.user_agent, } try: + print(self.verify) res = requests.get( url, allow_redirects=self.follow_redirects, headers=headers, stream=True, + proxies={'http': self.proxy, 'https': self.proxy} if self.proxy else None, + verify=self.verify, **kwargs ) From 6daeb872b17d0afbb411bf41489defc63ae5a755 Mon Sep 17 00:00:00 2001 From: SwiftExtender <69252550+SwiftExtender@users.noreply.github.com> Date: Wed, 29 Jan 2025 00:32:04 +0300 Subject: [PATCH 3/5] Fix typo --- waybackpack/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/waybackpack/cli.py b/waybackpack/cli.py index 5366aab..d256021 100644 --- a/waybackpack/cli.py +++ b/waybackpack/cli.py @@ -130,7 +130,7 @@ def parse_args(): "--no-verify-certificate", action='store_false', default=True, - help="Trust the certificate of archive.org" + help="Don`t verify the certificate of archive.org" ) args = parser.parse_args() From b67c7a544cf9c78f8cc9da18d55adea5021b0600 Mon Sep 17 00:00:00 2001 From: Andrew Malyshev Date: Wed, 29 Jan 2025 00:35:55 +0300 Subject: [PATCH 4/5] Update readme --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 08d90b2..50aecc3 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ usage: waybackpack [-h] [--version] (-d DIR | --list) [--raw] [--root ROOT] [--user-agent USER_AGENT] [--follow-redirects] [--uniques-only] [--collapse COLLAPSE] [--ignore-errors] [--max-retries MAX_RETRIES] [--no-clobber] [--quiet] - [--progress] [--delay DELAY] [--delay-retry DELAY_RETRY] + [--progress] [--delay DELAY] [--delay-retry DELAY_RETRY] [--proxy PROXY] [--no-verify-certificate] url positional arguments: @@ -100,6 +100,9 @@ options: --delay DELAY Sleep X seconds between each fetch. --delay-retry DELAY_RETRY Sleep X seconds between each post-error retry. + --proxy PROXY Use a proxy to connect to the archive.org + --no-verify-certificate + Don`t verify the certificate of archive.org ``` ## Support From 1749ab86c48475421248f49c475f2cf9aa4f073c Mon Sep 17 00:00:00 2001 From: SwiftExtender <69252550+SwiftExtender@users.noreply.github.com> Date: Wed, 29 Jan 2025 12:11:42 +0300 Subject: [PATCH 5/5] remove debug print --- waybackpack/session.py | 1 - 1 file changed, 1 deletion(-) diff --git a/waybackpack/session.py b/waybackpack/session.py index a2e697d..e27f417 100644 --- a/waybackpack/session.py +++ b/waybackpack/session.py @@ -30,7 +30,6 @@ def try_get(self, url, **kwargs): "User-Agent": self.user_agent, } try: - print(self.verify) res = requests.get( url, allow_redirects=self.follow_redirects,