diff --git a/.gitignore b/.gitignore index 3534373a..426296cc 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,7 @@ venv /.pytest_cache /.tox /.artifacts +/test_* +/*.yaml +*.bin +!/example_bins/*.bin diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 899de43a..0dd56c49 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,3 +1,4 @@ +exclude: ^example_bins repos: - repo: https://github.com/psf/black rev: 23.3.0 diff --git a/example_bins/202.bin b/example_bins/202.bin new file mode 100644 index 00000000..a0aba931 --- /dev/null +++ b/example_bins/202.bin @@ -0,0 +1 @@ +OK \ No newline at end of file diff --git a/example_bins/400.bin b/example_bins/400.bin new file mode 100644 index 00000000..bc4a7469 --- /dev/null +++ b/example_bins/400.bin @@ -0,0 +1 @@ +Invalid status code \ No newline at end of file diff --git a/example_bins/500.bin b/example_bins/500.bin new file mode 100644 index 00000000..87a218a5 --- /dev/null +++ b/example_bins/500.bin @@ -0,0 +1 @@ +500 Internal Server Error \ No newline at end of file diff --git a/responses/__init__.py b/responses/__init__.py index d990091f..ff280782 100644 --- a/responses/__init__.py +++ b/responses/__init__.py @@ -1,6 +1,7 @@ import inspect import json as json_module import logging +import os from functools import partialmethod from functools import wraps from http import client @@ -56,7 +57,6 @@ if TYPE_CHECKING: # pragma: no cover # import only for linter run - import os from typing import Protocol from unittest.mock import _patch as _mock_patcher @@ -529,6 +529,7 @@ def calls(self) -> CallList: def _form_response( + method: Optional[str], body: Union[BufferedReader, BytesIO], headers: Optional[Mapping[str, str]], status: int, @@ -566,6 +567,7 @@ def _form_response( headers=headers, original_response=orig_response, # type: ignore[arg-type] # See comment above preload_content=False, + request_method=method, ) @@ -632,7 +634,7 @@ def get_response(self, request: "PreparedRequest") -> HTTPResponse: content_length = len(body.getvalue()) headers["Content-Length"] = str(content_length) - return _form_response(body, headers, status) + return _form_response(request.method, body, headers, status) def __repr__(self) -> str: return ( @@ -695,7 +697,7 @@ def get_response(self, request: "PreparedRequest") -> HTTPResponse: body = _handle_body(body) headers.extend(r_headers) - return _form_response(body, headers, status) + return _form_response(request.method, body, headers, status) class PassthroughResponse(BaseResponse): @@ -842,14 +844,24 @@ def _parse_response_file( def _add_from_file(self, file_path: "Union[str, bytes, os.PathLike[Any]]") -> None: data = self._parse_response_file(file_path) + parent_directory = os.path.dirname(os.path.abspath(file_path)) for rsp in data["responses"]: rsp = rsp["response"] + headers = dict(rsp.get("headers") or {}) + if "Content-Type" in headers: + headers.pop("Content-Type") + if "body_file" in rsp: + with open(os.path.join(parent_directory, rsp["body_file"]), "rb") as f: + body = f.read() + else: + body = rsp["body"] self.add( method=rsp["method"], url=rsp["url"], - body=rsp["body"], + body=body, status=rsp["status"], + headers=headers, content_type=rsp["content_type"], auto_calculate_content_length=rsp["auto_calculate_content_length"], ) diff --git a/responses/_recorder.py b/responses/_recorder.py index a533e848..04e2f20e 100644 --- a/responses/_recorder.py +++ b/responses/_recorder.py @@ -1,16 +1,18 @@ +import copy +import os +import pathlib +import uuid from functools import wraps from typing import TYPE_CHECKING if TYPE_CHECKING: # pragma: no cover - import os - from typing import Any - from typing import BinaryIO from typing import Callable from typing import Dict from typing import List from typing import Type from typing import Union + from typing import IO from responses import FirstMatchRegistry from responses import HTTPAdapter from responses import PreparedRequest @@ -18,10 +20,9 @@ from responses import _F from responses import BaseResponse - from io import TextIOWrapper - import yaml +from responses import _UNSET from responses import RequestsMock from responses import Response from responses import _real_send @@ -38,19 +39,43 @@ def _remove_nones(d: "Any") -> "Any": def _dump( registered: "List[BaseResponse]", - destination: "Union[BinaryIO, TextIOWrapper]", - dumper: "Callable[[Union[Dict[Any, Any], List[Any]], Union[BinaryIO, TextIOWrapper]], Any]", + config_file: "Union[str, os.PathLike[str]]", + dumper: "Callable[[Union[Dict[Any, Any], List[Any]], Union[IO[Any]]], None]", + dumper_mode: "str" = "w", ) -> None: data: Dict[str, Any] = {"responses": []} + + # e.g. config_file = 'my/dir/responses.yaml' + # parent_directory = 'my/dir' + # binary_directory = 'my/dir/responses' + config_file = pathlib.Path(config_file) + fname, fext = os.path.splitext(config_file.name) + parent_directory = config_file.absolute().parent + binary_directory = parent_directory / (fname if fext else f"{fname}_bodies") + for rsp in registered: try: content_length = rsp.auto_calculate_content_length # type: ignore[attr-defined] + body = rsp.body + if isinstance(body, bytes): + os.makedirs(binary_directory, exist_ok=True) + bin_file = os.path.join(binary_directory, f"{uuid.uuid4()}.bin") + with open(bin_file, "wb") as bf: + bf.write(body) + + # make sure the stored binary file path is relative to config file + # or the config file and binary directory will be hard to move + body_file = os.path.relpath(bin_file, parent_directory) + body = None + else: + body_file = None data["responses"].append( { "response": { "method": rsp.method, "url": rsp.url, - "body": rsp.body, + "body": body, + "body_file": body_file, "status": rsp.status, "headers": rsp.headers, "content_type": rsp.content_type, @@ -63,7 +88,9 @@ def _dump( "Cannot dump response object." "Probably you use custom Response object that is missing required attributes" ) from exc - dumper(_remove_nones(data), destination) + + with open(config_file, dumper_mode) as cfile: + dumper(_remove_nones(data), cfile) class Recorder(RequestsMock): @@ -79,7 +106,7 @@ def reset(self) -> None: self._registry = OrderedRegistry() def record( - self, *, file_path: "Union[str, bytes, os.PathLike[Any]]" = "response.yaml" + self, *, file_path: "Union[str, os.PathLike[str]]" = "response.yaml" ) -> "Union[Callable[[_F], _F], _F]": def deco_record(function: "_F") -> "Callable[..., Any]": @wraps(function) @@ -99,11 +126,10 @@ def wrapper(*args: "Any", **kwargs: "Any") -> "Any": # type: ignore[misc] def dump_to_file( self, *, - file_path: "Union[str, bytes, os.PathLike[Any]]", + file_path: "Union[str, os.PathLike[str]]", registered: "List[BaseResponse]", ) -> None: - with open(file_path, "w") as file: - _dump(registered, file, yaml.dump) + _dump(registered, file_path, yaml.dump) def _on_request( self, @@ -116,11 +142,33 @@ def _on_request( request.params = self._parse_request_params(request.path_url) # type: ignore[attr-defined] request.req_kwargs = kwargs # type: ignore[attr-defined] requests_response = _real_send(adapter, request, **kwargs) + # the object is a requests.structures.CaseInsensitiveDict object, + # if you re-construct the headers with a primitive dict object, + # some lower case headers like 'content-type' will not be able to be processed properly + # the deepcopy is for making sure the original headers object + # not changed by the following operations + requests_headers = copy.deepcopy(requests_response.headers) + if "Content-Type" in requests_headers: + requests_content_type = requests_headers.pop("Content-Type") + else: + requests_content_type = _UNSET # type: ignore[assignment] + # Content-Encoding should be removed to + # avoid 'Content-Encoding: gzip' causing the error in requests + if "Content-Encoding" in requests_headers: + requests_headers.pop("Content-Encoding") + + # When something like 'Content-Encoding: gzip' is used + # the 'Content-Length' may be the length of compressed data, + # so we need to replace it with decompressed length + if "Content-Length" in requests_headers: + requests_headers["Content-Length"] = str(len(requests_response.content)) responses_response = Response( method=str(request.method), url=str(requests_response.request.url), status=requests_response.status_code, - body=requests_response.text, + headers=dict(requests_headers), + body=requests_response.content, + content_type=requests_content_type, ) self._registry.add(responses_response) return requests_response diff --git a/responses/tests/test_recorder.py b/responses/tests/test_recorder.py index 4508bb9a..57c66877 100644 --- a/responses/tests/test_recorder.py +++ b/responses/tests/test_recorder.py @@ -1,3 +1,5 @@ +import collections.abc +import shutil from pathlib import Path import pytest @@ -15,15 +17,118 @@ # python 3.11 import tomllib as _toml # type: ignore[no-redef] +_NOT_CARE = object() + + +class _CompareDict: + def __init__(self, obj): + self.obj = obj + + def __eq__(self, other): + if self.obj is _NOT_CARE: + return True + elif isinstance(self.obj, collections.abc.Mapping): + if not isinstance(other, collections.abc.Mapping): + return False + if sorted(self.obj.keys()) != sorted(other.keys()): + return False + for key in self.obj.keys(): + if _CompareDict(self.obj[key]) != other[key]: + return False + return True + elif isinstance(self.obj, list): + if not isinstance(other, list): + return False + if len(self.obj) != len(other): + return False + for i, (obj_item, other_item) in enumerate(zip(self.obj, other)): + if _CompareDict(obj_item) != other_item: + return False + return True + else: + return self.obj == other + + +def get_data_for_cmp(host, port): + data = { + "responses": [ + { + "response": { + "method": "GET", + "url": f"http://{host}:{port}/404", + "body_file": _NOT_CARE, + "headers": { + "Connection": _NOT_CARE, + "Content-Length": "13", + "Date": _NOT_CARE, + "Server": _NOT_CARE, + }, + "status": 404, + "content_type": "text/plain", + "auto_calculate_content_length": False, + } + }, + { + "response": { + "method": "GET", + "url": f"http://{host}:{port}/status/wrong", + "body_file": _NOT_CARE, + "headers": { + "Connection": _NOT_CARE, + "Content-Length": "19", + "Date": _NOT_CARE, + "Server": _NOT_CARE, + }, + "status": 400, + "content_type": "text/plain", + "auto_calculate_content_length": False, + } + }, + { + "response": { + "method": "GET", + "url": f"http://{host}:{port}/500", + "body_file": _NOT_CARE, + "headers": { + "Connection": _NOT_CARE, + "Content-Length": "25", + "Date": _NOT_CARE, + "Server": _NOT_CARE, + }, + "status": 500, + "content_type": "text/plain", + "auto_calculate_content_length": False, + } + }, + { + "response": { + "method": "PUT", + "url": f"http://{host}:{port}/202", + "body_file": _NOT_CARE, + "headers": { + "Connection": _NOT_CARE, + "Content-Length": "2", + "Date": _NOT_CARE, + "Server": _NOT_CARE, + }, + "status": 202, + "content_type": "text/plain", + "auto_calculate_content_length": False, + } + }, + ] + } + return _CompareDict(data) -def get_data(host, port): + +def get_data_for_dump(host, port): data = { "responses": [ { "response": { "method": "GET", "url": f"http://{host}:{port}/404", - "body": "404 Not Found", + "body": "404 Not Found", # test the backward support for 0.23.3 "status": 404, "content_type": "text/plain", "auto_calculate_content_length": False, @@ -33,7 +138,15 @@ def get_data(host, port): "response": { "method": "GET", "url": f"http://{host}:{port}/status/wrong", - "body": "Invalid status code", + "body_file": "example_bins/400.bin", + "headers": { + "Connection": "keep-alive", + "Content-Length": "19", + "Date": "Fri, 20 Oct 2023 10:12:13 " "GMT", + "Keep-Alive": "timeout=4", + "Proxy-Connection": "keep-alive", + "Server": "Werkzeug/3.0.0 " "Python/3.8.10", + }, "status": 400, "content_type": "text/plain", "auto_calculate_content_length": False, @@ -43,7 +156,15 @@ def get_data(host, port): "response": { "method": "GET", "url": f"http://{host}:{port}/500", - "body": "500 Internal Server Error", + "body_file": "example_bins/500.bin", + "headers": { + "Connection": "keep-alive", + "Content-Length": "25", + "Date": "Fri, 20 Oct 2023 10:12:13 " "GMT", + "Keep-Alive": "timeout=4", + "Proxy-Connection": "keep-alive", + "Server": "Werkzeug/3.0.0 " "Python/3.8.10", + }, "status": 500, "content_type": "text/plain", "auto_calculate_content_length": False, @@ -53,7 +174,15 @@ def get_data(host, port): "response": { "method": "PUT", "url": f"http://{host}:{port}/202", - "body": "OK", + "body_file": "example_bins/202.bin", + "headers": { + "Connection": "keep-alive", + "Content-Length": "2", + "Date": "Fri, 20 Oct 2023 10:12:13 " "GMT", + "Keep-Alive": "timeout=4", + "Proxy-Connection": "keep-alive", + "Server": "Werkzeug/3.0.0 " "Python/3.8.10", + }, "status": 202, "content_type": "text/plain", "auto_calculate_content_length": False, @@ -70,6 +199,18 @@ def setup_method(self): if self.out_file.exists(): self.out_file.unlink() # pragma: no cover + self.out_bins_dir = Path("response_record_bins") + if self.out_bins_dir.exists(): + shutil.rmtree(self.out_bins_dir) # pragma: no cover + + assert not self.out_file.exists() + + def teardown_method(self): + if self.out_file.exists(): + self.out_file.unlink() + if self.out_bins_dir.exists(): + shutil.rmtree(self.out_bins_dir) + assert not self.out_file.exists() def test_recorder(self, httpserver): @@ -90,14 +231,13 @@ def run(): with open(self.out_file) as file: data = yaml.safe_load(file) - assert data == get_data(httpserver.host, httpserver.port) + assert data == get_data_for_cmp(httpserver.host, httpserver.port) def test_recorder_toml(self, httpserver): custom_recorder = _recorder.Recorder() def dump_to_file(file_path, registered): - with open(file_path, "wb") as file: - _dump(registered, file, tomli_w.dump) # type: ignore[arg-type] + _dump(registered, file_path, tomli_w.dump, "wb") # type: ignore[arg-type] custom_recorder.dump_to_file = dump_to_file # type: ignore[method-assign] @@ -118,7 +258,7 @@ def run(): with open(self.out_file, "rb") as file: data = _toml.load(file) - assert data == get_data(httpserver.host, httpserver.port) + assert data == get_data_for_cmp(httpserver.host, httpserver.port) def prepare_server(self, httpserver): httpserver.expect_request("/500").respond_with_data( @@ -143,10 +283,13 @@ def prepare_server(self, httpserver): class TestReplay: def setup_method(self): self.out_file = Path("response_record") + self.out_bins_dir = Path("response_record_bins") def teardown_method(self): if self.out_file.exists(): self.out_file.unlink() + if self.out_bins_dir.exists(): + shutil.rmtree(self.out_bins_dir) assert not self.out_file.exists() @@ -154,10 +297,10 @@ def teardown_method(self): def test_add_from_file(self, parser): # type: ignore[misc] if parser == yaml: with open(self.out_file, "w") as file: - parser.dump(get_data("example.com", "8080"), file) + parser.dump(get_data_for_dump("example.com", "8080"), file) else: with open(self.out_file, "wb") as file: # type: ignore[assignment] - parser.dump(get_data("example.com", "8080"), file) + parser.dump(get_data_for_dump("example.com", "8080"), file) @responses.activate def run(): @@ -188,10 +331,15 @@ def _parse_resp_f(file_path): assert responses.registered()[4].method == "PUT" assert responses.registered()[5].method == "POST" + assert responses.registered()[1].status == 404 assert responses.registered()[2].status == 400 assert responses.registered()[3].status == 500 - assert responses.registered()[3].body == "500 Internal Server Error" + assert ( + responses.registered()[1].body == "404 Not Found" + ) # test the backward support for 0.23.3 + assert responses.registered()[2].body == b"Invalid status code" + assert responses.registered()[3].body == b"500 Internal Server Error" assert responses.registered()[3].content_type == "text/plain"