read custom ignore-list of headers for recording vcr cassettes

Author: sabrenner

README.md

@@ -163,6 +163,10 @@ With this configuration set, you can make the following request to the test agen
 curl -X POST 'http://127.0.0.1:9126/vcr/provider1/some/path'
 ```
 
+#### Ignoring Headers in Recorded Cassettes
+
+To ignore headers in recorded cassettes, you can use the `--vcr-ignore-headers` flag or `VCR_IGNORE_HEADERS` environment variable. The list should take the form of `header1,header2,header3`, and will be omitted from the recorded cassettes.
+
 #### AWS Services
 AWS service proxying, specifically recording cassettes for the first time, requires a `AWS_SECRET_ACCESS_KEY` environment variable to be set for the container running the test agent. This is used to recalculate the AWS signature for the request, as the one generated client-side likely used `{test-agent-host}:{test-agent-port}/vcr/{aws-service}` as the host, and the signature will mismatch that on the actual AWS service.
 

ddapm_test_agent/agent.py

@@ -1580,6 +1580,7 @@ def make_app(
     vcr_cassettes_directory: str,
     vcr_ci_mode: bool,
     vcr_provider_map: str,
+    vcr_ignore_headers: str,
 ) -> web.Application:
     agent = Agent()
     app = web.Application(
@@ -1641,7 +1642,9 @@ def make_app(
             web.route(
                 "*",
                 "/vcr/{path:.*}",
-                lambda request: proxy_request(request, vcr_cassettes_directory, vcr_ci_mode, vcr_provider_map),
+                lambda request: proxy_request(
+                    request, vcr_cassettes_directory, vcr_ci_mode, vcr_provider_map, vcr_ignore_headers
+                ),
             ),
         ]
     )
@@ -1955,6 +1958,12 @@ def main(args: Optional[List[str]] = None) -> None:
         default=os.environ.get("VCR_PROVIDER_MAP", ""),
         help="Comma-separated list of provider=base_url tuples to map providers to paths. Used in addition to the default provider paths.",
     )
+    parser.add_argument(
+        "--vcr-ignore-headers",
+        type=str,
+        default=os.environ.get("VCR_IGNORE_HEADERS", ""),
+        help="Comma-separated list of headers to ignore when recording VCR cassettes.",
+    )
     parsed_args = parser.parse_args(args=args)
     logging.basicConfig(level=parsed_args.log_level)
 
@@ -2000,6 +2009,7 @@ def main(args: Optional[List[str]] = None) -> None:
         vcr_cassettes_directory=parsed_args.vcr_cassettes_directory,
         vcr_ci_mode=parsed_args.vcr_ci_mode,
         vcr_provider_map=parsed_args.vcr_provider_map,
+        vcr_ignore_headers=parsed_args.vcr_ignore_headers,
     )
 
     # Validate port configuration

ddapm_test_agent/vcr_proxy.py

@@ -123,14 +123,15 @@ def parse_authorization_header(auth_header: str) -> Dict[str, str]:
     return parsed
 
 
-def get_vcr(subdirectory: str, vcr_cassettes_directory: str) -> vcr.VCR:
+def get_vcr(subdirectory: str, vcr_cassettes_directory: str, vcr_ignore_headers: str) -> vcr.VCR:
     cassette_dir = os.path.join(vcr_cassettes_directory, subdirectory)
+    extra_ignore_headers = vcr_ignore_headers.split(",")
 
     return vcr.VCR(
         cassette_library_dir=cassette_dir,
         record_mode="once",
         match_on=["path", "method"],
-        filter_headers=CASSETTE_FILTER_HEADERS,
+        filter_headers=CASSETTE_FILTER_HEADERS + extra_ignore_headers,
     )
 
 
@@ -156,7 +157,7 @@ def generate_cassette_name(path: str, method: str, body: bytes, vcr_cassette_pre
 
 
 async def proxy_request(
-    request: Request, vcr_cassettes_directory: str, vcr_ci_mode: bool, vcr_provider_map: str
+    request: Request, vcr_cassettes_directory: str, vcr_ci_mode: bool, vcr_provider_map: str, vcr_ignore_headers: str
 ) -> Response:
     set_custom_vcr_providers(vcr_provider_map)
 
@@ -213,7 +214,7 @@ async def proxy_request(
         auth = AWS4Auth(aws_access_key, AWS_SECRET_ACCESS_KEY, AWS_REGION, AWS_SERVICES[provider])
         request_kwargs["auth"] = auth
 
-    with get_vcr(provider, vcr_cassettes_directory).use_cassette(cassette_file_name):
+    with get_vcr(provider, vcr_cassettes_directory, vcr_ignore_headers).use_cassette(cassette_file_name):
         provider_response = requests.request(**request_kwargs)
 
     # Extract content type without charset

releasenotes/notes/vcr-custom-providers-map-a43b7e2d62d02015.yaml

@@ -2,3 +2,5 @@
 features:
   - |
     vcr: Adds support for specifying a list of custom providers in the ``--vcr-provider-map`` flag or ``VCR_PROVIDER_MAP`` environment variable. The list should take the form of ``provider1=http://provider1.com/,provider2=http://provider2.com/``.
+  - |
+    vcr: Adds support for specifying a list of headers to ignore when recording VCR cassettes in the ``--vcr-ignore-headers`` flag or ``VCR_IGNORE_HEADERS`` environment variable. The list should take the form of ``header1,header2,header3``.

tests/conftest.py

@@ -127,6 +127,11 @@ def vcr_provider_map() -> Generator[str, None, None]:
     yield ""
 
 
+@pytest.fixture
+def vcr_ignore_headers() -> Generator[str, None, None]:
+    yield ""
+
+
 @pytest.fixture
 async def agent_app(
     aiohttp_server,
@@ -145,6 +150,7 @@ async def agent_app(
     vcr_cassettes_directory,
     vcr_ci_mode,
     vcr_provider_map,
+    vcr_ignore_headers,
 ):
     app = await aiohttp_server(
         make_app(
@@ -163,6 +169,7 @@ async def agent_app(
             vcr_cassettes_directory=vcr_cassettes_directory,
             vcr_ci_mode=vcr_ci_mode,
             vcr_provider_map=vcr_provider_map,
+            vcr_ignore_headers=vcr_ignore_headers,
         )
     )
     yield app

tests/test_vcr_proxy.py

@@ -5,14 +5,17 @@
 import threading
 from typing import Any
 from typing import AsyncGenerator
+from typing import Dict
 from typing import Generator
 from typing import List
 from typing import Optional
+from typing import cast
 
 from aiohttp import FormData
 from aiohttp.multipart import MultipartWriter
 from aiohttp.test_utils import TestClient
 import pytest
+import yaml
 
 
 class DummyHandler(BaseHTTPRequestHandler):
@@ -79,13 +82,24 @@ def vcr_provider_map(dummy_server: HTTPServer) -> Generator[str, None, None]:
     yield provider_map
 
 
+@pytest.fixture
+def vcr_ignore_headers() -> Generator[str, None, None]:
+    yield "foo-bar,user-super-secret-api-key"
+
+
 @pytest.fixture
 async def vcr_test_name(agent: TestClient[Any, Any]) -> AsyncGenerator[None, None]:
     await agent.post("/vcr/test/start", json={"test_name": "test_name_prefix"})
     yield
     await agent.post("/vcr/test/stop")
 
 
+def get_recorded_request_from_yaml(file_path: str) -> Dict[str, Any]:
+    with open(file_path, "r") as file:
+        content = yaml.load(file, Loader=yaml.UnsafeLoader)
+    return cast(Dict[str, Any], content["interactions"][0])
+
+
 async def test_vcr_proxy_make_cassette(agent: TestClient[Any, Any], vcr_cassettes_directory: str) -> None:
     resp = await agent.post("/vcr/custom/serve", json={"foo": "bar"})
 
@@ -177,3 +191,32 @@ async def test_vcr_proxy_with_multipart_form_data(agent: TestClient[Any, Any], v
 
     cassette_files = get_cassettes_for_provider("custom", vcr_cassettes_directory)
     assert len(cassette_files) == 1
+
+
+async def test_vcr_proxy_does_not_record_ignored_headers(
+    agent: TestClient[Any, Any], vcr_cassettes_directory: str
+) -> None:
+    resp = await agent.post(
+        "/vcr/custom/serve",
+        json={"foo": "bar"},
+        headers={
+            "User-Super-Secret-Api-Key": "secret",
+            "Foo-Bar": "foo",
+            "Authorization": "test",
+            "Please-Record-Header": "test",
+        },
+    )
+
+    assert resp.status == 200
+    assert await resp.text() == "OK"
+
+    cassette_files = get_cassettes_for_provider("custom", vcr_cassettes_directory)
+    assert len(cassette_files) == 1
+
+    cassette_file = cassette_files[0]
+    recorded_request = get_recorded_request_from_yaml(os.path.join(vcr_cassettes_directory, "custom", cassette_file))
+
+    assert recorded_request["request"]["headers"]["Please-Record-Header"] == ["test"]
+    assert "User-Super-Secret-Api-Key" not in recorded_request["request"]["headers"]
+    assert "Foo-Bar" not in recorded_request["request"]["headers"]
+    assert "Authorization" not in recorded_request["request"]["headers"]