chore: Fix unexpected connection-pool shut-down of HttpClient4

cloudplatform/cloudplatform-connectivity/src/main/java/com/sap/cloud/sdk/cloudplatform/connectivity/DefaultHttpDestination.java

@@ -511,8 +511,7 @@ public static Builder fromDestination( @Nonnull final Destination destination )
             .getPropertyNames()
             .forEach(propertyName -> builder.property(propertyName, destination.get(propertyName).get()));
 
-        if( destination instanceof DefaultHttpDestination ) {
-            final DefaultHttpDestination httpDestination = (DefaultHttpDestination) destination;
+        if( destination instanceof DefaultHttpDestination httpDestination ) {
             builder.headers(httpDestination.customHeaders);
             builder
                 .headerProviders(httpDestination.getCustomHeaderProviders().toArray(new DestinationHeaderProvider[0]));
@@ -543,6 +542,7 @@ public boolean equals( @Nullable final Object o )
                 resolveCertificatesOnly(keyStoreSupplier.get().getOrNull()),
                 resolveCertificatesOnly(that.keyStoreSupplier.get().getOrNull()))
             .append(resolveCertificatesOnly(trustStore), resolveCertificatesOnly(that.trustStore))
+            .append(customHeaderProviders, that.customHeaderProviders)
             .isEquals();
     }
 
@@ -554,9 +554,28 @@ public int hashCode()
             .append(customHeaders)
             .append(resolveKeyStoreHashCode(keyStoreSupplier.get().getOrNull()))
             .append(resolveKeyStoreHashCode(trustStore))
+            .append(computeHeaderProvidersHashCode(customHeaderProviders))
             .toHashCode();
     }
 
+    /**
+     * Computes a hash code for the custom header providers list. Since header providers can be lambda functions that
+     * don't have meaningful equals/hashCode implementations, we use the identity hash code of each provider to uniquely
+     * identify them.
+     *
+     * @param providers
+     *            the list of header providers
+     * @return a hash code based on the identity of each provider
+     */
+    private static int computeHeaderProvidersHashCode( @Nonnull final List<DestinationHeaderProvider> providers )
+    {
+        int result = 0;
+        for( final DestinationHeaderProvider provider : providers ) {
+            result = 31 * result + System.identityHashCode(provider);
+        }
+        return result;
+    }
+
     /**
      * Builder class to allow for easy creation of an immutable {@code DefaultHttpDestination} instance.
      */

cloudplatform/connectivity-apache-httpclient4/src/main/java/com/sap/cloud/sdk/cloudplatform/connectivity/HttpClientWrapper.java

@@ -93,9 +93,9 @@ public String toString()
 
     HttpClientWrapper withDestination( final HttpDestinationProperties destination )
     {
-        // explicitly check the reference equality, since equals doesn't check header providers
-        // this is a slight improvement, avoiding unnecessary wrapper instantiation
-        // in cases where destination objects are reused / served from cache
+        // Since equals() now includes customHeaderProviders in the equality check,
+        // this method will throw an exception if the destination has different header providers.
+        // This is the expected behavior to ensure HTTP clients are properly isolated by their configuration.
         if( !destination.equals(this.destination) ) {
             throw new ShouldNotHappenException(
                 "This method must not be used outside of updating an instance of HttpClientWrapper for http clients served from the HttpClientCache.");

cloudplatform/connectivity-apache-httpclient4/src/test/java/com/sap/cloud/sdk/cloudplatform/connectivity/DefaultHttpClientCacheTest.java

@@ -28,6 +28,8 @@
 import com.sap.cloud.sdk.cloudplatform.tenant.Tenant;
 import com.sap.cloud.sdk.testutil.TestContext;
 
+import lombok.SneakyThrows;
+
 @Isolated
 class DefaultHttpClientCacheTest
 {
@@ -179,8 +181,7 @@ void testGetClientWithDestinationUsesTenantOptionalForIsolation()
     }
 
     @Test
-    //This is a known limitation of excluding header providers in the equality check of destinations
-    void testGetClientReturnsSameClientForDestinationsWithOnlyDifferentHeaderProviders()
+    void testGetClientReturnsDifferentClientForDestinationsWithDifferentHeaderProviders()
     {
         final Header header1 = new Header("foo", "bar");
         final Header header2 = new Header("foo1", "bar1");
@@ -193,25 +194,28 @@ void testGetClientReturnsSameClientForDestinationsWithOnlyDifferentHeaderProvide
 
         final DefaultHttpDestination secondDestination =
             DefaultHttpDestination
-                .fromDestination(firstDestination)
+                .builder("http://some-uri")
                 .headerProviders(( any ) -> Collections.singletonList(header2))
                 .build();
 
+        // Verify that destinations with different header providers are not equal
+        assertThat(firstDestination).isNotEqualTo(secondDestination);
+
         final HttpClientWrapper client1 = (HttpClientWrapper) sut.tryGetHttpClient(firstDestination, FACTORY).get();
         final HttpClientWrapper client2 = (HttpClientWrapper) sut.tryGetHttpClient(secondDestination, FACTORY).get();
 
         assertThat(client1.getDestination()).isSameAs(firstDestination);
         assertThat(client2.getDestination()).isSameAs(secondDestination);
 
+        // Each client should be a distinct instance now
+        assertThat(client1).isNotSameAs(client2);
+
         final HttpUriRequest request1 = client1.wrapRequest(new HttpGet());
         final HttpUriRequest request2 = client2.wrapRequest(new HttpGet());
 
-        // This behavior is to be improved by https://github.com/SAP/cloud-sdk-java-backlog/issues/396
+        // Each destination's header provider should only add its own headers
         assertThat(request1.getAllHeaders()).containsExactly(new HttpClientWrapper.ApacheHttpHeader(header1));
-        assertThat(request2.getAllHeaders())
-            .containsExactly(
-                new HttpClientWrapper.ApacheHttpHeader(header1),
-                new HttpClientWrapper.ApacheHttpHeader(header2));
+        assertThat(request2.getAllHeaders()).containsExactly(new HttpClientWrapper.ApacheHttpHeader(header2));
     }
 
     @Test
@@ -357,6 +361,41 @@ void testInvalidatePrincipalCacheEntriesWithUserTokenExchangeDestination()
         assertThat(unclearedClientWithoutDestination).isSameAs(sut.tryGetHttpClient(FACTORY).get());
     }
 
+    @SneakyThrows
+    @Test
+    void testCachedEqualHttpClientsClosingBehavior()
+    {
+        final DefaultHttpDestination destination1 = DefaultHttpDestination.builder("http://foo.com").build();
+        final HttpClient client1 = sut.tryGetHttpClient(destination1, FACTORY).get();
+        assertThat(((HttpClientWrapper) client1).getDestination()).isSameAs(destination1);
+
+        final DefaultHttpDestination destination2 =
+            DefaultHttpDestination
+                .builder("http://foo.com")
+                .headerProviders(c -> List.of(new Header("Authorization", "Bearer foo")))
+                .build();
+        final HttpClient client2 = sut.tryGetHttpClient(destination2, FACTORY).get();
+        assertThat(((HttpClientWrapper) client2).getDestination()).isSameAs(destination2);
+
+        // Verify the destinations are not equal due to different header providers
+        assertThat(destination1).isNotEqualTo(destination2); // header providers are now included in the equality check
+        assertThat(destination1).isNotSameAs(destination2);
+
+        // Http clients are distinct instances, since the cache key contains the destination reference and not its content
+        assertThat(client1).isNotSameAs(client2);
+
+        // When using the exact same destination object, the same http-client wrapper should be returned
+        final HttpClient client1Again = sut.tryGetHttpClient(destination1, FACTORY).get();
+        assertThat(client1Again).isSameAs(client1);
+        assertThat(((HttpClientWrapper) client1Again).getDestination()).isSameAs(destination1);
+
+        // simulate garbage collection on client1
+        ((HttpClientWrapper) client1).close();
+
+        // since client1 did not inherit client2 connection manager, client2 is not shut down
+        client2.execute(new HttpGet());
+    }
+
     @Test
     void testPrincipalPropagationIsPrincipalIsolated()
     {

cloudplatform/connectivity-apache-httpclient4/src/test/java/com/sap/cloud/sdk/cloudplatform/connectivity/HttpClientWrapperTest.java

@@ -22,9 +22,13 @@ void testDestinationWrapping()
         final DefaultHttpDestination thirdDestination = DefaultHttpDestination.builder("http://bar.com").build();
         final HttpClientWrapper sut = new HttpClientWrapper(mock(CloseableHttpClient.class), firstDestination);
 
+        // withDestination returns the same wrapper instance when the destination reference is identical
         assertThat(sut.withDestination(firstDestination)).isSameAs(sut);
-        assertThat(sut.withDestination(firstDestination)).isNotSameAs(sut.withDestination(secondDestination));
 
+        // withDestination throws an exception when destinations are not equal (different header providers)
+        assertThatThrownBy(() -> sut.withDestination(secondDestination)).isInstanceOf(ShouldNotHappenException.class);
+
+        // withDestination throws an exception when destinations have different URIs
         assertThatThrownBy(() -> sut.withDestination(thirdDestination)).isInstanceOf(ShouldNotHappenException.class);
     }
 }

cloudplatform/connectivity-apache-httpclient5/src/main/java/com/sap/cloud/sdk/cloudplatform/connectivity/ApacheHttpClient5Wrapper.java

@@ -78,9 +78,9 @@ public void close( final CloseMode closeMode )
 
     ApacheHttpClient5Wrapper withDestination( final HttpDestinationProperties destination )
     {
-        // explicitly check the reference equality, since equals doesn't check header providers
-        // this is a slight improvement, avoiding unnecessary wrapper instantiation
-        // in cases where destination objects are reused / served from cache
+        // Since equals() now includes customHeaderProviders in the equality check,
+        // this method will throw an exception if the destination has different header providers.
+        // This is the expected behavior to ensure HTTP clients are properly isolated by their configuration.
         if( !destination.equals(this.destination) ) {
             throw new ShouldNotHappenException(
                 "This method must not be used outside of updating an instance of ApacheHttpClient5Wrapper for http clients served from the ApacheHttpClient5Cache.");

cloudplatform/connectivity-apache-httpclient5/src/test/java/com/sap/cloud/sdk/cloudplatform/connectivity/DefaultApacheHttpClient5CacheTest.java

@@ -348,8 +348,7 @@ void testInvalidatePrincipalCacheEntriesWithUserTokenExchangeDestination()
     }
 
     @Test
-    //This is a known limitation of excluding header providers in the equality check of destinations
-    void testGetClientReturnsSameClientForDestinationsWithOnlyDifferentHeaderProviders()
+    void testGetClientReturnsDifferentClientForDestinationsWithDifferentHeaderProviders()
     {
         final Header header1 = new Header("foo", "bar");
         final Header header2 = new Header("foo1", "bar1");
@@ -362,10 +361,13 @@ void testGetClientReturnsSameClientForDestinationsWithOnlyDifferentHeaderProvide
 
         final DefaultHttpDestination secondDestination =
             DefaultHttpDestination
-                .fromDestination(firstDestination)
+                .builder("http://some-uri")
                 .headerProviders(( any ) -> Collections.singletonList(header2))
                 .build();
 
+        // Verify that destinations with different header providers are not equal
+        assertThat(firstDestination).isNotEqualTo(secondDestination);
+
         final ApacheHttpClient5Wrapper client1 =
             (ApacheHttpClient5Wrapper) sut.tryGetHttpClient(firstDestination, FACTORY).get();
         final ApacheHttpClient5Wrapper client2 =
@@ -374,6 +376,9 @@ void testGetClientReturnsSameClientForDestinationsWithOnlyDifferentHeaderProvide
         assertThat(client1.getDestination()).isSameAs(firstDestination);
         assertThat(client2.getDestination()).isSameAs(secondDestination);
 
+        // Each client should be a distinct instance now
+        assertThat(client1).isNotSameAs(client2);
+
         final ClassicHttpRequest request1 = client1.wrapRequest(new HttpGet("/"));
         final ClassicHttpRequest request2 = client2.wrapRequest(new HttpGet("/"));
 
@@ -382,15 +387,13 @@ void testGetClientReturnsSameClientForDestinationsWithOnlyDifferentHeaderProvide
         request1.headerIterator().forEachRemaining(headersRequest1::add);
         request2.headerIterator().forEachRemaining(headersRequest2::add);
 
-        // recursive comparison because BasicHeader doesn't implement equals/hashCode
+        // Each destination's header provider should only add its own headers
         assertThat(headersRequest1)
             .usingRecursiveFieldByFieldElementComparator()
             .containsExactly(new BasicHeader(header1.getName(), header1.getValue()));
         assertThat(headersRequest2)
             .usingRecursiveFieldByFieldElementComparator()
-            .containsExactly(
-                new BasicHeader(header1.getName(), header1.getValue()),
-                new BasicHeader(header2.getName(), header2.getValue()));
+            .containsExactly(new BasicHeader(header2.getName(), header2.getValue()));
     }
 
     @Test

cloudplatform/connectivity-apache-httpclient5/src/test/java/com/sap/cloud/sdk/cloudplatform/connectivity/HttpClientWrapperTest.java

@@ -24,9 +24,13 @@ void testDestinationWrapping()
         final ApacheHttpClient5Wrapper sut =
             new ApacheHttpClient5Wrapper(mock(CloseableHttpClient.class), firstDestination, mock(RequestConfig.class));
 
+        // withDestination returns the same wrapper instance when the destination reference is identical
         assertThat(sut.withDestination(firstDestination)).isSameAs(sut);
-        assertThat(sut.withDestination(firstDestination)).isNotSameAs(sut.withDestination(secondDestination));
 
+        // withDestination throws an exception when destinations are not equal (different header providers)
+        assertThatThrownBy(() -> sut.withDestination(secondDestination)).isInstanceOf(ShouldNotHappenException.class);
+
+        // withDestination throws an exception when destinations have different URIs
         assertThatThrownBy(() -> sut.withDestination(thirdDestination)).isInstanceOf(ShouldNotHappenException.class);
     }
 }