8238270: java.net HTTP/2 client does not decrease stream count when receives 204 response

The HTTP/2 Stream is updated to register a trivial data subscriber in case of 204 so that the END_STREAM is correctly processed.

Reviewed-by: chegar

src/java.net.http/share/classes/jdk/internal/net/http/ExchangeImpl.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -166,8 +166,8 @@ abstract class ExchangeImpl<T> {
 
     // Called for 204 response - when no body is permitted
     void nullBody(HttpResponse<T> resp, Throwable t) {
-        // only needed for HTTP/1.1 to close the connection
+        // Needed for HTTP/1.1 to close the connection or return it to the pool
-        // or return it to the pool
+        // Needed for HTTP/2 to subscribe a dummy subscriber and close the stream
     }
 
     /* The following methods have separate HTTP/1.1 and HTTP/2 implementations */

src/java.net.http/share/classes/jdk/internal/net/http/Stream.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -259,6 +259,14 @@ class Stream<T> extends ExchangeImpl<T> {
         }
     }
 
+    @Override
+    void nullBody(HttpResponse<T> resp, Throwable t) {
+        if (debug.on()) debug.log("nullBody: streamid=%d", streamid);
+        // We should have an END_STREAM data frame waiting in the inputQ.
+        // We need a subscriber to force the scheduler to process it.
+        pendingResponseSubscriber = HttpResponse.BodySubscribers.replacing(null);
+        sched.runOrSchedule();
+    }
 
     // Callback invoked after the Response BodySubscriber has consumed the
     // buffers contained in a DataFrame.
@@ -382,6 +390,7 @@ class Stream<T> extends ExchangeImpl<T> {
                 Log.logTrace("handling response (streamid={0})", streamid);
                 handleResponse();
                 if (hframe.getFlag(HeaderFrame.END_STREAM)) {
+                    if (debug.on()) debug.log("handling END_STREAM: %d", streamid);
                     receiveDataFrame(new DataFrame(streamid, DataFrame.END_STREAM, List.of()));
                 }
             }
@@ -688,6 +697,7 @@ class Stream<T> extends ExchangeImpl<T> {
 
     /** Sets endStreamReceived. Should be called only once. */
     void setEndStreamReceived() {
+        if (debug.on()) debug.log("setEndStreamReceived: streamid=%d", streamid);
         assert remotelyClosed == false: "Unexpected endStream already set";
         remotelyClosed = true;
         responseReceived();
@@ -1032,14 +1042,24 @@ class Stream<T> extends ExchangeImpl<T> {
     synchronized void requestSent() {
         requestSent = true;
         if (responseReceived) {
+            if (debug.on()) debug.log("requestSent: streamid=%d", streamid);
             close();
+        } else {
+            if (debug.on()) {
+                debug.log("requestSent: streamid=%d but response not received", streamid);
+            }
         }
     }
 
     synchronized void responseReceived() {
         responseReceived = true;
         if (requestSent) {
+            if (debug.on()) debug.log("responseReceived: streamid=%d", streamid);
             close();
+        } else {
+            if (debug.on()) {
+                debug.log("responseReceived: streamid=%d but request not sent", streamid);
+            }
         }
     }
 
@@ -1162,6 +1182,7 @@ class Stream<T> extends ExchangeImpl<T> {
             if (closed) return;
             closed = true;
         }
+        if (debug.on()) debug.log("close stream %d", streamid);
         Log.logTrace("Closing stream {0}", streamid);
         connection.closeStream(streamid);
         Log.logTrace("Stream {0} closed", streamid);

test/jdk/java/net/httpclient/Response204V2Test.java

@@ -0,0 +1,315 @@
+/*
+ * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test
+ * @bug 8238270
+ * @library /test/lib http2/server
+ * @build jdk.test.lib.net.SimpleSSLContext HttpServerAdapters
+ *       ReferenceTracker Response204V2Test
+ * @modules java.base/sun.net.www.http
+ *          java.net.http/jdk.internal.net.http.common
+ *          java.net.http/jdk.internal.net.http.frame
+ *          java.net.http/jdk.internal.net.http.hpack
+ * @run testng/othervm -Djdk.internal.httpclient.debug=true
+ *                     -Djdk.httpclient.HttpClient.log=requests,responses,errors
+ *                     Response204V2Test
+ * @summary Tests that streams are closed after receiving a 204 response.
+ *          This test uses the OperationsTracker and will fail in
+ *          teardown if the tracker reports that some HTTP/2 streams
+ *          are still open.
+ */
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.net.http.HttpClient;
+import java.net.http.HttpRequest;
+import java.net.http.HttpResponse;
+import java.net.http.HttpResponse.BodyHandlers;
+import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.Executor;
+import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicLong;
+import jdk.test.lib.net.SimpleSSLContext;
+import org.testng.ITestContext;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.AfterTest;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.BeforeTest;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+import javax.net.ssl.SSLContext;
+
+import static java.lang.System.out;
+
+public class Response204V2Test implements HttpServerAdapters {
+
+    SSLContext sslContext;
+    HttpTestServer http2TestServer;   // HTTP/2 ( h2c )
+    HttpTestServer https2TestServer;  // HTTP/2 ( h2  )
+    String http2URI;
+    String https2URI;
+
+    static final int RESPONSE_CODE = 204;
+    static final int ITERATION_COUNT = 4;
+    // a shared executor helps reduce the amount of threads created by the test
+    static final Executor executor = new TestExecutor(Executors.newCachedThreadPool());
+    static final ConcurrentMap<String, Throwable> FAILURES = new ConcurrentHashMap<>();
+    static volatile boolean tasksFailed;
+    static final AtomicLong serverCount = new AtomicLong();
+    static final AtomicLong clientCount = new AtomicLong();
+    static final long start = System.nanoTime();
+    public static String now() {
+        long now = System.nanoTime() - start;
+        long secs = now / 1000_000_000;
+        long mill = (now % 1000_000_000) / 1000_000;
+        long nan = now % 1000_000;
+        return String.format("[%d s, %d ms, %d ns] ", secs, mill, nan);
+    }
+
+    final ReferenceTracker TRACKER = ReferenceTracker.INSTANCE;
+    private volatile HttpClient sharedClient;
+
+    static class TestExecutor implements Executor {
+        final AtomicLong tasks = new AtomicLong();
+        Executor executor;
+        TestExecutor(Executor executor) {
+            this.executor = executor;
+        }
+
+        @Override
+        public void execute(Runnable command) {
+            long id = tasks.incrementAndGet();
+            executor.execute(() -> {
+                try {
+                    command.run();
+                } catch (Throwable t) {
+                    tasksFailed = true;
+                    System.out.printf(now() + "Task %s failed: %s%n", id, t);
+                    System.err.printf(now() + "Task %s failed: %s%n", id, t);
+                    FAILURES.putIfAbsent("Task " + id, t);
+                    throw t;
+                }
+            });
+        }
+    }
+
+    protected boolean stopAfterFirstFailure() {
+        return Boolean.getBoolean("jdk.internal.httpclient.debug");
+    }
+
+    @BeforeMethod
+    void beforeMethod(ITestContext context) {
+        if (stopAfterFirstFailure() && context.getFailedTests().size() > 0) {
+            throw new RuntimeException("some tests failed");
+        }
+    }
+
+    @AfterClass
+    static final void printFailedTests() {
+        out.println("\n=========================");
+        try {
+            out.printf("%n%sCreated %d servers and %d clients%n",
+                    now(), serverCount.get(), clientCount.get());
+            if (FAILURES.isEmpty()) return;
+            out.println("Failed tests: ");
+            FAILURES.entrySet().forEach((e) -> {
+                out.printf("\t%s: %s%n", e.getKey(), e.getValue());
+                e.getValue().printStackTrace(out);
+                e.getValue().printStackTrace();
+            });
+            if (tasksFailed) {
+                System.out.println("WARNING: Some tasks failed");
+            }
+        } finally {
+            out.println("\n=========================\n");
+        }
+    }
+
+    private String[] uris() {
+        return new String[] {
+                http2URI,
+                https2URI,
+        };
+    }
+
+    static AtomicLong URICOUNT = new AtomicLong();
+
+    @DataProvider(name = "variants")
+    public Object[][] variants(ITestContext context) {
+        if (stopAfterFirstFailure() && context.getFailedTests().size() > 0) {
+            return new Object[0][];
+        }
+        String[] uris = uris();
+        Object[][] result = new Object[uris.length * 2][];
+        int i = 0;
+        for (boolean sameClient : List.of(false, true)) {
+            for (String uri : uris()) {
+                result[i++] = new Object[]{uri, sameClient};
+            }
+        }
+        assert i == uris.length * 2;
+        return result;
+    }
+
+    private HttpClient makeNewClient() {
+        clientCount.incrementAndGet();
+        HttpClient client =  HttpClient.newBuilder()
+                .proxy(HttpClient.Builder.NO_PROXY)
+                .executor(executor)
+                .sslContext(sslContext)
+                .build();
+        return TRACKER.track(client);
+    }
+
+    HttpClient newHttpClient(boolean share) {
+        if (!share) return makeNewClient();
+        HttpClient shared = sharedClient;
+        if (shared != null) return shared;
+        synchronized (this) {
+            shared = sharedClient;
+            if (shared == null) {
+                shared = sharedClient = makeNewClient();
+            }
+            return shared;
+        }
+    }
+
+
+    static void checkStatus(int expected, int found) throws Exception {
+        if (expected != found) {
+            System.err.printf ("Test failed: wrong status code %d/%d\n",
+                expected, found);
+            throw new RuntimeException("Test failed");
+        }
+    }
+
+    static void checkStrings(String expected, String found) throws Exception {
+        if (!expected.equals(found)) {
+            System.err.printf ("Test failed: wrong string %s/%s\n",
+                expected, found);
+            throw new RuntimeException("Test failed");
+        }
+    }
+
+
+    @Test(dataProvider = "variants")
+    public void test(String uri, boolean sameClient) throws Exception {
+        System.out.println("Request to " + uri);
+
+        HttpClient client = newHttpClient(sameClient);
+
+        HttpRequest request = HttpRequest.newBuilder(URI.create(uri))
+                .GET()
+                .build();
+        for (int i = 0; i < ITERATION_COUNT; i++) {
+            System.out.println("Iteration: " + i);
+            HttpResponse<String> response = client.send(request, BodyHandlers.ofString());
+            int expectedResponse =  RESPONSE_CODE;
+            if (response.statusCode() != expectedResponse)
+                throw new RuntimeException("wrong response code " + Integer.toString(response.statusCode()));
+        }
+        System.out.println("test: DONE");
+    }
+
+    @BeforeTest
+    public void setup() throws Exception {
+        sslContext = new SimpleSSLContext().get();
+        if (sslContext == null)
+            throw new AssertionError("Unexpected null sslContext");
+
+        // HTTP/2
+        HttpTestHandler handler204 = new Handler204();
+
+        http2TestServer = HttpTestServer.of(new Http2TestServer("localhost", false, 0));
+        http2TestServer.addHandler(handler204, "/http2/test204/");
+        http2URI = "http://" + http2TestServer.serverAuthority() + "/http2/test204/x";
+
+        https2TestServer = HttpTestServer.of(new Http2TestServer("localhost", true, sslContext));
+        https2TestServer.addHandler(handler204, "/https2/test204/");
+        https2URI = "https://" + https2TestServer.serverAuthority() + "/https2/test204/x";
+
+        serverCount.addAndGet(4);
+        http2TestServer.start();
+        https2TestServer.start();
+    }
+
+    @AfterTest
+    public void teardown() throws Exception {
+        String sharedClientName =
+                sharedClient == null ? null : sharedClient.toString();
+        sharedClient = null;
+        Thread.sleep(100);
+        AssertionError fail = TRACKER.check(500);
+        try {
+            http2TestServer.stop();
+            https2TestServer.stop();
+        } finally {
+            if (fail != null) {
+                if (sharedClientName != null) {
+                    System.err.println("Shared client name is: " + sharedClientName);
+                }
+                throw fail;
+            }
+        }
+    }
+
+    static class Handler204 implements HttpTestHandler {
+
+        public Handler204() {}
+
+        volatile int invocation = 0;
+
+        @Override
+        public void handle(HttpTestExchange t)
+                throws IOException {
+            try {
+                URI uri = t.getRequestURI();
+                System.err.printf("Handler received request for %s\n", uri);
+                String type = uri.getScheme().toLowerCase();
+                InputStream is = t.getRequestBody();
+                while (is.read() != -1);
+                is.close();
+
+
+                if ((invocation++ % 2) == 1) {
+                    System.err.printf("Server sending %d - chunked\n", RESPONSE_CODE);
+                    t.sendResponseHeaders(RESPONSE_CODE, -1);
+                    OutputStream os = t.getResponseBody();
+                    os.close();
+                } else {
+                    System.err.printf("Server sending %d - 0 length\n", RESPONSE_CODE);
+                    t.sendResponseHeaders(RESPONSE_CODE, 0);
+                }
+            } catch (Throwable e) {
+                e.printStackTrace(System.err);
+                throw new IOException(e);
+            }
+        }
+    }
+}

test/jdk/java/net/httpclient/http2/server/Http2TestExchangeImpl.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -143,7 +143,7 @@ public class Http2TestExchangeImpl implements Http2TestExchange {
         response.setFlag(HeaderFrame.END_HEADERS);
 
 
-        if (responseLength < 0) {
+        if (responseLength < 0 || rCode == 204) {
             response.setFlag(HeadersFrame.END_STREAM);
             os.closeInternal();
         }