From dc8bc6c98ca1f9b441cf71c641675fe29dda9162 Mon Sep 17 00:00:00 2001 From: Severin Gehwolf Date: Fri, 2 Jun 2023 08:58:20 +0000 Subject: [PATCH] 8308090: Add container tests for on-the-fly resource quota updates Reviewed-by: dholmes, mseledtsov --- .../containers/docker/LimitUpdateChecker.java | 57 +++++++ .../containers/docker/TestLimitsUpdating.java | 148 +++++++++++++++++ .../platform/docker/LimitUpdateChecker.java | 64 +++++++ .../platform/docker/TestLimitsUpdating.java | 156 ++++++++++++++++++ .../containers/docker/DockerTestUtils.java | 10 +- 5 files changed, 432 insertions(+), 3 deletions(-) create mode 100644 test/hotspot/jtreg/containers/docker/LimitUpdateChecker.java create mode 100644 test/hotspot/jtreg/containers/docker/TestLimitsUpdating.java create mode 100644 test/jdk/jdk/internal/platform/docker/LimitUpdateChecker.java create mode 100644 test/jdk/jdk/internal/platform/docker/TestLimitsUpdating.java diff --git a/test/hotspot/jtreg/containers/docker/LimitUpdateChecker.java b/test/hotspot/jtreg/containers/docker/LimitUpdateChecker.java new file mode 100644 index 00000000000..f8c971b4817 --- /dev/null +++ b/test/hotspot/jtreg/containers/docker/LimitUpdateChecker.java @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2023, Red Hat, Inc. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +import java.io.File; +import java.io.FileOutputStream; +import jdk.test.whitebox.WhiteBox; + +// Check dynamic limits updating. HotSpot side. +public class LimitUpdateChecker { + + private static final File UPDATE_FILE = new File("/tmp", "limitsUpdated"); + private static final File STARTED_FILE = new File("/tmp", "started"); + + public static void main(String[] args) throws Exception { + System.out.println("LimitUpdateChecker: Entering"); + WhiteBox wb = WhiteBox.getWhiteBox(); + printMetrics(wb); // print initial limits + createStartedFile(); + while (!UPDATE_FILE.exists()) { + Thread.sleep(200); + } + System.out.println("'limitsUpdated' file appeared. Stopped loop."); + printMetrics(wb); // print limits after update + System.out.println("LimitUpdateChecker DONE."); + + } + + private static void printMetrics(WhiteBox wb) { + wb.printOsInfo(); + } + + private static void createStartedFile() throws Exception { + FileOutputStream fout = new FileOutputStream(STARTED_FILE); + fout.close(); + } +} diff --git a/test/hotspot/jtreg/containers/docker/TestLimitsUpdating.java b/test/hotspot/jtreg/containers/docker/TestLimitsUpdating.java new file mode 100644 index 00000000000..e15ab9b2b81 --- /dev/null +++ b/test/hotspot/jtreg/containers/docker/TestLimitsUpdating.java @@ -0,0 +1,148 @@ +/* + * Copyright (c) 2023, Red Hat, Inc. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +/* + * @test + * @bug 8308090 + * @key cgroups + * @summary Test container limits updating as they get updated at runtime without restart + * @requires docker.support + * @library /test/lib + * @build jdk.test.whitebox.WhiteBox LimitUpdateChecker + * @run driver jdk.test.lib.helpers.ClassFileInstaller -jar whitebox.jar jdk.test.whitebox.WhiteBox + * @run driver TestLimitsUpdating + */ + +import java.io.File; +import java.io.FileOutputStream; +import java.util.List; +import java.util.regex.Pattern; +import java.util.regex.Matcher; +import jdk.test.lib.Asserts; +import jdk.test.lib.Utils; +import jdk.test.lib.containers.docker.Common; +import jdk.test.lib.containers.docker.DockerRunOptions; +import jdk.test.lib.containers.docker.DockerTestUtils; +import jdk.test.lib.process.OutputAnalyzer; + +public class TestLimitsUpdating { + private static final String TARGET_CONTAINER = "limitsUpdatingHS_" + Runtime.getRuntime().version().major(); + private static final String imageName = Common.imageName("limitsUpdating"); + + public static void main(String[] args) throws Exception { + if (!DockerTestUtils.canTestDocker()) { + return; + } + + Common.prepareWhiteBox(); + DockerTestUtils.buildJdkContainerImage(imageName); + + try { + testLimitUpdates(); + } finally { + if (!DockerTestUtils.RETAIN_IMAGE_AFTER_TEST) { + DockerTestUtils.removeDockerImage(imageName); + } + } + } + + private static void testLimitUpdates() throws Exception { + File sharedtmpdir = new File("test-sharedtmp"); + File flag = new File(sharedtmpdir, "limitsUpdated"); // shared with LimitUpdateChecker + File started = new File(sharedtmpdir, "started"); // shared with LimitUpdateChecker + sharedtmpdir.mkdir(); + flag.delete(); + started.delete(); + DockerRunOptions opts = new DockerRunOptions(imageName, "/jdk/bin/java", "LimitUpdateChecker"); + opts.addDockerOpts("--volume", Utils.TEST_CLASSES + ":/test-classes/"); + opts.addDockerOpts("--volume", sharedtmpdir.getAbsolutePath() + ":/tmp"); + opts.addDockerOpts("--cpu-period", "100000"); + opts.addDockerOpts("--cpu-quota", "200000"); + opts.addDockerOpts("--memory", "500m"); + opts.addDockerOpts("--memory-swap", "500m"); + opts.addDockerOpts("--name", TARGET_CONTAINER); + opts.addJavaOpts("-cp", "/test-classes/"); + Common.addWhiteBoxOpts(opts); + final OutputAnalyzer out[] = new OutputAnalyzer[1]; + Thread t1 = new Thread() { + public void run() { + try { + out[0] = DockerTestUtils.dockerRunJava(opts).shouldHaveExitValue(0); + } catch (Exception e) { + e.printStackTrace(); + } + } + }; + t1.start(); + + // Wait for target container (that we later update) to complete its + // initial starting-up phase. Prints initial container limits. + while (!started.exists()) { + System.out.println("Wait for target container to start"); + Thread.sleep(100); + } + + final List containerCommand = getContainerUpdate(300_000, 100_000, "300m"); + // Run the update command so as to increase resources once the container signaled it has started. + Thread t2 = new Thread() { + public void run() { + try { + DockerTestUtils.execute(containerCommand).shouldHaveExitValue(0); + } catch (Exception e) { + e.printStackTrace(); + } + } + }; + t2.start(); + t2.join(); + + // Set the flag for the to-get updated container, indicating the update + // has completed. + FileOutputStream fout = new FileOutputStream(flag); + fout.close(); + + t1.join(); + + // Do assertions based on the output in target container + OutputAnalyzer targetOut = out[0]; + targetOut.shouldContain("active_processor_count: 2"); // initial value + targetOut.shouldContain("active_processor_count: 3"); // updated value + targetOut.shouldContain("memory_limit_in_bytes: 512000 k"); // initial value + targetOut.shouldContain("memory_and_swap_limit_in_bytes: 512000 k"); // initial value + targetOut.shouldContain("memory_limit_in_bytes: 307200 k"); // updated value + targetOut.shouldContain("memory_and_swap_limit_in_bytes: 307200 k"); // updated value + } + + private static List getContainerUpdate(int cpuQuota, int cpuPeriod, String memory) { + List cmd = DockerTestUtils.buildContainerCommand(); + cmd.add("update"); + cmd.add("--cpu-period=" + cpuPeriod); + cmd.add("--cpu-quota=" + cpuQuota); + cmd.add("--memory=" + memory); + cmd.add("--memory-swap=" + memory); // no swap + cmd.add(TARGET_CONTAINER); + return cmd; + } +} diff --git a/test/jdk/jdk/internal/platform/docker/LimitUpdateChecker.java b/test/jdk/jdk/internal/platform/docker/LimitUpdateChecker.java new file mode 100644 index 00000000000..cb570b757cd --- /dev/null +++ b/test/jdk/jdk/internal/platform/docker/LimitUpdateChecker.java @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2023, Red Hat Inc. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +import java.io.File; +import java.io.FileOutputStream; +import com.sun.management.OperatingSystemMXBean; +import java.lang.management.ManagementFactory; +import jdk.internal.platform.Metrics; + + +// Check dynamic limits updating. Metrics (java) side. +public class LimitUpdateChecker { + + private static final File UPDATE_FILE = new File("/tmp", "limitsUpdated"); + private static final File STARTED_FILE = new File("/tmp", "started"); + + public static void main(String[] args) throws Exception { + System.out.println("Running LimitUpdateChecker..."); + Metrics metrics = jdk.internal.platform.Container.metrics(); + OperatingSystemMXBean osBean = (OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean(); + printMetrics(osBean, metrics); // initial limits + createStartedFile(); + while (!UPDATE_FILE.exists()) { + Thread.sleep(200); + } + System.out.println("'limitsUpdated' file appeared. Stopped loop."); + printMetrics(osBean, metrics); // updated limits + System.out.println("LimitUpdateChecker DONE."); + } + + private static void printMetrics(OperatingSystemMXBean osBean, Metrics metrics) { + System.out.println(String.format("Runtime.availableProcessors: %d", Runtime.getRuntime().availableProcessors())); + System.out.println(String.format("OperatingSystemMXBean.getAvailableProcessors: %d", osBean.getAvailableProcessors())); + System.out.println("Metrics.getMemoryLimit() == " + metrics.getMemoryLimit()); + System.out.println(String.format("OperatingSystemMXBean.getTotalMemorySize: %d", osBean.getTotalMemorySize())); + } + + private static void createStartedFile() throws Exception { + FileOutputStream fout = new FileOutputStream(STARTED_FILE); + fout.close(); + } + +} diff --git a/test/jdk/jdk/internal/platform/docker/TestLimitsUpdating.java b/test/jdk/jdk/internal/platform/docker/TestLimitsUpdating.java new file mode 100644 index 00000000000..22e03293c48 --- /dev/null +++ b/test/jdk/jdk/internal/platform/docker/TestLimitsUpdating.java @@ -0,0 +1,156 @@ +/* + * Copyright (c) 2023, Red Hat, Inc. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +/* + * @test + * @bug 8308090 + * @key cgroups + * @summary Test container limits updating as they get updated at runtime without restart + * @requires docker.support + * @library /test/lib + * @modules java.base/jdk.internal.platform + * @build LimitUpdateChecker + * @run driver TestLimitsUpdating + */ + +import java.io.File; +import java.io.FileOutputStream; +import java.util.List; +import java.util.regex.Pattern; +import java.util.regex.Matcher; +import jdk.test.lib.Asserts; +import jdk.test.lib.Utils; +import jdk.test.lib.containers.docker.Common; +import jdk.test.lib.containers.docker.DockerRunOptions; +import jdk.test.lib.containers.docker.DockerTestUtils; +import jdk.test.lib.process.OutputAnalyzer; + +public class TestLimitsUpdating { + private static final long M = 1024 * 1024; + private static final String TARGET_CONTAINER = "limitsUpdatingJDK_" + Runtime.getRuntime().version().major(); + private static final String imageName = Common.imageName("limitsUpdatingJDK"); + + public static void main(String[] args) throws Exception { + if (!DockerTestUtils.canTestDocker()) { + return; + } + + DockerTestUtils.buildJdkContainerImage(imageName); + + try { + testLimitUpdates(); + } finally { + if (!DockerTestUtils.RETAIN_IMAGE_AFTER_TEST) { + DockerTestUtils.removeDockerImage(imageName); + } + } + } + + private static void testLimitUpdates() throws Exception { + File sharedtmpdir = new File("jdk-sharedtmp"); + File flag = new File(sharedtmpdir, "limitsUpdated"); // shared with LimitUpdateChecker + File started = new File(sharedtmpdir, "started"); // shared with LimitUpdateChecker + sharedtmpdir.mkdir(); + flag.delete(); + started.delete(); + DockerRunOptions opts = new DockerRunOptions(imageName, "/jdk/bin/java", "LimitUpdateChecker"); + opts.addDockerOpts("--volume", Utils.TEST_CLASSES + ":/test-classes/"); + opts.addDockerOpts("--volume", sharedtmpdir.getAbsolutePath() + ":/tmp"); + opts.addDockerOpts("--cpu-period", "100000"); + opts.addDockerOpts("--cpu-quota", "200000"); + opts.addDockerOpts("--memory", "500m"); + opts.addDockerOpts("--memory-swap", "500m"); + opts.addDockerOpts("--name", TARGET_CONTAINER); + opts.addJavaOpts("-cp", "/test-classes/"); + // LimitUpdateChecker uses Metrics (jdk.internal.platform) for + // printing JDK container limits + opts.addJavaOpts("--add-exports"); + opts.addJavaOpts("java.base/jdk.internal.platform=ALL-UNNAMED"); + final OutputAnalyzer out[] = new OutputAnalyzer[1]; + Thread t1 = new Thread() { + public void run() { + try { + out[0] = DockerTestUtils.dockerRunJava(opts).shouldHaveExitValue(0); + } catch (Exception e) { + e.printStackTrace(); + } + } + }; + t1.start(); + + // Wait for target container (that we later update) to complete its + // initial starting-up phase. Prints initial container limits using + // OS MXBean and Metrics API + while (!started.exists()) { + System.out.println("Wait for target container to start"); + Thread.sleep(100); + } + + final List containerCommand = getContainerUpdate(300_000, 100_000, "300m"); + // Run the update command so as to increase resources once the container signaled it has started. + Thread t2 = new Thread() { + public void run() { + try { + DockerTestUtils.execute(containerCommand).shouldHaveExitValue(0); + } catch (Exception e) { + e.printStackTrace(); + } + } + }; + t2.start(); + t2.join(); + + // Set the flag for the to-get updated container, indicating the update + // has completed. + FileOutputStream fout = new FileOutputStream(flag); + fout.close(); + + t1.join(); + + // Do assertions based on the output in target container + OutputAnalyzer targetOut = out[0]; + targetOut.shouldContain("Runtime.availableProcessors: 2"); // initial value + targetOut.shouldContain("OperatingSystemMXBean.getAvailableProcessors: 2"); // initial value + targetOut.shouldContain("Runtime.availableProcessors: 3"); // updated value + targetOut.shouldContain("OperatingSystemMXBean.getAvailableProcessors: 3"); // updated value + long memoryInBytes = 500 * M; + targetOut.shouldContain("Metrics.getMemoryLimit() == " + memoryInBytes); // initial value + targetOut.shouldContain("OperatingSystemMXBean.getTotalMemorySize: " + memoryInBytes); // initial value + long updatedValue = 300 * M; + targetOut.shouldContain("Metrics.getMemoryLimit() == " + updatedValue); // updated value + targetOut.shouldContain("OperatingSystemMXBean.getTotalMemorySize: " + updatedValue); // updated value + } + + private static List getContainerUpdate(int cpuQuota, int cpuPeriod, String memory) { + List cmd = DockerTestUtils.buildContainerCommand(); + cmd.add("update"); + cmd.add("--cpu-period=" + cpuPeriod); + cmd.add("--cpu-quota=" + cpuQuota); + cmd.add("--memory=" + memory); + cmd.add("--memory-swap=" + memory); // no swap + cmd.add(TARGET_CONTAINER); + return cmd; + } +} diff --git a/test/lib/jdk/test/lib/containers/docker/DockerTestUtils.java b/test/lib/jdk/test/lib/containers/docker/DockerTestUtils.java index d3db6cd022d..2404394690e 100644 --- a/test/lib/jdk/test/lib/containers/docker/DockerTestUtils.java +++ b/test/lib/jdk/test/lib/containers/docker/DockerTestUtils.java @@ -201,9 +201,7 @@ public class DockerTestUtils { * @throws Exception */ public static List buildJavaCommand(DockerRunOptions opts) throws Exception { - List cmd = new ArrayList<>(); - - cmd.add(Container.ENGINE_COMMAND); + List cmd = buildContainerCommand(); cmd.add("run"); if (opts.tty) cmd.add("--tty=true"); @@ -226,6 +224,12 @@ public class DockerTestUtils { return cmd; } + public static List buildContainerCommand() { + List cmd = new ArrayList<>(); + cmd.add(Container.ENGINE_COMMAND); + return cmd; + } + /** * Run Java inside the docker image with specified parameters and options. *